0: ch1=8074 mean# 8074 dev# 0 part# 0
1: ch1=8074 ch2=8074 mean# 8074 dev# 0 part# 0
2: ch1=13338 ch2=10884 mean# 12111 dev# 1227 part# 0.1013128561
3: ch1=18148 ch2=14148 mean# 16148 dev# 2000 part# 0.1238543473
4: ch1=20188 ch2=20182 mean# 20185 dev# 3 part# 0.0001486252167
5: ch1=27764 ch2=20680 mean# 24222 dev# 3542 part# 0.1462306994
6: ch1=28222 ch2=28296 mean# 28259 dev# 37 part# 0.001309317386
7: ch1=34120 ch2=30472 mean# 32296 dev# 1824 part# 0.05647758236
8: ch1=37842 ch2=34824 mean# 36333 dev# 1509 part# 0.04153249112
9: ch1=40380 ch2=40360 mean# 40370 dev# 10 part# 0.0002477086946
10: ch1=47458 ch2=41356 mean# 44407 dev# 3051 part# 0.06870538429
11: ch1=49004 ch2=47884 mean# 48444 dev# 560 part# 0.01155973908
12: ch1=53814 ch2=51148 mean# 52481 dev# 1333 part# 0.02539966845
13: ch1=58624 ch2=54412 mean# 56518 dev# 2106 part# 0.03726246506
14: ch1=60572 ch2=60538 mean# 60555 dev# 17 part# 0.0002807365205
15: ch1=68240 ch2=60944 mean# 64592 dev# 3648 part# 0.05647758236
16: ch1=68698 ch2=68560 mean# 68629 dev# 69 part# 0.001005405878
17: ch1=74596 ch2=70736 mean# 72666 dev# 1930 part# 0.0265598767
18: ch1=78318 ch2=75088 mean# 76703 dev# 1615 part# 0.02105523904
19: ch1=80764 ch2=80716 mean# 80740 dev# 24 part# 0.0002972504335
20: ch1=87934 ch2=81620 mean# 84777 dev# 3157 part# 0.03723887375
21: ch1=89480 ch2=88148 mean# 88814 dev# 666 part# 0.007498817754
22: ch1=94290 ch2=91412 mean# 92851 dev# 1439 part# 0.01549794833
23: ch1=99100 ch2=94676 mean# 96888 dev# 2212 part# 0.02283048468
24: ch1=100956 ch2=100894 mean# 100925 dev# 31 part# 0.0003071587813
25: ch1=108716 ch2=101208 mean# 104962 dev# 3754 part# 0.03576532459
26: ch1=109174 ch2=108824 mean# 108999 dev# 175 part# 0.001605519317
27: ch1=113984 ch2=112088 mean# 113036 dev# 948 part# 0.008386708659
28: ch1=119882 ch2=114264 mean# 117073 dev# 2809 part# 0.02399357666
29: ch1=121148 ch2=121072 mean# 121110 dev# 38 part# 0.0003137643465
30: ch1=128410 ch2=121884 mean# 125147 dev# 3263 part# 0.02607333775
31: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
32: ch1=126692 ch2=123602 mean# 125147 dev# 1545 part# 0.01234548171
33: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
34: ch1=123192 ch2=127102 mean# 125147 dev# 1955 part# 0.01562162896
35: ch1=129004 ch2=121290 mean# 125147 dev# 3857 part# 0.03081975597
36: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
37: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
38: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
39: ch1=123686 ch2=126608 mean# 125147 dev# 1461 part# 0.01167427106
40: ch1=128502 ch2=121792 mean# 125147 dev# 3355 part# 0.02680847324
41: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
42: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
43: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
44: ch1=123096 ch2=127198 mean# 125147 dev# 2051 part# 0.01638872686
45: ch1=129092 ch2=121202 mean# 125147 dev# 3945 part# 0.03152292904
46: ch1=121424 ch2=128870 mean# 125147 dev# 3723 part# 0.02974901516
47: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
48: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
49: ch1=123590 ch2=126704 mean# 125147 dev# 1557 part# 0.01244136895
50: ch1=128590 ch2=121704 mean# 125147 dev# 3443 part# 0.0275116463
51: ch1=121878 ch2=128416 mean# 125147 dev# 3269 part# 0.02612128137
52: ch1=126230 ch2=124064 mean# 125147 dev# 1083 part# 0.008653823104
53: ch1=125142 ch2=125152 mean# 125147 dev# 5 part# 3.995301525e-05
54: ch1=123000 ch2=127244 mean# 125122 dev# 2122 part# 0.01695944758
55: ch1=128046 ch2=122198 mean# 125122 dev# 2924 part# 0.02336919167
56: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
57: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
58: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
59: ch1=122410 ch2=127834 mean# 125122 dev# 2712 part# 0.02167484535
60: ch1=128636 ch2=121608 mean# 125122 dev# 3514 part# 0.02808458944
61: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
62: ch1=126184 ch2=124060 mean# 125122 dev# 1062 part# 0.008487715989
63: ch1=125096 ch2=125148 mean# 125122 dev# 26 part# 0.0002077971899
64: ch1=122908 ch2=127336 mean# 125122 dev# 2214 part# 0.01769472994
65: ch1=128138 ch2=122106 mean# 125122 dev# 3016 part# 0.02410447403
66: ch1=121832 ch2=128412 mean# 125122 dev# 3290 part# 0.02629433673
67: ch1=126642 ch2=123602 mean# 125122 dev# 1520 part# 0.01214814341
68: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
69: ch1=122322 ch2=127922 mean# 125122 dev# 2800 part# 0.02237815892
70: ch1=128732 ch2=121512 mean# 125122 dev# 3610 part# 0.0288518406
71: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
72: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
73: ch1=126188 ch2=124056 mean# 125122 dev# 1066 part# 0.008519684788
74: ch1=123908 ch2=126336 mean# 125122 dev# 1214 part# 0.00970253033
75: ch1=128234 ch2=122010 mean# 125122 dev# 3112 part# 0.0248717252
76: ch1=121836 ch2=128408 mean# 125122 dev# 3286 part# 0.02626236793
77: ch1=126646 ch2=123598 mean# 125122 dev# 1524 part# 0.01218011221
78: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
79: ch1=123322 ch2=126922 mean# 125122 dev# 1800 part# 0.0143859593
80: ch1=128828 ch2=121416 mean# 125122 dev# 3706 part# 0.02961909177
81: ch1=121840 ch2=128404 mean# 125122 dev# 3282 part# 0.02623039913
82: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
83: ch1=126192 ch2=124052 mean# 125122 dev# 1070 part# 0.008551653586
84: ch1=123820 ch2=126424 mean# 125122 dev# 1302 part# 0.0104058439
85: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
86: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
87: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
88: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
89: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
90: ch1=128920 ch2=121374 mean# 125147 dev# 3773 part# 0.03014854531
91: ch1=121428 ch2=128866 mean# 125147 dev# 3719 part# 0.02971705275
92: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
93: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
94: ch1=123770 ch2=126524 mean# 125147 dev# 1377 part# 0.0110030604
95: ch1=128418 ch2=121876 mean# 125147 dev# 3271 part# 0.02613726258
96: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
97: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
98: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
99: ch1=123180 ch2=127114 mean# 125147 dev# 1967 part# 0.0157175162
100: ch1=126832 ch2=122374 ch3=1088 mean# 83431.33333 dev# 58253.9661 part# 0.6982264788
101: ch1=119248 ch2=125606 ch3=5440 mean# 83431.33333 dev# 55209.25102 part# 0.6617328145
102: ch1=123198 ch2=119712 ch3=7384 mean# 83431.33333 dev# 53792.41415 part# 0.6447507429
103: ch1=118610 ch2=120800 ch3=10884 mean# 83431.33333 dev# 51306.5019 part# 0.6149548359
104: ch1=120786 ch2=118624 ch3=10884 mean# 83431.33333 dev# 51306.30397 part# 0.6149524636
105: ch1=119344 ch2=115714 ch3=15236 mean# 83431.33333 dev# 48244.14881 part# 0.578249764
106: ch1=112258 ch2=118624 ch3=19412 mean# 83431.33333 dev# 45343.04628 part# 0.5434774259
107: ch1=115342 ch2=114272 ch3=20680 mean# 83431.33333 dev# 44374.04347 part# 0.5318630507
108: ch1=112078 ch2=114272 ch3=23944 mean# 83431.33333 dev# 42073.43207 part# 0.504288142
109: ch1=112988 ch2=112096 ch3=25210 mean# 83431.33333 dev# 41170.31015 part# 0.4934634089
110: ch1=111048 ch2=109858 ch3=29388 mean# 83431.33333 dev# 38217.49541 part# 0.4580712531
111: ch1=107722 ch2=111008 ch3=31564 mean# 83431.33333 dev# 36700.26929 part# 0.4398859257
112: ch1=107264 ch2=107744 ch3=35286 mean# 83431.33333 dev# 34044.45566 part# 0.4080535968
113: ch1=108196 ch2=105568 ch3=36530 mean# 83431.33333 dev# 33181.6003 part# 0.3977114948
114: ch1=102298 ch2=107660 ch3=40336 mean# 83431.33333 dev# 30551.52563 part# 0.3661876708
115: ch1=106818 ch2=102108 ch3=41368 mean# 83431.33333 dev# 29805.35783 part# 0.3572441748
116: ch1=99640 ch2=104934 ch3=45720 mean# 83431.33333 dev# 26753.38116 part# 0.3206634737
117: ch1=103404 ch2=99078 ch3=47812 mean# 83431.33333 dev# 25248.5148 part# 0.3026262891
118: ch1=97914 ch2=101220 ch3=51160 mean# 83431.33333 dev# 22859.15754 part# 0.273987681
119: ch1=100090 ch2=97956 ch3=52248 mean# 83431.33333 dev# 22067.15051 part# 0.2644947603
120: ch1=98242 ch2=96082 ch3=55970 mean# 83431.33333 dev# 19438.10727 part# 0.2329832989
121: ch1=92474 ch2=99040 ch3=58780 mean# 83431.33333 dev# 17636.02875 part# 0.2113837578
122: ch1=96196 ch2=93142 ch3=60956 mean# 83431.33333 dev# 15941.29199 part# 0.1910708046
123: ch1=92478 ch2=93596 ch3=64220 mean# 83431.33333 dev# 13592.12952 part# 0.1629139674
124: ch1=91390 ch2=92508 ch3=66396 mean# 83431.33333 dev# 12054.44364 part# 0.1444834112
125: ch1=90132 ch2=90044 ch3=70118 mean# 83431.33333 dev# 9414.016831 part# 0.1128355074
126: ch1=87038 ch2=91416 ch3=71840 mean# 83431.33333 dev# 8388.920365 part# 0.100548799
127: ch1=88126 ch2=87064 ch3=75104 mean# 83431.33333 dev# 5904.253909 part# 0.07076782395
128: ch1=87330 ch2=85976 ch3=76988 mean# 83431.33333 dev# 4589.534278 part# 0.05500971991
129: ch1=83608 ch2=85976 ch3=80710 mean# 83431.33333 dev# 2153.461916 part# 0.02581118903
130: ch1=86820 ch2=81834 ch3=81640 mean# 83431.33333 dev# 2397.457728 part# 0.02873569955
131: ch1=81136 ch2=84254 ch3=84904 mean# 83431.33333 dev# 1644.595459 part# 0.01971196423
132: ch1=85172 ch2=82394 ch3=82728 mean# 83431.33333 dev# 1238.367025 part# 0.01484294899
133: ch1=81358 ch2=84888 ch3=84048 mean# 83431.33333 dev# 1505.641244 part# 0.01804647228
134: ch1=87034 ch2=81624 ch3=81636 mean# 83431.33333 dev# 2547.474741 part# 0.03053378915
135: ch1=78960 ch2=85346 ch3=85988 mean# 83431.33333 dev# 3172.554947 part# 0.03802594085
136: ch1=82564 ch2=83918 ch3=83812 mean# 83431.33333 dev# 614.8221062 part# 0.007369199096
137: ch1=84678 ch2=82716 ch3=82900 mean# 83431.33333 dev# 884.7211739 part# 0.01060418357
138: ch1=84858 ch2=82716 ch3=82720 mean# 83431.33333 dev# 1008.806996 part# 0.01209146439
139: ch1=83312 ch2=83174 ch3=83808 mean# 83431.33333 dev# 272.236825 part# 0.003263004607
140: ch1=80764 ch2=84812 ch3=84718 mean# 83431.33333 dev# 1886.479849 part# 0.02261116746
141: ch1=84402 ch2=83176 ch3=82716 mean# 83431.33333 dev# 711.5922678 part# 0.008529077019
142: ch1=81598 ch2=84892 ch3=83804 mean# 83431.33333 dev# 1370.345293 part# 0.01642482792
143: ch1=86408 ch2=81628 ch3=82258 mean# 83431.33333 dev# 2120.476886 part# 0.02541583361
144: ch1=80666 ch2=84892 ch3=84736 mean# 83431.33333 dev# 1956.422813 part# 0.02344949714
145: ch1=87018 ch2=81712 ch3=81564 mean# 83431.33333 dev# 2536.875944 part# 0.03040675299
146: ch1=81148 ch2=84262 ch3=84884 mean# 83431.33333 dev# 1634.406994 part# 0.01958984627
147: ch1=84236 ch2=83350 ch3=82708 mean# 83431.33333 dev# 626.4488983 part# 0.007508556717
148: ch1=81560 ch2=84854 ch3=83880 mean# 83431.33333 dev# 1381.686248 part# 0.01656075953
149: ch1=87050 ch2=81624 ch3=81620 mean# 83431.33333 dev# 2558.78426 part# 0.03066934397
150: ch1=80064 ch2=85346 ch3=84884 mean# 83431.33333 dev# 2388.522742 part# 0.02862860566
151: ch1=83478 ch2=83478 ch3=83338 mean# 83431.33333 dev# 65.99663291 part# 0.0007910293444
152: ch1=83782 ch2=82720 ch3=83792 mean# 83431.33333 dev# 503.005191 part# 0.006028972221
153: ch1=82694 ch2=83808 ch3=83792 mean# 83431.33333 dev# 521.414316 part# 0.006249622236
154: ch1=84870 ch2=82720 ch3=82704 mean# 83431.33333 dev# 1017.311926 part# 0.01219340367
155: ch1=80518 ch2=84896 ch3=84880 mean# 83431.33333 dev# 2060.048112 part# 0.02469154009
156: ch1=85328 ch2=82720 ch3=82246 mean# 83431.33333 dev# 1355.034399 part# 0.016241313
157: ch1=82698 ch2=83808 ch3=83788 mean# 83431.33333 dev# 518.6092513 part# 0.006216000999
158: ch1=84874 ch2=82720 ch3=82700 mean# 83431.33333 dev# 1020.152058 part# 0.01222744523
159: ch1=80780 ch2=84896 ch3=84618 mean# 83431.33333 dev# 1878.207893 part# 0.02251202058
160: ch1=87212 ch2=81632 ch3=81450 mean# 83431.33333 dev# 2674.367381 part# 0.03205471223
161: ch1=81156 ch2=84266 ch3=84872 mean# 83431.33333 dev# 1627.81353 part# 0.01951081764
162: ch1=84244 ch2=83354 ch3=82696 mean# 83431.33333 dev# 634.3297425 part# 0.007603015763
163: ch1=81754 ch2=84756 ch3=83784 mean# 83431.33333 dev# 1250.67484 part# 0.01499046929
164: ch1=87058 ch2=81628 ch3=81608 mean# 83431.33333 dev# 2564.453591 part# 0.03073729604
165: ch1=81160 ch2=84262 ch3=84872 mean# 83431.33333 dev# 1625.267431 part# 0.01948030034
166: ch1=84248 ch2=83350 ch3=82696 mean# 83431.33333 dev# 636.2061161 part# 0.007625505799
167: ch1=82190 ch2=84320 ch3=83784 mean# 83431.33333 dev# 904.6197482 part# 0.01084268598
168: ch1=85974 ch2=82712 ch3=81608 mean# 83431.33333 dev# 1853.567611 part# 0.02221668451
169: ch1=81164 ch2=84258 ch3=84872 mean# 83431.33333 dev# 1622.72391 part# 0.01944981394
170: ch1=83960 ch2=83346 ch3=82988 mean# 83431.33333 dev# 401.378735 part# 0.004810887216
171: ch1=84798 ch2=82796 ch3=82700 mean# 83431.33333 dev# 967.1736602 part# 0.0115924512
172: ch1=81160 ch2=84258 ch3=84876 mean# 83431.33333 dev# 1625.771065 part# 0.01948633685
173: ch1=85696 ch2=81898 ch3=82700 mean# 83431.33333 dev# 1634.490203 part# 0.0195908436
174: ch1=80886 ch2=84896 ch3=84512 mean# 83431.33333 dev# 1806.6369 part# 0.02165417749
175: ch1=87504 ch2=81632 ch3=81158 mean# 83431.33333 dev# 2886.304365 part# 0.0345949687
176: ch1=81152 ch2=84266 ch3=84876 mean# 83431.33333 dev# 1630.857716 part# 0.01954730496
177: ch1=84240 ch2=83354 ch3=82700 mean# 83431.33333 dev# 631.0759771 part# 0.007564016442
178: ch1=81610 ch2=83808 ch3=84876 mean# 83431.33333 dev# 1359.680681 part# 0.0162970029
179: ch1=88138 ch2=80544 ch3=81612 mean# 83431.33333 dev# 3356.554715 part# 0.04023134452
180: ch1=82240 ch2=83808 ch3=84246 mean# 83431.33333 dev# 861.1687149 part# 0.01032188604
181: ch1=84194 ch2=85572 ch3=80528 mean# 83431.33333 dev# 2128.650485 part# 0.02551380159
182: ch1=87808 ch2=84134 ch3=78352 mean# 83431.33333 dev# 3892.23923 part# 0.046652008
183: ch1=87050 ch2=88156 ch3=75088 mean# 83431.33333 dev# 5916.880785 part# 0.07091916848
184: ch1=89226 ch2=88156 ch3=72912 mean# 83431.33333 dev# 7451.107539 part# 0.08930826395
185: ch1=90314 ch2=89244 ch3=70736 mean# 83431.33333 dev# 8987.578144 part# 0.107724254
186: ch1=91854 ch2=92056 ch3=66384 mean# 83431.33333 dev# 12054.56708 part# 0.1444848908
187: ch1=94662 ch2=90336 ch3=65296 mean# 83431.33333 dev# 12944.65927 part# 0.1551534508
188: ch1=93574 ch2=94688 ch3=62032 mean# 83431.33333 dev# 15138.44663 part# 0.1814479767
189: ch1=95750 ch2=94688 ch3=59856 mean# 83431.33333 dev# 16675.91511 part# 0.1998759273
190: ch1=98464 ch2=95776 ch3=56054 mean# 83431.33333 dev# 19389.77602 part# 0.2324040052
191: ch1=94528 ch2=100266 ch3=55500 mean# 83431.33333 dev# 19888.86954 part# 0.2383860924
192: ch1=101648 ch2=97498 ch3=51148 mean# 83431.33333 dev# 22890.54875 part# 0.2743639331
193: ch1=100106 ch2=101216 ch3=48972 mean# 83431.33333 dev# 24370.6417 part# 0.2921041859
194: ch1=103370 ch2=101216 ch3=45708 mean# 83431.33333 dev# 26688.91577 part# 0.3198907976
195: ch1=102282 ch2=103392 ch3=44620 mean# 83431.33333 dev# 27447.49802 part# 0.3289830921
196: ch1=104188 ch2=105838 ch3=40268 mean# 83431.33333 dev# 30528.51818 part# 0.3659119058
197: ch1=108810 ch2=103392 ch3=38092 mean# 83431.33333 dev# 32135.96159 part# 0.3851785691
198: ch1=107722 ch2=107744 ch3=34828 mean# 83431.33333 dev# 34367.74776 part# 0.4119285452
199: ch1=106634 ch2=109920 ch3=33740 mean# 83431.33333 dev# 35162.67818 part# 0.4214565053
200: ch1=112532 ch2=108374 ch3=29388 mean# 83431.33333 dev# 38252.09057 part# 0.4584859074
201: ch1=111052 ch2=112322 ch3=26920 mean# 83431.33333 dev# 39962.91048 part# 0.4789916316
202: ch1=114254 ch2=111008 ch3=25032 mean# 83431.33333 dev# 41315.82206 part# 0.4952075008
203: ch1=114712 ch2=114902 ch3=20680 mean# 83431.33333 dev# 44371.96113 part# 0.5318380919
204: ch1=114528 ch2=116174 ch3=19592 mean# 83431.33333 dev# 45146.22678 part# 0.5411183662
205: ch1=119338 ch2=115352 ch3=15604 mean# 83431.33333 dev# 47988.76542 part# 0.575188763
206: ch1=114534 ch2=120524 ch3=15236 mean# 83431.33333 dev# 48283.34869 part# 0.57871961
207: ch1=122336 ch2=117074 ch3=10884 mean# 83431.33333 dev# 51343.67109 part# 0.6154003423
208: ch1=119248 ch2=122338 ch3=8708 mean# 83431.33333 dev# 52852.43251 part# 0.6334842127
209: ch1=124054 ch2=121884 ch3=4356 mean# 83431.33333 dev# 55921.72197 part# 0.6702724232
210: ch1=121878 ch2=124060 ch3=4356 mean# 83431.33333 dev# 55921.79979 part# 0.670273356
211: ch1=123102 ch2=126558 ch3=634 mean# 83431.33333 dev# 58563.55402 part# 0.7019371701
212: ch1=128452 ch2=121842 mean# 125147 dev# 3305 part# 0.02640894308
213: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
214: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
215: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
216: ch1=123146 ch2=127148 mean# 125147 dev# 2001 part# 0.0159891967
217: ch1=127954 ch2=122340 mean# 125147 dev# 2807 part# 0.02242962276
218: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
219: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
220: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
221: ch1=123644 ch2=126650 mean# 125147 dev# 1503 part# 0.01200987639
222: ch1=128544 ch2=121750 mean# 125147 dev# 3397 part# 0.02714407856
223: ch1=121882 ch2=128412 mean# 125147 dev# 3265 part# 0.02608931896
224: ch1=126234 ch2=124060 mean# 125147 dev# 1087 part# 0.008685785516
225: ch1=125146 ch2=125148 mean# 125147 dev# 1 part# 7.990603051e-06
226: ch1=123054 ch2=127240 mean# 125147 dev# 2093 part# 0.01672433219
227: ch1=128050 ch2=122244 mean# 125147 dev# 2903 part# 0.02319672066
228: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
229: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
230: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
231: ch1=122464 ch2=127830 mean# 125147 dev# 2683 part# 0.02143878799
232: ch1=128640 ch2=121654 mean# 125147 dev# 3493 part# 0.02791117646
233: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
234: ch1=127326 ch2=122968 mean# 125147 dev# 2179 part# 0.01741152405
235: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
236: ch1=122962 ch2=127332 mean# 125147 dev# 2185 part# 0.01745946767
237: ch1=128142 ch2=122152 mean# 125147 dev# 2995 part# 0.02393185614
238: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
239: ch1=126696 ch2=123598 mean# 125147 dev# 1549 part# 0.01237744413
240: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
241: ch1=122376 ch2=127918 mean# 125147 dev# 2771 part# 0.02214196105
242: ch1=128736 ch2=121558 mean# 125147 dev# 3589 part# 0.02867827435
243: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
244: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
245: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
246: ch1=123962 ch2=126332 mean# 125147 dev# 1185 part# 0.009468864615
247: ch1=128238 ch2=122056 mean# 125147 dev# 3091 part# 0.02469895403
248: ch1=122978 ch2=127316 mean# 125147 dev# 2169 part# 0.01733161802
249: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
250: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
251: ch1=123372 ch2=126922 mean# 125147 dev# 1775 part# 0.01418332042
252: ch1=128828 ch2=121466 mean# 125147 dev# 3681 part# 0.02941340983
253: ch1=121890 ch2=128404 mean# 125147 dev# 3257 part# 0.02602539414
254: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
255: ch1=126242 ch2=124052 mean# 125147 dev# 1095 part# 0.008749710341
256: ch1=123870 ch2=126424 mean# 125147 dev# 1277 part# 0.0102040001
257: ch1=128330 ch2=121964 mean# 125147 dev# 3183 part# 0.02543408951
258: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
259: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
260: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
261: ch1=123276 ch2=127018 mean# 125147 dev# 1871 part# 0.01495041831
262: ch1=127832 ch2=122462 mean# 125147 dev# 2685 part# 0.02145476919
263: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
264: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
265: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
266: ch1=123774 ch2=126520 mean# 125147 dev# 1373 part# 0.01097109799
267: ch1=128422 ch2=121872 mean# 125147 dev# 3275 part# 0.02616922499
268: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
269: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
270: ch1=125150 ch2=125144 mean# 125147 dev# 3 part# 2.397180915e-05
271: ch1=123184 ch2=127110 mean# 125147 dev# 1963 part# 0.01568555379
272: ch1=127924 ch2=122370 mean# 125147 dev# 2777 part# 0.02218990467
273: ch1=121886 ch2=128408 mean# 125147 dev# 3261 part# 0.02605735655
274: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
275: ch1=126238 ch2=124056 mean# 125147 dev# 1091 part# 0.008717747928
276: ch1=122594 ch2=127700 mean# 125147 dev# 2553 part# 0.02040000959
277: ch1=121152 ch2=129142 mean# 125147 dev# 3995 part# 0.03192245919
278: ch1=114066 ch2=136228 mean# 125147 dev# 11081 part# 0.08854387241
279: ch1=112520 ch2=137774 mean# 125147 dev# 12627 part# 0.1008973447
280: ch1=107710 ch2=142584 mean# 125147 dev# 17437 part# 0.1393321454
281: ch1=102900 ch2=147394 mean# 125147 dev# 22247 part# 0.1777669461
282: ch1=100960 ch2=149334 mean# 125147 dev# 24187 part# 0.193268716
283: ch1=93284 ch2=157010 mean# 125147 dev# 31863 part# 0.254604585
284: ch1=92826 ch2=157468 mean# 125147 dev# 32321 part# 0.2582642812
285: ch1=88016 ch2=162278 mean# 125147 dev# 37131 part# 0.2966990819
286: ch1=82118 ch2=166520 mean# 124319 dev# 42201 part# 0.3394573637
2025-12-24T02:10:45.155224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241172081748327:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.157191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241172081748327:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:1236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:45.157208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241172081748327:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:1236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.157210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:1236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.159070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171865621042:2048] [node 2] ICP01 ready to work
VERIFY failed (2025-12-24T02:10:45.158547Z): 
  ydb/library/actors/interconnect/ut/connection_checker_ut.cpp:158
  Execute_(): requirement numConnected.load() == 100 failed
NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf<char, std::__y1::char_traits<char>>, char const*, unsigned long)+438 (0xBDCC06)
NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+263 (0xBD3447)
NTestSuiteConnectionChecker::TTestCaseCheckParams::Execute_(NUnitTest::TTestContext&)+1114 (0xA7786A)
NTestSuiteConnectionChecker::TCurrentTest::Execute()::'lambda'()::operator()() const+71 (0xA7A9A7)
NUnitTest::TTestBase::Run(std::__y1::function<void ()>, TBasicString<char, std::__y1::char_traits<char>> const&, char const*, bool)+120 (0xC6AE38)
NTestSuiteConnectionChecker::TCurrentTest::Execute()+421 (0xA7A355)
NUnitTest::TTestFactory::Execute()+817 (0xC6B581)
NUnitTest::RunMain(int, char**)+3229 (0xC7D74D)
??+0 (0x7FC94221DD90)
__libc_start_main+128 (0x7FC94221DE40)
_start+41 (0xA3D029)
2025-12-24T02:10:45.544385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171072588844:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:45.541068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241172937740334:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.619340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171072588854:2057] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:45.619767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241171072588844:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:45.619997Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [1:7587241171072588854:2057] [node 2] ICH03 handshake failed, explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected
2025-12-24T02:10:45.620603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:404: Proxy [1:7587241171072588844:2048] [node 2] ICP24 incoming handshake failed, temporary: 1 explanation: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected outgoing: [0:0:0]
2025-12-24T02:10:45.620618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7587241171072588844:2048] [node 2] ICP32 transit to hold-by-error state Explanation# incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.620620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: incoming handshake Peer# ::1(::1) connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.622820Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1583: Handshake [2:7587241172937740345:2057] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8740) error from peer: connection checker refused to proceed with incoming connection: rejected
2025-12-24T02:10:45.623646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241172937740334:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# ::1(::1:8740) error from peer: connection checker refused to proceed with incoming connection: rejected incoming: [0:0:0] held: no
2025-12-24T02:10:45.623675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241172937740334:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# ::1(::1:8740) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.623679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# ::1(::1:8740) error from peer: connection checker refused to proceed with incoming connection: rejected LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:45.634783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7587241172937740334:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:45.650389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241170859164202:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.650408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241170859164203:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:45.647512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171367794169:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:45.647520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171367794170:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:45.664290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241169435470772:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.664308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241169435470773:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:45.678448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241170771246669:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.678457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241170771246670:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:45.678475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241170771246671:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:45.671710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241172400586163:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.671718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241172400586164:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:45.671721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241172400586165:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:45.710670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241170771246730:2106] [node 6] ICP01 ready to work
2025-12-24T02:10:45.711246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171367794230:2104] [node 6] ICP01 ready to work
2025-12-24T02:10:45.711347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171367794231:2105] [node 4] ICP01 ready to work
2025-12-24T02:10:45.684486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241169872316468:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:45.684494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241169872316469:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:45.684497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241169872316470:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:45.710532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241169872316530:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:45.736065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241170859164265:2106] [node 4] ICP01 ready to work
2025-12-24T02:10:45.739133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241170771246747:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.740234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241170771246746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:45.743928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241169872316468:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16936271ns) {Configure:104(16933615ns) {ProcessConfigured:135(9321342ns) {StartInitialHandshake:157(9320679ns) {DropHandshakes:470(1139ns) {DropIncomingHandshake:442(120ns) DropOutgoingHandshake:457(133ns)} GenerateSessionVirtualId:988(443ns) PrepareNewSessionHandshake:477(1194ns) {SwitchToState:179(263ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.744121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241169872316538:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:45.745048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316539:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.740487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241171367794170:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22902713ns) {Configure:104(22901828ns) {ProcessConfigured:135(20352191ns) {StartInitialHandshake:157(20350879ns) {DropHandshakes:470(1052ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(517ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(1580ns) {SwitchToState:179(660ns)}}}}}
2025-12-24T02:10:45.750873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241169435470773:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39922684ns) {Configure:104(39921743ns) {ProcessConfigured:135(29269506ns) {StartInitialHandshake:157(29269035ns) {DropHandshakes:470(1618ns) {DropIncomingHandshake:442(441ns) DropOutgoingHandshake:457(259ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1606ns) {SwitchToState:179(672ns)}}}}}
2025-12-24T02:10:45.754932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241172400586226:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:45.755113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241172400586227:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:45.758771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241170859164266:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:45.758988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241170859164267:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.764020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241170859164270:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
2025-12-24T02:10:45.764049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241170859164271:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.766227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241172400586238:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.767107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316548:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.767140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316549:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.771096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241171367794230:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30528248ns) {Configure:104(20281639ns) {ProcessConfigured:135(10944547ns) {StartInitialHandshake:157(10944061ns) {DropHandshakes:470(761ns) {DropIncomingHandshake:442(117ns) DropOutgoingHandshake:457(185ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(1500ns) {SwitchToState:179(742ns)}}}}}
2025-12-24T02:10:45.771632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241171367794243:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:45.771250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164271:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:45.771104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241169435470846:2116] [node 5] ICP01 ready to work
2025-12-24T02:10:45.771236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241169435470847:2118] [node 6] ICP01 ready to work
2025-12-24T02:10:45.773750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241169435470848:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:45.787089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241170771246746:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.847021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241171367794244:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32790
2025-12-24T02:10:45.847109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241171367794245:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32790
2025-12-24T02:10:45.847152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241171367794246:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32790
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.873669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241169435470849:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:45.873691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241169435470850:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.927012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316551:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.927033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316550:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.930835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241169872316549:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:45.930885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241169872316538:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:45.943113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241172400586229:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:45.943147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241172400586229:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:45.971550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241172400586229:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:45.979131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241172400586226:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:45.979175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:45.979187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241172400586226:2108] [node 6] ICP22 created new session: [4:7587241172400586243:2108]
2025-12-24T02:10:45.979194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241172400586243:2108] [node 6] ICS09 handshake done sender: [4:7587241172400586229:2110] self: [4:7587241172400586228:0] peer: [6:7587241169872316552:0] socket: 52 qp: -1
2025-12-24T02:10:45.979205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241172400586243:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:45.979228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.981640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241169435470862:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:45.981878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470862:2126] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:45.983022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241169872316553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:45.999018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241169872316549:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.013992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241169872316538:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.014032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.014042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241169872316538:2111] [node 4] ICP22 created new session: [6:7587241174167283850:2111]
2025-12-24T02:10:46.014049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241174167283850:2111] [node 4] ICS09 handshake done sender: [6:7587241169872316549:2118] self: [6:7587241169872316552:0] peer: [4:7587241172400586228:0] socket: 57 qp: -1
2025-12-24T02:10:46.014061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241174167283850:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:46.014075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.030223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241169872316539:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.052747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214053:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.053012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241169872316530:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.058808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241170771246732:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.058828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241170771246732:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.086371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241169872316550:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.086425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241169872316468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.086065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794245:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.086100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794246:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.086155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794244:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.086664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794239:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.086668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241171367794239:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.087229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241170771246732:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.103547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241170771246730:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.103585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.103594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241170771246730:2106] [node 6] ICP22 created new session: [5:7587241175066214056:2106]
2025-12-24T02:10:46.103601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175066214056:2106] [node 6] ICS09 handshake done sender: [5:7587241170771246732:2108] self: [5:7587241170771246731:0] peer: [6:7587241174167283853:0] socket: 39 qp: -1
2025-12-24T02:10:46.103610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175066214056:2106] [node 6] ICS10 traffic start
2025-12-24T02:10:46.103619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.104061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241171367794239:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.112911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241171367794230:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.112959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.112971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241171367794230:2104] [node 6] ICP22 created new session: [1:7587241175662761551:2104]
2025-12-24T02:10:46.112978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241175662761551:2104] [node 6] ICS09 handshake done sender: [1:7587241171367794239:2112] self: [1:7587241171367794238:0] peer: [6:7587241174167283854:0] socket: 65 qp: -1
2025-12-24T02:10:46.112987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241175662761551:2104] [node 6] ICS10 traffic start
2025-12-24T02:10:46.113001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.126420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241169872316530:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13549) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241169872316539:2113] held: no
2025-12-24T02:10:46.126915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241174167283855:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.131993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241174167283856:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.165441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553542:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
2025-12-24T02:10:46.173303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241176695553542:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.173373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241172400586163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.177326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794241:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.177356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241171367794241:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.194841Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241169872316468:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241169872316550:2119] held: no
2025-12-24T02:10:46.194895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241169872316539:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.198623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241169872316550:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.195637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241171367794241:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.198802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241171367794231:2105] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.199457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553544:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
2025-12-24T02:10:46.199723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.199735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241171367794231:2105] [node 4] ICP22 created new session: [1:7587241175662761555:2105]
2025-12-24T02:10:46.199740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241175662761555:2105] [node 4] ICS09 handshake done sender: [1:7587241171367794241:2113] self: [1:7587241171367794240:0] peer: [4:7587241176695553543:0] socket: 36 qp: -1
2025-12-24T02:10:46.199745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241175662761555:2105] [node 4] ICS10 traffic start
2025-12-24T02:10:46.199751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.200168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241176695553542:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.200241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241172400586163:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.200268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.200274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241172400586163:2048] [node 1] ICP22 created new session: [4:7587241176695553545:2048]
2025-12-24T02:10:46.200279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241176695553545:2048] [node 1] ICS09 handshake done sender: [4:7587241176695553542:2116] self: [4:7587241176695553543:0] peer: [1:7587241171367794240:0] socket: 61 qp: -1
2025-12-24T02:10:46.200284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241176695553545:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.200289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.201133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214059:2126] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.199368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241169872316530:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.199403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.199413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241169872316530:2107] [node 5] ICP22 created new session: [6:7587241174167283857:2107]
2025-12-24T02:10:46.199421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241174167283857:2107] [node 5] ICS09 handshake done sender: [6:7587241169872316539:2113] self: [6:7587241174167283853:0] peer: [5:7587241170771246731:0] socket: 42 qp: -1
2025-12-24T02:10:46.199431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241174167283857:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:46.199441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.199523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241169872316468:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.199537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.199542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241169872316468:2048] [node 1] ICP22 created new session: [6:7587241174167283859:2048]
2025-12-24T02:10:46.199547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241174167283859:2048] [node 1] ICS09 handshake done sender: [6:7587241169872316550:2119] self: [6:7587241174167283854:0] peer: [1:7587241171367794238:0] socket: 69 qp: -1
2025-12-24T02:10:46.199551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241174167283859:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.199560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.202050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241169872316551:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.202072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241169872316548:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.202097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241169872316470:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.202105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241169872316469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.201743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241170859164283:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
2025-12-24T02:10:46.201804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241170859164285:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
2025-12-24T02:10:46.201859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164270:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.201937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241170859164286:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
2025-12-24T02:10:46.203914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175066214059:2126] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.203956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241170771246669:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.204114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794248:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.204127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241171367794248:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.204642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241171367794248:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.204695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214061:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.204755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241171367794243:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.204780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.204787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241171367794243:2107] [node 5] ICP22 created new session: [1:7587241175662761559:2107]
2025-12-24T02:10:46.204792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241175662761559:2107] [node 5] ICS09 handshake done sender: [1:7587241171367794248:2117] self: [1:7587241171367794247:0] peer: [5:7587241175066214060:0] socket: 62 qp: -1
2025-12-24T02:10:46.204798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241175662761559:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:46.204812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.204862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175066214059:2126] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.204912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241170771246669:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.204941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.204951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241170771246669:2048] [node 1] ICP22 created new session: [5:7587241175066214062:2048]
2025-12-24T02:10:46.204956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175066214062:2048] [node 1] ICS09 handshake done sender: [5:7587241175066214059:2126] self: [5:7587241175066214060:0] peer: [1:7587241171367794247:0] socket: 88 qp: -1
2025-12-24T02:10:46.204962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175066214062:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.204969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.206765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164285:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.206808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241170859164202:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.206882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164275:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.206885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241170859164275:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.207602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241174167283863:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.207796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470850:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.207822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470849:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.207878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470854:2123] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.207881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241169435470854:2123] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.208474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241173730438160:2127] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:46.208599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241169435470772:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.208608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241169435470773:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.208772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241173730438161:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:46.208857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241169435470854:2123] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.209125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175066214053:2125] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.209208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241170771246671:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.209616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241174167283864:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5642
2025-12-24T02:10:46.210136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794229:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.210147Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241171367794229:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.210583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241171367794237:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.210590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241171367794237:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.211970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241169872316551:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.212038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241169872316470:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.212074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.212086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241169872316470:2050] [node 3] ICP22 created new session: [6:7587241174167283865:2050]
2025-12-24T02:10:46.212093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241174167283865:2050] [node 3] ICS09 handshake done sender: [6:7587241169872316551:2120] self: [6:7587241174167283861:0] peer: [3:7587241169435470853:0] socket: 75 qp: -1
2025-12-24T02:10:46.212105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241174167283865:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.212114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.213007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241172400586238:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.213169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241172400586164:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.213324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553547:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.214875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214066:2128] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.215203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241171367794229:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.215356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241171367794237:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.215429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241171367794169:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.215477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.215486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241171367794169:2048] [node 2] ICP22 created new session: [1:7587241175662761563:2048]
2025-12-24T02:10:46.215492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241175662761563:2048] [node 2] ICS09 handshake done sender: [1:7587241171367794229:2106] self: [1:7587241171367794228:0] peer: [2:7587241175154131584:0] socket: 63 qp: -1
2025-12-24T02:10:46.215500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241175662761563:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:46.215509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:46.215626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241171367794170:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.215634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.215640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241171367794170:2049] [node 3] ICP22 created new session: [1:7587241175662761565:2049]
2025-12-24T02:10:46.215643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241175662761565:2049] [node 3] ICS09 handshake done sender: [1:7587241171367794237:2111] self: [1:7587241171367794233:0] peer: [3:7587241173730438163:0] socket: 64 qp: -1
2025-12-24T02:10:46.215647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241175662761565:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:46.215665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.217081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241176695553547:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.217236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553550:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
2025-12-24T02:10:46.217385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175066214066:2128] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.217436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241170771246747:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.217397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470852:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.217409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241169435470852:2122] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.217919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241173730438160:2127] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.217980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241169435470847:2118] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.218008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.218015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241169435470847:2118] [node 6] ICP22 created new session: [3:7587241173730438166:2118]
2025-12-24T02:10:46.218021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241173730438166:2118] [node 6] ICS09 handshake done sender: [3:7587241169435470854:2123] self: [3:7587241169435470853:0] peer: [6:7587241174167283861:0] socket: 73 qp: -1
2025-12-24T02:10:46.218030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241173730438166:2118] [node 6] ICS10 traffic start
2025-12-24T02:10:46.218041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.218348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241169435470847:2118] [node 6] ICP18 handshake (actor [3:7587241173730438160:2127]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.218837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214068:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.215902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164286:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.215941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164269:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.215945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241170859164269:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.216360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164283:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.216379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164273:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.216384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241170859164273:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.216992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241175154131586:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25882
2025-12-24T02:10:46.218931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241172400586237:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.218941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241172400586237:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.219689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214069:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.219796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241172400586237:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.219977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175066214066:2128] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.220037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241170771246747:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.220057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.220062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241170771246747:2111] [node 4] ICP22 created new session: [5:7587241175066214070:2111]
2025-12-24T02:10:46.220099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175066214070:2111] [node 4] ICS09 handshake done sender: [5:7587241175066214066:2128] self: [5:7587241175066214067:0] peer: [4:7587241172400586236:0] socket: 99 qp: -1
2025-12-24T02:10:46.220107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175066214070:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:46.220122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.221276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241172400586227:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.221304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.221311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241172400586227:2109] [node 5] ICP22 created new session: [4:7587241176695553553:2109]
2025-12-24T02:10:46.221317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241176695553553:2109] [node 5] ICS09 handshake done sender: [4:7587241172400586237:2114] self: [4:7587241172400586236:0] peer: [5:7587241175066214067:0] socket: 98 qp: -1
2025-12-24T02:10:46.221323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241176695553553:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:46.221335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.221960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241169872316548:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.222033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241169872316469:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.222054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.222060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241169872316469:2049] [node 2] ICP22 created new session: [6:7587241174167283867:2049]
2025-12-24T02:10:46.222065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241174167283867:2049] [node 2] ICS09 handshake done sender: [6:7587241169872316548:2117] self: [6:7587241174167283862:0] peer: [2:7587241170859164274:0] socket: 56 qp: -1
2025-12-24T02:10:46.222069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241174167283867:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:46.222085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:46.221176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241170859164273:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.221794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241170859164275:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.223022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241170859164265:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.223047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.223054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241170859164265:2106] [node 4] ICP22 created new session: [2:7587241175154131591:2106]
2025-12-24T02:10:46.223059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241175154131591:2106] [node 4] ICS09 handshake done sender: [2:7587241170859164273:2113] self: [2:7587241170859164272:0] peer: [4:7587241176695553549:0] socket: 49 qp: -1
2025-12-24T02:10:46.223065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241175154131591:2106] [node 4] ICS10 traffic start
2025-12-24T02:10:46.223071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.223833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241172400586238:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.223977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241173730438161:2128] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.224066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241173730438165:2129] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:46.224149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241172400586164:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.224178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.224184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241172400586164:2049] [node 2] ICP22 created new session: [4:7587241176695553555:2049]
2025-12-24T02:10:46.224190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241176695553555:2049] [node 2] ICS09 handshake done sender: [4:7587241172400586238:2115] self: [4:7587241176695553549:0] peer: [2:7587241170859164272:0] socket: 51 qp: -1
2025-12-24T02:10:46.224196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241176695553555:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:46.224201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:46.224994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241169435470850:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.225267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241169435470772:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.225297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.225302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241169435470772:2048] [node 1] ICP22 created new session: [3:7587241173730438171:2048]
2025-12-24T02:10:46.225307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241173730438171:2048] [node 1] ICS09 handshake done sender: [3:7587241169435470850:2121] self: [3:7587241173730438163:0] peer: [1:7587241171367794233:0] socket: 70 qp: -1
2025-12-24T02:10:46.225313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241173730438171:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.225320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.225798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241169435470852:2122] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.226033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241175662761567:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32790
2025-12-24T02:10:46.226227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175066214053:2125] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.223183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241170859164266:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.223210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.223214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241170859164266:2107] [node 6] ICP22 created new session: [2:7587241175154131593:2107]
2025-12-24T02:10:46.223218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241175154131593:2107] [node 6] ICS09 handshake done sender: [2:7587241170859164275:2114] self: [2:7587241170859164274:0] peer: [6:7587241174167283862:0] socket: 50 qp: -1
2025-12-24T02:10:46.223221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241175154131593:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:46.223224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.226297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241170771246671:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.226325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.226331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241170771246671:2050] [node 3] ICP22 created new session: [5:7587241175066214072:2050]
2025-12-24T02:10:46.226337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175066214072:2050] [node 3] ICS09 handshake done sender: [5:7587241175066214053:2125] self: [5:7587241175066214065:0] peer: [3:7587241169435470851:0] socket: 82 qp: -1
2025-12-24T02:10:46.226342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175066214072:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.226348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.227711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241175662761568:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32790
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.227860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214074:2131] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.228068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553557:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
2025-12-24T02:10:46.228554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241169435470846:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.228578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.228584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241169435470846:2116] [node 5] ICP22 created new session: [3:7587241173730438174:2116]
2025-12-24T02:10:46.228590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241173730438174:2116] [node 5] ICS09 handshake done sender: [3:7587241169435470852:2122] self: [3:7587241169435470851:0] peer: [5:7587241175066214065:0] socket: 72 qp: -1
2025-12-24T02:10:46.228595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241173730438174:2116] [node 5] ICS10 traffic start
2025-12-24T02:10:46.228610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.229942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241173730438176:2130] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38482
2025-12-24T02:10:46.230286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241169435470849:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.227566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241170859164285:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.228168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241170859164202:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.228184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.228190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241170859164202:2048] [node 1] ICP22 created new session: [2:7587241175154131597:2048]
2025-12-24T02:10:46.228195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241175154131597:2048] [node 1] ICS09 handshake done sender: [2:7587241170859164285:2118] self: [2:7587241175154131584:0] peer: [1:7587241171367794228:0] socket: 67 qp: -1
2025-12-24T02:10:46.228199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241175154131597:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.228211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.229156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241170859164269:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.232069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241169435470773:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.232102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.232111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241169435470773:2049] [node 2] ICP22 created new session: [3:7587241173730438177:2049]
2025-12-24T02:10:46.232120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241173730438177:2049] [node 2] ICS09 handshake done sender: [3:7587241169435470849:2120] self: [3:7587241173730438164:0] peer: [2:7587241170859164268:0] socket: 68 qp: -1
2025-12-24T02:10:46.232128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241173730438177:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:46.232135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:46.233389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241170859164203:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.233430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.233436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241170859164203:2049] [node 3] ICP22 created new session: [2:7587241175154131599:2049]
2025-12-24T02:10:46.233441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241175154131599:2049] [node 3] ICS09 handshake done sender: [2:7587241170859164269:2110] self: [2:7587241170859164268:0] peer: [3:7587241173730438164:0] socket: 48 qp: -1
2025-12-24T02:10:46.233447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241175154131599:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:46.233453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.233952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175066214074:2131] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.233988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241170771246670:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.234834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175066214076:2132] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60724
2025-12-24T02:10:46.235108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175066214074:2131] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.235178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241170771246670:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.235200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.235204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241170771246670:2049] [node 2] ICP22 created new session: [5:7587241175066214077:2049]
2025-12-24T02:10:46.235209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175066214077:2049] [node 2] ICS09 handshake done sender: [5:7587241175066214074:2131] self: [5:7587241175066214075:0] peer: [2:7587241170859164276:0] socket: 107 qp: -1
2025-12-24T02:10:46.235215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175066214077:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:46.235229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:46.237707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241176695553557:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.237740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241172400586165:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.237916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241169435470859:2125] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.237925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241169435470859:2125] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.234149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241170859164277:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.234159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241170859164277:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.234941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241170859164277:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.237221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241170859164267:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.237239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.237245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241170859164267:2108] [node 5] ICP22 created new session: [2:7587241175154131602:2108]
2025-12-24T02:10:46.237250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241175154131602:2108] [node 5] ICS09 handshake done sender: [2:7587241170859164277:2115] self: [2:7587241170859164276:0] peer: [5:7587241175066214075:0] socket: 58 qp: -1
2025-12-24T02:10:46.237256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241175154131602:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:46.237274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.238924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241176695553559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49160
2025-12-24T02:10:46.239238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241169435470859:2125] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.239433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241176695553557:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.239554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241172400586165:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.239579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.239585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241172400586165:2050] [node 3] ICP22 created new session: [4:7587241176695553560:2050]
2025-12-24T02:10:46.239590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241176695553560:2050] [node 3] ICS09 handshake done sender: [4:7587241176695553557:2120] self: [4:7587241176695553558:0] peer: [3:7587241169435470858:0] socket: 54 qp: -1
2025-12-24T02:10:46.239596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241176695553560:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.239611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.240620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241169435470848:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.240646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.240652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241169435470848:2107] [node 4] ICP22 created new session: [3:7587241173730438180:2107]
2025-12-24T02:10:46.240656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241173730438180:2107] [node 4] ICS09 handshake done sender: [3:7587241169435470859:2125] self: [3:7587241169435470858:0] peer: [4:7587241176695553558:0] socket: 44 qp: -1
2025-12-24T02:10:46.240662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241173730438180:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:46.240704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.251889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:46.251998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241173730438171:2048] [node 1] ICS07 socket disconnect 70 reason# EndOfStream
2025-12-24T02:10:46.252005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241173730438171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.252073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:46.252095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241173730438171:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.251177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:46.251283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241175154131597:2048] [node 1] ICS07 socket disconnect 67 reason# EndOfStream
2025-12-24T02:10:46.251290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241175154131597:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.251335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:46.251353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241175154131597:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.253992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241170859164202:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:46.254002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241175154131597:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.254252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241170859164202:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.254261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241175154131597:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.254266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241170859164202:2048] [node 1] ICP30 unregister session Session# [2:7587241175154131597:2048] VirtualId# [2:7587241175154131584:0]
2025-12-24T02:10:46.254287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:46.254294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241170859164202:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.254304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.256204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:46.256296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241174167283859:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-12-24T02:10:46.256305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241169872316468:2048] [node 1] ICP30 unregister session Session# [6:7587241174167283859:2048] VirtualId# [6:7587241174167283854:0]
2025-12-24T02:10:46.256313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241174167283859:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.256377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:46.256406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:46.256445Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:46.256506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241176695553545:2048] [node 1] ICS07 socket disconnect 61 reason# EndOfStream
2025-12-24T02:10:46.256510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241176695553545:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.256540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:46.256563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241176695553545:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.257074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241172400586163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.257087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241176695553545:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.257091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241172400586163:2048] [node 1] ICP30 unregister session Session# [4:7587241176695553545:2048] VirtualId# [4:7587241176695553543:0]
2025-12-24T02:10:46.257106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:46.257111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241172400586163:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.257114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.258103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:46.258191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241175066214062:2048] [node 1] ICS07 socket disconnect 88 reason# EndOfStream
2025-12-24T02:10:46.258196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241175066214062:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.258262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:46.258286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241175066214062:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.258138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241169435470772:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:46.258155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241173730438171:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:46.258426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241169435470772:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.258438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241173730438171:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.258444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241169435470772:2048] [node 1] ICP30 unregister session Session# [3:7587241173730438171:2048] VirtualId# [3:7587241173730438163:0]
2025-12-24T02:10:46.258461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:46.258467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241169435470772:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.258469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.258775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241170771246669:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.258788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241175066214062:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.258793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241170771246669:2048] [node 1] ICP30 unregister session Session# [5:7587241175066214062:2048] VirtualId# [5:7587241175066214060:0]
2025-12-24T02:10:46.258804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:46.258810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241170771246669:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.258812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.260646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:46.260730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241173730438177:2049] [node 2] ICS07 socket disconnect 68 reason# EndOfStream
2025-12-24T02:10:46.260737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241173730438177:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.260763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:46.260778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241173730438177:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:46.263010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241169435470773:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:46.263026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241173730438177:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:46.263298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241169435470773:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.263310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241173730438177:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.263314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241169435470773:2049] [node 2] ICP30 unregister session Session# [3:7587241173730438177:2049] VirtualId# [3:7587241173730438164:0]
2025-12-24T02:10:46.263329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:46.263335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241169435470773:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.263339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.266608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241172400586163:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:46.266661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241170771246669:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:46.270795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:46.270909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241176695553555:2049] [node 2] ICS07 socket disconnect 51 reason# EndOfStream
2025-12-24T02:10:46.270915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241176695553555:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.271018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:46.271043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241176695553555:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:46.271415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241172400586164:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.271428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241176695553555:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.271434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241172400586164:2049] [node 2] ICP30 unregister session Session# [4:7587241176695553555:2049] VirtualId# [4:7587241176695553549:0]
2025-12-24T02:10:46.271451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:46.271457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241172400586164:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.271460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.271603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:46.271682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241174167283867:2049] [node 2] ICS07 socket disconnect 56 reason# EndOfStream
2025-12-24T02:10:46.271687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241174167283867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.271739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:46.271542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:46.271641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241175066214077:2049] [node 2] ICS07 socket disconnect 107 reason# EndOfStream
2025-12-24T02:10:46.271648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241175066214077:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.271704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:46.271738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241175066214077:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:46.271762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241174167283867:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:46.272086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241170771246670:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.272097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241175066214077:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.272102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241170771246670:2049] [node 2] ICP30 unregister session Session# [5:7587241175066214077:2049] VirtualId# [5:7587241175066214075:0]
2025-12-24T02:10:46.272115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:46.272119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241170771246670:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.272122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.273322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241169872316469:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.273339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241174167283867:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.273345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241169872316469:2049] [node 2] ICP30 unregister session Session# [6:7587241174167283867:2049] VirtualId# [6:7587241174167283862:0]
2025-12-24T02:10:46.273359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:46.273365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241169872316469:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.273366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6757) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.281666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241172400586164:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:46.283579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:46.283693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241176695553560:2050] [node 3] ICS07 socket disconnect 54 reason# EndOfStream
2025-12-24T02:10:46.283705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241176695553560:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.283772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:46.283807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241176695553560:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:46.284154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241172400586165:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.284165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241176695553560:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.284168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241172400586165:2050] [node 3] ICP30 unregister session Session# [4:7587241176695553560:2050] VirtualId# [4:7587241176695553558:0]
2025-12-24T02:10:46.284182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:46.284188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241172400586165:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.284198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.283604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:46.283692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241174167283865:2050] [node 3] ICS07 socket disconnect 75 reason# EndOfStream
2025-12-24T02:10:46.283699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241174167283865:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.283766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:46.283793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241174167283865:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:46.284132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241169872316469:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:46.284153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241169872316470:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.284160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241174167283865:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.284165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241169872316470:2050] [node 3] ICP30 unregister session Session# [6:7587241174167283865:2050] VirtualId# [6:7587241174167283861:0]
2025-12-24T02:10:46.284184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:46.284189Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241169872316470:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.284191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.286714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:46.286805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241175066214072:2050] [node 3] ICS07 socket disconnect 82 reason# EndOfStream
2025-12-24T02:10:46.286811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241175066214072:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.286875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:46.286902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241175066214072:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:46.286945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241170771246670:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:46.293791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:46.293899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241174167283850:2111] [node 4] ICS07 socket disconnect 57 reason# EndOfStream
2025-12-24T02:10:46.293914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241174167283850:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:46.293974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:46.294005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241174167283850:2111] [node 4] ICS15 start handshake
2025-12-24T02:10:46.294186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241169872316470:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:46.294362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241169872316538:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2240) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:46.294380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241174167283850:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:46.294385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241169872316538:2111] [node 4] ICP30 unregister session Session# [6:7587241174167283850:2111] VirtualId# [6:7587241169872316552:0]
2025-12-24T02:10:46.294398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:46.294406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241169872316538:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2240) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:46.294410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2240) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:46.319055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241177282848688:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:46.319067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241177282848689:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:46.318638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241176727722349:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:46.318653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241176727722350:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:46.321850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241174169514722:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:46.321859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241174169514723:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:46.334830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241175649265920:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:46.334849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241175649265921:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:46.334860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241175649265922:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:46.350978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241175085893146:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:46.350999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241175085893147:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:46.340841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241176146881778:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:46.340854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241176146881779:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:46.340867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241176146881780:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:46.351159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241175085893145:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:46.371873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241176146881823:2086] [node 5] ICP01 ready to work
2025-12-24T02:10:46.371980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241176146881824:2091] [node 4] ICP01 ready to work
2025-12-24T02:10:46.375167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241177282848738:2087] [node 4] ICP01 ready to work
2025-12-24T02:10:46.375267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241177282848739:2091] [node 5] ICP01 ready to work
2025-12-24T02:10:46.379854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241175085893201:2097] [node 6] ICP01 ready to work
2025-12-24T02:10:46.392046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241174169514789:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.399108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241175649265989:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:46.399202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241175649265990:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:46.400253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649265987:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.406815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241177282848688:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31479521ns) {Configure:104(31478868ns) {ProcessConfigured:135(23631687ns) {StartInitialHandshake:157(23629735ns) {DropHandshakes:470(896ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(2536ns) {SwitchToState:179(1093ns)}}}}}
2025-12-24T02:10:46.407893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241177282848755:2098] [node 6] ICP01 ready to work
2025-12-24T02:10:46.408574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241177282848756:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57360
2025-12-24T02:10:46.408654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241177282848757:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57360
2025-12-24T02:10:46.413823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241176727722415:2100] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.414982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241175085893146:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27865808ns) {Configure:104(27863158ns) {ProcessConfigured:135(27554679ns) {StartInitialHandshake:157(27553559ns) {DropHandshakes:470(1077ns) {DropIncomingHandshake:442(154ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(307ns) PrepareNewSessionHandshake:477(1935ns) {SwitchToState:179(1410ns)}}}}}
2025-12-24T02:10:46.415452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241175085893215:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:46.415956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893207:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.416035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893216:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.418733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241175649265921:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19108746ns) {Configure:104(19108168ns) {ProcessConfigured:135(18698369ns) {StartInitialHandshake:157(18697879ns) {DropHandshakes:470(193ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(2798ns) {SwitchToState:179(1099ns)}}}}}
2025-12-24T02:10:46.418851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649265987:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.419209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241175085893201:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39247496ns) {Configure:104(39245781ns) {ProcessConfigured:135(38834764ns) {StartInitialHandshake:157(38834098ns) {DropHandshakes:470(951ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:477(1179ns) {SwitchToState:179(968ns)}}}}}
2025-12-24T02:10:46.419276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893207:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.425557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848757:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.434234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241174169514789:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39843306ns) {Configure:104(39839231ns) {ProcessConfigured:135(39491007ns) {StartInitialHandshake:157(39489868ns) {DropHandshakes:470(1502ns) {DropIncomingHandshake:442(171ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(90379ns) PrepareNewSessionHandshake:477(1702ns) {SwitchToState:179(655ns)}}}}}
2025-12-24T02:10:46.434436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241174169514795:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:46.434560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241174169514793:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:46.434973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722412:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-12-24T02:10:46.439039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241176727722416:2103] [node 4] ICP01 ready to work
2025-12-24T02:10:46.439104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722413:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.446876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241177282848738:2087] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(51353178ns) {Configure:104(51350540ns) {ProcessConfigured:135(43517456ns) {StartInitialHandshake:157(43516454ns) {DropHandshakes:470(396ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(982ns) {SwitchToState:179(208ns)}}}}}
2025-12-24T02:10:46.447050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848756:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.450968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514801:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.451059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514802:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.451070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.458967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514801:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.455027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241176727722419:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:46.455141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722412:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.455167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722413:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.471096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241177282848769:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57360
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.475116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848769:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.475223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514802:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.475260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241174169514722:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.478743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893226:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.483538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848752:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.483549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241177282848752:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.482816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722414:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-12-24T02:10:46.482930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722417:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-12-24T02:10:46.482975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722418:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-12-24T02:10:46.486762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722418:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.486837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241176727722349:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47597786ns) {Configure:104(47596218ns) {ProcessConfigured:135(11740550ns) {StartInitialHandshake:157(11738950ns) {DropHandshakes:470(1105ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(812ns) PrepareNewSessionHandshake:477(1071ns) {SwitchToState:179(517ns)}}}}}
2025-12-24T02:10:46.486223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893226:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.486266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241175085893147:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.490901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266000:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.490965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722414:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.490841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514794:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.490858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241174169514794:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.507384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514806:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.508634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514807:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.514668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241177282848752:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.514834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893216:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.514893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241175085893145:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.534974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241174169514802:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.557569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848754:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.557589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241177282848754:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.557925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241177282848689:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.557950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.557963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241177282848689:2049] [node 3] ICP22 created new session: [1:7587241177282848774:2049]
2025-12-24T02:10:46.557970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241177282848774:2049] [node 3] ICS09 handshake done sender: [1:7587241177282848752:2106] self: [1:7587241177282848751:0] peer: [3:7587241174169514805:0] socket: 61 qp: -1
2025-12-24T02:10:46.557979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241177282848774:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:46.557993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.558801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241177282848754:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.558896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241177282848739:2091] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.558914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.558920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241177282848739:2091] [node 5] ICP22 created new session: [1:7587241177282848778:2091]
2025-12-24T02:10:46.558926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241177282848778:2091] [node 5] ICS09 handshake done sender: [1:7587241177282848754:2107] self: [1:7587241177282848753:0] peer: [5:7587241175085893228:0] socket: 62 qp: -1
2025-12-24T02:10:46.558930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241177282848778:2091] [node 5] ICS10 traffic start
2025-12-24T02:10:46.558937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.558910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241174169514722:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.558937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.558947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241174169514722:2048] [node 1] ICP22 created new session: [3:7587241174169514809:2048]
2025-12-24T02:10:46.558954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241174169514809:2048] [node 1] ICS09 handshake done sender: [3:7587241174169514802:2116] self: [3:7587241174169514805:0] peer: [1:7587241177282848751:0] socket: 75 qp: -1
2025-12-24T02:10:46.558972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241174169514809:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.558981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.585657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241174169514794:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.585748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241174169514789:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.585767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.585776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241174169514789:2109] [node 5] ICP22 created new session: [3:7587241174169514812:2109]
2025-12-24T02:10:46.585783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241174169514812:2109] [node 5] ICS09 handshake done sender: [3:7587241174169514794:2112] self: [3:7587241174169514791:0] peer: [5:7587241175085893227:0] socket: 72 qp: -1
2025-12-24T02:10:46.585789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241174169514812:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:46.585802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.582975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881856:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.583088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.631284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241175085893145:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241175085893216:2113] held: no
2025-12-24T02:10:46.631488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.631140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.638921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.638966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266001:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.639158Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241174169514722:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(80204307ns) {HandleHandshakeStatus:329(80202920ns) {DropIncomingHandshake:442(162ns) DropOutgoingHandshake:457(1495ns) SwitchToState:179(104ns) ProcessPendingSessionEvents:495(44243076ns) {ForwardSessionEventToSession:612(44224373ns) ForwardSessionEventToSession:612(2566ns) ForwardSessionEventToSession:612(1251ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(827ns) ForwardSessionEventToSession:612(1690ns) ForwardSessionEventToSession:612(926ns) ForwardSessionEventToSession:612(1038ns) ForwardSessionEventToSession:612(734ns)}}}
2025-12-24T02:10:46.639214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514803:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.639247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241174169514789:2109] [node 5] ICP18 handshake (actor [3:7587241174169514803:2117]) is too old
2025-12-24T02:10:46.640019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241174169514722:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:46.643011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241176146881857:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.650585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241176146881858:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.650627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241176146881780:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.658763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893230:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.666849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514799:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.666746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649266001:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.666867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241174169514799:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.666820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241175649265922:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.666726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241176146881778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.666794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241176146881778:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23703143ns) {IncomingHandshake:261(23702361ns) {DropIncomingHandshake:442(258ns) PrepareNewSessionHandshake:477(890ns) {SwitchToState:179(358ns)} GenerateSessionVirtualId:988(313ns) IssueIncomingHandshakeReply:193(23694965ns) {SwitchToState:179(1311ns)}}}
2025-12-24T02:10:46.682948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175085893216:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.683589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241175085893147:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241175085893226:2117] held: no
2025-12-24T02:10:46.683628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175085893226:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.683696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241175085893147:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.683717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.683726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241175085893147:2050] [node 3] ICP22 created new session: [5:7587241175085893231:2050]
2025-12-24T02:10:46.683742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175085893231:2050] [node 3] ICS09 handshake done sender: [5:7587241175085893226:2117] self: [5:7587241175085893227:0] peer: [3:7587241174169514791:0] socket: 73 qp: -1
2025-12-24T02:10:46.683752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175085893231:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.683780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.682764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514797:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.682792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241174169514797:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.684351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241175085893145:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.684374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.684382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241175085893145:2048] [node 1] ICP22 created new session: [5:7587241175085893235:2048]
2025-12-24T02:10:46.684388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175085893235:2048] [node 1] ICS09 handshake done sender: [5:7587241175085893216:2113] self: [5:7587241175085893228:0] peer: [1:7587241177282848753:0] socket: 65 qp: -1
2025-12-24T02:10:46.684393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175085893235:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.684400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.685615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848759:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.685635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241177282848759:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.707620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241174169514797:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.710870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.710891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.710834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241177282848759:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.711009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241177282848755:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.711021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.711028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241177282848755:2098] [node 6] ICP22 created new session: [1:7587241177282848782:2098]
2025-12-24T02:10:46.711035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241177282848782:2098] [node 6] ICS09 handshake done sender: [1:7587241177282848759:2110] self: [1:7587241177282848758:0] peer: [6:7587241176146881859:0] socket: 86 qp: -1
2025-12-24T02:10:46.711042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241177282848782:2098] [node 6] ICS10 traffic start
2025-12-24T02:10:46.711055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.714784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649266002:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.714835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241175649265920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.714840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241174169514799:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.734408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848768:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.734443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241177282848768:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.750942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.751023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.754803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241174169514795:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.754844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.754860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241174169514795:2110] [node 4] ICP22 created new session: [3:7587241174169514817:2110]
2025-12-24T02:10:46.754869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241174169514817:2110] [node 4] ICS09 handshake done sender: [3:7587241174169514797:2113] self: [3:7587241174169514796:0] peer: [4:7587241175649266003:0] socket: 49 qp: -1
2025-12-24T02:10:46.754880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241174169514817:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:46.754894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.755047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241174169514793:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.755065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.755072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241174169514793:2108] [node 6] ICP22 created new session: [3:7587241174169514819:2108]
2025-12-24T02:10:46.755079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241174169514819:2108] [node 6] ICS09 handshake done sender: [3:7587241174169514799:2114] self: [3:7587241174169514798:0] peer: [6:7587241176146881860:0] socket: 90 qp: -1
2025-12-24T02:10:46.755086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241174169514819:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:46.755093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.755026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649266005:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.762765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241175649266001:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.778867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241177282848768:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.778926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241176146881857:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.779048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.779204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241176146881778:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.779226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.779235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241176146881778:2048] [node 1] ICP22 created new session: [6:7587241176146881864:2048]
2025-12-24T02:10:46.779242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241176146881864:2048] [node 1] ICS09 handshake done sender: [6:7587241176146881857:2116] self: [6:7587241176146881859:0] peer: [1:7587241177282848758:0] socket: 88 qp: -1
2025-12-24T02:10:46.779249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241176146881864:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.779259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.779619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241176146881858:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:46.779679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241176146881780:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.779690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.779694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241176146881780:2050] [node 3] ICP22 created new session: [6:7587241176146881866:2050]
2025-12-24T02:10:46.779698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241176146881866:2050] [node 3] ICS09 handshake done sender: [6:7587241176146881858:2117] self: [6:7587241176146881860:0] peer: [3:7587241174169514798:0] socket: 91 qp: -1
2025-12-24T02:10:46.779704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241176146881866:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.779711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.806802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241175649265922:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.806840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.806853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241175649265922:2050] [node 3] ICP22 created new session: [4:7587241175649266008:2050]
2025-12-24T02:10:46.806861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241175649266008:2050] [node 3] ICS09 handshake done sender: [4:7587241175649266001:2117] self: [4:7587241175649266003:0] peer: [3:7587241174169514796:0] socket: 89 qp: -1
2025-12-24T02:10:46.806875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241175649266008:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:46.806907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.807813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893224:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.807829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241175085893224:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.794787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241176146881861:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.807621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241177282848738:2087] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.807650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.807657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241177282848738:2087] [node 4] ICP22 created new session: [1:7587241177282848785:2087]
2025-12-24T02:10:46.794848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241176146881823:2086] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.807663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241177282848785:2087] [node 4] ICS09 handshake done sender: [1:7587241177282848768:2116] self: [1:7587241177282848748:0] peer: [4:7587241175649266004:0] socket: 93 qp: -1
2025-12-24T02:10:46.807668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241177282848785:2087] [node 4] ICS10 traffic start
2025-12-24T02:10:46.807676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.810809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241174169514795:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(55966088ns) {HandleHandshakeStatus:329(55963998ns) {DropIncomingHandshake:442(9725ns) DropOutgoingHandshake:457(342ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(55796866ns) {ForwardSessionEventToSession:612(55776113ns) ForwardSessionEventToSession:612(4244ns) ForwardSessionEventToSession:612(1430ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(1572ns) ForwardSessionEventToSession:612(961ns) ForwardSessionEventToSession:612(1308ns) ForwardSessionEventToSession:612(793ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(685ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.814785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241175649266010:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:46.815104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241175649266002:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.815164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241175649265920:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.815179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:46.815184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241175649265920:2048] [node 1] ICP22 created new session: [4:7587241175649266012:2048]
2025-12-24T02:10:46.815192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241175649266012:2048] [node 1] ICS09 handshake done sender: [4:7587241175649266002:2118] self: [4:7587241175649266004:0] peer: [1:7587241177282848748:0] socket: 94 qp: -1
2025-12-24T02:10:46.815202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241175649266012:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:46.815207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:46.822787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175085893224:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.824068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241175085893201:2097] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.838726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881870:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.838776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.838803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241175085893201:2097] [node 6] ICP22 created new session: [5:7587241175085893238:2097]
2025-12-24T02:10:46.838813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175085893238:2097] [node 6] ICS09 handshake done sender: [5:7587241175085893224:2116] self: [5:7587241175085893202:0] peer: [6:7587241176146881869:0] socket: 68 qp: -1
2025-12-24T02:10:46.838824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175085893238:2097] [node 6] ICS10 traffic start
2025-12-24T02:10:46.838841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:46.852404Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241175649265920:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(37188097ns) {HandleHandshakeStatus:329(37187473ns) {DropIncomingHandshake:442(241ns) DropOutgoingHandshake:457(1590ns) SwitchToState:179(316ns) ProcessPendingSessionEvents:495(13679032ns) {ForwardSessionEventToSession:612(13661836ns) ForwardSessionEventToSession:612(3394ns) ForwardSessionEventToSession:612(1644ns) ForwardSessionEventToSession:612(1099ns) ForwardSessionEventToSession:612(1413ns) ForwardSessionEventToSession:612(941ns) ForwardSessionEventToSession:612(1030ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(680ns) ForwardSessionEventToSession:612(915ns)}}}
2025-12-24T02:10:46.854863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241174169514806:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.854915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241174169514723:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.861885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241176146881856:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.862002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241176146881779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.886863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722423:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.886893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241176727722423:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.906866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881872:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.928112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722421:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.928130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241176727722421:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.926895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241175085893201:2097] [node 6] ICP35 event processing took too much time PendingConnection:279(102744559ns) {HandleHandshakeStatus:329(102742372ns) {DropIncomingHandshake:442(6414ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(618ns) ProcessPendingSessionEvents:495(39991132ns) {ForwardSessionEventToSession:612(39958772ns) ForwardSessionEventToSession:612(7613ns) ForwardSessionEventToSession:612(1298ns) ForwardSessionEventToSession:612(927ns) ForwardSessionEventToSession:612(1387ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(1154ns) ForwardSessionEventToSession:612(1263ns) ForwardSessionEventToSession:612(2008ns) ForwardSessionEventToSession:612(924ns)}}}
2025-12-24T02:10:46.930792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241174169514793:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(83639107ns) {HandleHandshakeStatus:329(83638035ns) {DropIncomingHandshake:442(91ns) DropOutgoingHandshake:457(141ns) SwitchToState:179(320ns) ProcessPendingSessionEvents:495(27831141ns) {ForwardSessionEventToSession:612(27811840ns) ForwardSessionEventToSession:612(4859ns) ForwardSessionEventToSession:612(945ns) ForwardSessionEventToSession:612(820ns) ForwardSessionEventToSession:612(806ns) ForwardSessionEventToSession:612(800ns) ForwardSessionEventToSession:612(1040ns) ForwardSessionEventToSession:612(843ns) ForwardSessionEventToSession:612(810ns) ForwardSessionEventToSession:612(781ns)}}}
2025-12-24T02:10:46.935988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241176727722423:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.970883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241176146881870:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.970931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241176146881823:2086] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8666) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241176146881861:2118] held: no
2025-12-24T02:10:46.970999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241176146881861:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.971442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241174169514822:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6749
2025-12-24T02:10:46.971676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241174169514806:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.971761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241174169514723:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.971782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.971790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241174169514723:2049] [node 2] ICP22 created new session: [3:7587241174169514823:2049]
2025-12-24T02:10:46.971799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241174169514823:2049] [node 2] ICS09 handshake done sender: [3:7587241174169514806:2118] self: [3:7587241174169514821:0] peer: [2:7587241176727722422:0] socket: 80 qp: -1
2025-12-24T02:10:46.971808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241174169514823:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:46.971821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.978851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241176146881823:2086] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.978904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.978914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241176146881823:2086] [node 5] ICP22 created new session: [6:7587241176146881873:2086]
2025-12-24T02:10:46.978922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241176146881873:2086] [node 5] ICS09 handshake done sender: [6:7587241176146881861:2118] self: [6:7587241176146881869:0] peer: [5:7587241175085893202:0] socket: 95 qp: -1
2025-12-24T02:10:46.978934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241176146881873:2086] [node 5] ICS10 traffic start
2025-12-24T02:10:46.978947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.982773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893240:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.982884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241176727722421:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:46.983206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893240:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.983259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241175085893215:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.983423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649265996:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.983437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241175649265996:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.984437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893242:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:46.984550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649265994:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.984559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241175649265994:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.985040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241175649265996:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:46.985284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175085893240:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.985377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241175085893215:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.985398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.985405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241175085893215:2110] [node 4] ICP22 created new session: [5:7587241175085893243:2110]
2025-12-24T02:10:46.985411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241175085893243:2110] [node 4] ICS09 handshake done sender: [5:7587241175085893240:2120] self: [5:7587241175085893241:0] peer: [4:7587241175649265995:0] socket: 44 qp: -1
2025-12-24T02:10:46.985419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241175085893243:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:46.985428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.986307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241175649265990:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.986329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:46.986338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241175649265990:2110] [node 5] ICP22 created new session: [4:7587241175649266017:2110]
2025-12-24T02:10:46.986345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241175649266017:2110] [node 5] ICS09 handshake done sender: [4:7587241175649265996:2115] self: [4:7587241175649265995:0] peer: [5:7587241175085893241:0] socket: 70 qp: -1
2025-12-24T02:10:46.986351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241175649266017:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:46.986358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:46.983433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241176146881824:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.988451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241175649266000:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.988485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241175649265921:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.988596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241175649265994:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.988809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241175085893245:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.989452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848750:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.989464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241177282848750:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.989953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241177282848787:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57360
2025-12-24T02:10:46.990063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241177282848750:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:46.990137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241177282848688:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.990150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:46.990156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241177282848688:2048] [node 2] ICP22 created new session: [1:7587241177282848789:2048]
2025-12-24T02:10:46.990161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241177282848789:2048] [node 2] ICS09 handshake done sender: [1:7587241177282848750:2105] self: [1:7587241177282848747:0] peer: [2:7587241176727722445:0] socket: 60 qp: -1
2025-12-24T02:10:46.990166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241177282848789:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:46.990172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:46.991126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881876:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.991188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241176146881877:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9846
2025-12-24T02:10:46.987667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241176727722350:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.987694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:46.987701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241176727722350:2049] [node 3] ICP22 created new session: [2:7587241176727722439:2049]
2025-12-24T02:10:46.987706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241176727722439:2049] [node 3] ICS09 handshake done sender: [2:7587241176727722423:2116] self: [2:7587241176727722422:0] peer: [3:7587241174169514821:0] socket: 78 qp: -1
2025-12-24T02:10:46.987712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241176727722439:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:46.987718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:46.987892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722417:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.987900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241176727722415:2100] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.987920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:46.987925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241176727722415:2100] [node 6] ICP22 created new session: [2:7587241176727722441:2100]
2025-12-24T02:10:46.987929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241176727722441:2100] [node 6] ICS09 handshake done sender: [2:7587241176727722421:2115] self: [2:7587241176727722420:0] peer: [6:7587241176146881871:0] socket: 77 qp: -1
2025-12-24T02:10:46.987933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241176727722441:2100] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-12-24T02:10:46.987937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
2025-12-24T02:10:46.988778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722426:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.988782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241176727722426:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.989196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241176727722349:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.990387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241176727722426:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.990485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241176727722416:2103] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:46.990496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.990500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241176727722416:2103] [node 4] ICP22 created new session: [2:7587241176727722448:2103]
2025-12-24T02:10:46.990505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241176727722448:2103] [node 4] ICS09 handshake done sender: [2:7587241176727722426:2117] self: [2:7587241176727722424:0] peer: [4:7587241175649266019:0] socket: 79 qp: -1
2025-12-24T02:10:46.990509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241176727722448:2103] [node 4] ICS10 traffic start
2025-12-24T02:10:46.990514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.991973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241176727722447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34427
2025-12-24T02:10:46.992029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241175085893245:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.992059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241175085893146:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:46.993753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241176727722428:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.993766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241176727722428:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:46.992707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241176146881779:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31622) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241176146881856:2115] held: no
2025-12-24T02:10:46.992753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241176146881870:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:46.992856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241176146881824:2091] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:46.992867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:46.992873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241176146881824:2091] [node 4] ICP22 created new session: [6:7587241176146881878:2091]
2025-12-24T02:10:46.992879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241176146881878:2091] [node 4] ICS09 handshake done sender: [6:7587241176146881870:2121] self: [6:7587241176146881875:0] peer: [4:7587241175649265993:0] socket: 105 qp: -1
2025-12-24T02:10:46.992885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241176146881878:2091] [node 4] ICS10 traffic start
2025-12-24T02:10:46.992891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:46.999128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241176727722417:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:46.999206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241177282848787:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:46.999246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241177282848688:2048] [node 2] ICP18 handshake (actor [1:7587241177282848787:2118]) is too old
2025-12-24T02:10:47.004119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241176146881856:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:47.008301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241176146881779:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.008331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:47.008341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241176146881779:2049] [node 2] ICP22 created new session: [6:7587241180441849176:2049]
2025-12-24T02:10:47.008353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241180441849176:2049] [node 2] ICS09 handshake done sender: [6:7587241176146881856:2115] self: [6:7587241176146881871:0] peer: [2:7587241176727722420:0] socket: 81 qp: -1
2025-12-24T02:10:47.008363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241180441849176:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:47.008376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:47.008713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241175649265989:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.008744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.008755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241175649265989:2108] [node 6] ICP22 created new session: [4:7587241179944233317:2108]
2025-12-24T02:10:47.008762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241179944233317:2108] [node 6] ICS09 handshake done sender: [4:7587241175649265994:2114] self: [4:7587241175649265993:0] peer: [6:7587241176146881875:0] socket: 54 qp: -1
2025-12-24T02:10:47.008770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241179944233317:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:47.008779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:47.009030Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241175649265921:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31622) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241175649266000:2116] held: no
2025-12-24T02:10:47.009285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241176727722349:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4320) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241176727722417:2113] held: no
2025-12-24T02:10:47.009314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241176727722349:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.009330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:47.009336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241176727722349:2048] [node 1] ICP22 created new session: [2:7587241181022689747:2048]
2025-12-24T02:10:47.009344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241181022689747:2048] [node 1] ICS09 handshake done sender: [2:7587241176727722417:2113] self: [2:7587241176727722445:0] peer: [1:7587241177282848747:0] socket: 64 qp: -1
2025-12-24T02:10:47.009352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241181022689747:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:47.009361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.009978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241179380860543:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55841
2025-12-24T02:10:47.010132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241176727722428:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:47.010315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241175085893245:2122] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:47.010407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241175085893146:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.010422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:47.010428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241175085893146:2049] [node 2] ICP22 created new session: [5:7587241179380860544:2049]
2025-12-24T02:10:47.010434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241179380860544:2049] [node 2] ICS09 handshake done sender: [5:7587241175085893245:2122] self: [5:7587241175085893246:0] peer: [2:7587241176727722427:0] socket: 76 qp: -1
2025-12-24T02:10:47.010439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241179380860544:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:47.010447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:47.010876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241176727722419:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.010894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:47.010900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241176727722419:2108] [node 5] ICP22 created new session: [2:7587241181022689749:2108]
2025-12-24T02:10:47.010906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241181022689749:2108] [node 5] ICS09 handshake done sender: [2:7587241176727722428:2118] self: [2:7587241176727722427:0] peer: [5:7587241175085893246:0] socket: 48 qp: -1
2025-12-24T02:10:47.010911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241181022689749:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:47.010918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.011343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179944233316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5714
2025-12-24T02:10:47.011788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241175649266000:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:47.014860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241175649265921:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.014876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:47.014890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241175649265921:2049] [node 2] ICP22 created new session: [4:7587241179944233319:2049]
2025-12-24T02:10:47.014897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241179944233319:2049] [node 2] ICS09 handshake done sender: [4:7587241175649266000:2116] self: [4:7587241175649266019:0] peer: [2:7587241176727722424:0] socket: 83 qp: -1
2025-12-24T02:10:47.014904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241179944233319:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:47.014932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:47.022794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:47.022909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241181022689747:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-12-24T02:10:47.022921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241176727722349:2048] [node 1] ICP30 unregister session Session# [2:7587241181022689747:2048] VirtualId# [2:7587241176727722445:0]
2025-12-24T02:10:47.022935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241181022689747:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.023020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:47.023050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:47.023099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:47.023180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241174169514809:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-12-24T02:10:47.023187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241174169514722:2048] [node 1] ICP30 unregister session Session# [3:7587241174169514809:2048] VirtualId# [3:7587241174169514805:0]
2025-12-24T02:10:47.023193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241174169514809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.023228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:47.023248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:47.034780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:47.034887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241175649266012:2048] [node 1] ICS01 socket: 94 reason# EndOfStream
2025-12-24T02:10:47.034897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241175649265920:2048] [node 1] ICP30 unregister session Session# [4:7587241175649266012:2048] VirtualId# [4:7587241175649266004:0]
2025-12-24T02:10:47.034906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241175649266012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.034977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:47.035007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:47.034775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:47.034887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241175085893235:2048] [node 1] ICS07 socket disconnect 65 reason# EndOfStream
2025-12-24T02:10:47.034896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241175085893235:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.034976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:47.035054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:47.035000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241175085893235:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:47.035117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241176146881864:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-12-24T02:10:47.035124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241176146881778:2048] [node 1] ICP30 unregister session Session# [6:7587241176146881864:2048] VirtualId# [6:7587241176146881859:0]
2025-12-24T02:10:47.035129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241176146881864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.035154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:47.035174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:47.039202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:47.039301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241174169514823:2049] [node 2] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:10:47.039312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241174169514723:2049] [node 2] ICP30 unregister session Session# [3:7587241174169514823:2049] VirtualId# [3:7587241174169514821:0]
2025-12-24T02:10:47.039321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241174169514823:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.039357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:47.039378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:47.042952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241175085893145:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4320) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:47.042990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241175085893235:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:47.042996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241175085893145:2048] [node 1] ICP30 unregister session Session# [5:7587241175085893235:2048] VirtualId# [5:7587241175085893228:0]
2025-12-24T02:10:47.043021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:47.043028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241175085893145:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:47.043032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4320) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:47.052291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:47.052404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241179380860544:2049] [node 2] ICS01 socket: 76 reason# EndOfStream
2025-12-24T02:10:47.052415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241175085893146:2049] [node 2] ICP30 unregister session Session# [5:7587241179380860544:2049] VirtualId# [5:7587241175085893246:0]
2025-12-24T02:10:47.052425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241179380860544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.052481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:47.052508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:47.053682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:47.053773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241179944233319:2049] [node 2] ICS01 socket: 83 reason# EndOfStream
2025-12-24T02:10:47.053782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241175649265921:2049] [node 2] ICP30 unregister session Session# [4:7587241179944233319:2049] VirtualId# [4:7587241175649266019:0]
2025-12-24T02:10:47.053790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241179944233319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.053849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:47.053880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:47.054852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:47.054960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241180441849176:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-12-24T02:10:47.054976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241176146881779:2049] [node 2] ICP30 unregister session Session# [6:7587241180441849176:2049] VirtualId# [6:7587241176146881871:0]
2025-12-24T02:10:47.054983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241180441849176:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.055026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:47.055055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:47.058746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241175085893145:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:47.082788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:47.082912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241175085893231:2050] [node 3] ICS01 socket: 73 reason# EndOfStream
2025-12-24T02:10:47.082925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241175085893147:2050] [node 3] ICP30 unregister session Session# [5:7587241175085893231:2050] VirtualId# [5:7587241175085893227:0]
2025-12-24T02:10:47.082934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241175085893231:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.083019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:47.083051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:47.086800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:47.086901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241176146881866:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-12-24T02:10:47.086909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241176146881780:2050] [node 3] ICP30 unregister session Session# [6:7587241176146881866:2050] VirtualId# [6:7587241176146881860:0]
2025-12-24T02:10:47.086917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241176146881866:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.086994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:47.087024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:47.102779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:47.102907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241176146881878:2091] [node 4] ICS01 socket: 105 reason# EndOfStream
2025-12-24T02:10:47.102921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241176146881824:2091] [node 4] ICP30 unregister session Session# [6:7587241176146881878:2091] VirtualId# [6:7587241176146881875:0]
2025-12-24T02:10:47.102935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241176146881878:2091] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:47.103025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:47.103064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:47.142871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241179698535399:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:47.142892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241179698535400:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:47.156544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241180625360510:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:47.156555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241180625360511:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:47.154865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241181910667792:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:47.154883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241181910667793:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:47.170831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241179893565846:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:47.170842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241179893565847:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:47.170848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241179893565848:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:47.178835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241178751147646:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:47.178846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241178751147647:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:47.178851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241178751147648:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:47.198803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241181704043076:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:47.198821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241181704043077:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:47.198833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241181704043078:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:47.232717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241180625360571:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.233973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667852:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.234767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241179698535459:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-12-24T02:10:47.234596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241180625360580:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:47.234675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241180625360581:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:47.234873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241179893565909:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:47.234938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241179893565910:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.239085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241181704043091:2060] [node 5] ICP01 ready to work
2025-12-24T02:10:47.241123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667856:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.251146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241179698535463:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-12-24T02:10:47.255317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241179698535462:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:47.258717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241178751147692:2092] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.260242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147711:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:47.260307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147712:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:47.260319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147713:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.263265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241180625360589:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:47.267014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241181910667861:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:47.267102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667852:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.266744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241178751147704:2098] [node 6] ICP01 ready to work
2025-12-24T02:10:47.266884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241178751147711:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.267025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241178751147648:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.267192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241181704043115:2068] [node 4] ICP01 ready to work
2025-12-24T02:10:47.270776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360589:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.273682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179893565922:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:47.276093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241179893565847:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41107965ns) {Configure:104(41107372ns) {ProcessConfigured:135(40342766ns) {StartInitialHandshake:157(40342352ns) {DropHandshakes:470(111ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(39ns) PrepareNewSessionHandshake:477(1325ns) {SwitchToState:179(341ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.276318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179893565924:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:47.274828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241179698535399:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40094411ns) {Configure:104(40092759ns) {ProcessConfigured:135(39641300ns) {StartInitialHandshake:157(39640558ns) {DropHandshakes:470(440ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(797ns) {SwitchToState:179(317ns)}}}}}
2025-12-24T02:10:47.275047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241179698535469:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:47.275130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241179698535470:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:47.275191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535463:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.282802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241179893565848:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47846160ns) {Configure:104(47845688ns) {ProcessConfigured:135(47440587ns) {StartInitialHandshake:157(47439814ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(130ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(1887ns) {SwitchToState:179(1052ns)}}}}}
2025-12-24T02:10:47.287745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360575:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.287768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241180625360575:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.290894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241181704043078:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49304091ns) {Configure:104(49303106ns) {ProcessConfigured:135(49030911ns) {StartInitialHandshake:157(49029148ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(2686ns) {SwitchToState:179(1009ns)}}}}}
2025-12-24T02:10:47.290964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043148:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.291082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043149:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.291763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147725:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:47.295180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241181704043148:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.295253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241181704043149:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.298813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241181704043078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.299076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241178751147713:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.299173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241178751147692:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.299444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360584:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.299459Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241180625360584:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.300776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241178751147712:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.300904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241180625360584:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:47.300083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565916:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.300103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241179893565916:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.298897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241179698535400:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32033645ns) {Configure:104(32032164ns) {ProcessConfigured:135(31645613ns) {StartInitialHandshake:157(31644609ns) {DropHandshakes:470(991ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(2349ns) {SwitchToState:179(1619ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.305210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241181910667867:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:47.305409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667860:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
2025-12-24T02:10:47.305450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667866:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
2025-12-24T02:10:47.300456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565922:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.306791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241178751147646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47992849ns) {Configure:104(47991921ns) {ProcessConfigured:135(47538483ns) {StartInitialHandshake:157(47538040ns) {DropHandshakes:470(495ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:477(994ns) {SwitchToState:179(556ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.308374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241179698535462:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52974689ns) {Configure:104(52973467ns) {ProcessConfigured:135(52581238ns) {StartInitialHandshake:157(52580705ns) {DropHandshakes:470(1476ns) {DropIncomingHandshake:442(190ns) DropOutgoingHandshake:457(462ns)} GenerateSessionVirtualId:988(356ns) PrepareNewSessionHandshake:477(1696ns) {SwitchToState:179(1041ns)}}}}}
2025-12-24T02:10:47.308635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241179698535481:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-12-24T02:10:47.310042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241179893565916:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:47.306909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241181910667793:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39769141ns) {Configure:104(39768463ns) {ProcessConfigured:135(19827109ns) {StartInitialHandshake:157(19825192ns) {DropHandshakes:470(1620ns) {DropIncomingHandshake:442(254ns) DropOutgoingHandshake:457(390ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(733ns) {SwitchToState:179(25ns)}}}}}
2025-12-24T02:10:47.313877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241179893565909:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.313917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:47.313928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241179893565909:2108] [node 5] ICP22 created new session: [4:7587241179893565928:2108]
2025-12-24T02:10:47.313937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241179893565928:2108] [node 5] ICS09 handshake done sender: [4:7587241179893565916:2111] self: [4:7587241179893565915:0] peer: [5:7587241178751147726:0] socket: 62 qp: -1
2025-12-24T02:10:47.313949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241179893565928:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:47.313963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:47.314192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241180625360575:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:47.318735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241181910667862:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:47.318850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667866:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.318878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667860:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.331243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241180625360571:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.331277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:47.331289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241180625360571:2107] [node 5] ICP22 created new session: [3:7587241180625360594:2107]
2025-12-24T02:10:47.331297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241180625360594:2107] [node 5] ICS09 handshake done sender: [3:7587241180625360575:2109] self: [3:7587241180625360574:0] peer: [5:7587241178751147724:0] socket: 49 qp: -1
2025-12-24T02:10:47.331304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241180625360594:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:47.331320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:47.334885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241178751147704:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(67752506ns) {Configure:104(67750964ns) {ProcessConfigured:135(31899312ns) {StartInitialHandshake:157(31898099ns) {DropHandshakes:470(1593ns) {DropIncomingHandshake:442(420ns) DropOutgoingHandshake:457(290ns)} GenerateSessionVirtualId:988(662ns) PrepareNewSessionHandshake:477(1711ns) {SwitchToState:179(845ns)}}}}}
2025-12-24T02:10:47.334924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241178751147648:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241178751147711:2109] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.338814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241181704043115:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(71472018ns) {Configure:104(71471360ns) {ProcessConfigured:135(35752431ns) {StartInitialHandshake:157(35751629ns) {DropHandshakes:470(701ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(1323ns) {SwitchToState:179(948ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.339049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:47.339452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241178751147732:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.342962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179893565930:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:47.343012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179893565931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:47.344993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565931:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.343448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241178751147713:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:47.343697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241178751147692:2092] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.343711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:47.343719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241178751147692:2092] [node 4] ICP22 created new session: [5:7587241178751147733:2092]
2025-12-24T02:10:47.343728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241178751147733:2092] [node 4] ICS09 handshake done sender: [5:7587241178751147713:2111] self: [5:7587241178751147726:0] peer: [4:7587241179893565915:0] socket: 67 qp: -1
2025-12-24T02:10:47.343736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241178751147733:2092] [node 4] ICS10 traffic start
2025-12-24T02:10:47.343747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:47.354777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241180625360580:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.354826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.354845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241180625360580:2110] [node 6] ICP22 created new session: [3:7587241180625360597:2110]
2025-12-24T02:10:47.354853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241180625360597:2110] [node 6] ICS09 handshake done sender: [3:7587241180625360584:2112] self: [3:7587241180625360582:0] peer: [6:7587241181704043154:0] socket: 58 qp: -1
2025-12-24T02:10:47.354865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241180625360597:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:47.354905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:47.367084Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241179893565909:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(53158900ns) {HandleHandshakeStatus:329(53156888ns) {DropIncomingHandshake:442(7747ns) DropOutgoingHandshake:457(801ns) SwitchToState:179(244ns) ProcessPendingSessionEvents:495(51945420ns) {ForwardSessionEventToSession:612(51927737ns) ForwardSessionEventToSession:612(2544ns) ForwardSessionEventToSession:612(1032ns) ForwardSessionEventToSession:612(866ns) ForwardSessionEventToSession:612(1064ns) ForwardSessionEventToSession:612(1056ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(794ns)}}}
2025-12-24T02:10:47.371020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360586:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.371040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241180625360586:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.372069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241180625360586:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:47.372169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241180625360581:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.372188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:47.372198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241180625360581:2111] [node 4] ICP22 created new session: [3:7587241180625360600:2111]
2025-12-24T02:10:47.372206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241180625360600:2111] [node 4] ICS09 handshake done sender: [3:7587241180625360586:2113] self: [3:7587241180625360583:0] peer: [4:7587241179893565933:0] socket: 91 qp: -1
2025-12-24T02:10:47.372213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241180625360600:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:47.372224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:47.375260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241178751147711:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.375705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241178751147648:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.375725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:47.375734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241178751147648:2050] [node 3] ICP22 created new session: [5:7587241178751147736:2050]
2025-12-24T02:10:47.375742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241178751147736:2050] [node 3] ICS09 handshake done sender: [5:7587241178751147711:2109] self: [5:7587241178751147724:0] peer: [3:7587241180625360574:0] socket: 54 qp: -1
2025-12-24T02:10:47.375751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241178751147736:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:47.375759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.375480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241179893565934:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:47.378904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241181704043115:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.379058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043157:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.382909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565918:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.382928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241179893565918:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.383614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241179893565918:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:47.383703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241179893565910:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.383717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.383731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241179893565910:2109] [node 6] ICP22 created new session: [4:7587241179893565936:2109]
2025-12-24T02:10:47.383739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241179893565936:2109] [node 6] ICS09 handshake done sender: [4:7587241179893565918:2112] self: [4:7587241179893565917:0] peer: [6:7587241181704043159:0] socket: 63 qp: -1
2025-12-24T02:10:47.383746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241179893565936:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:47.383756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:47.384151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241180625360602:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:47.385629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360602:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.385668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241180625360581:2111] [node 4] ICP18 handshake (actor [3:7587241180625360602:2116]) is too old
2025-12-24T02:10:47.387602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241179893565848:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.411041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241179893565848:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(39293179ns) {IncomingHandshake:261(39292422ns) {DropIncomingHandshake:442(488ns) PrepareNewSessionHandshake:477(1344ns) {SwitchToState:179(846ns)} GenerateSessionVirtualId:988(280ns) IssueIncomingHandshakeReply:193(39283964ns) {SwitchToState:179(1085ns)}}}
2025-12-24T02:10:47.411125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241179893565848:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16702) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241179893565931:2118] held: no
2025-12-24T02:10:47.411168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241179893565931:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:47.418753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241179893565848:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.418787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:47.418802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241179893565848:2050] [node 3] ICP22 created new session: [4:7587241179893565939:2050]
2025-12-24T02:10:47.418811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241179893565939:2050] [node 3] ICS09 handshake done sender: [4:7587241179893565931:2118] self: [4:7587241179893565933:0] peer: [3:7587241180625360583:0] socket: 97 qp: -1
2025-12-24T02:10:47.418824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241179893565939:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:47.418845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.459004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241179698535482:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
2025-12-24T02:10:47.459100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241179698535483:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.462994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535482:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.462847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241181910667862:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(143846644ns) {Configure:104(143844679ns) {ProcessConfigured:135(143464132ns) {StartInitialHandshake:157(143462864ns) {DropHandshakes:470(1161ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(1009ns) PrepareNewSessionHandshake:477(2129ns) {SwitchToState:179(1384ns)}}}}}
2025-12-24T02:10:47.462965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667875:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
2025-12-24T02:10:47.463068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241181910667792:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.466976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241181704043148:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:47.466853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535468:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.466870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241179698535468:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.467659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241179698535468:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:47.467744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241179698535399:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.467768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:47.467776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241179698535399:2048] [node 2] ICP22 created new session: [1:7587241179698535486:2048]
2025-12-24T02:10:47.467786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241179698535486:2048] [node 2] ICS09 handshake done sender: [1:7587241179698535468:2112] self: [1:7587241179698535460:0] peer: [2:7587241181910667877:0] socket: 77 qp: -1
2025-12-24T02:10:47.467793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241179698535486:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:47.467803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:47.474831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241180625360580:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(173764300ns) {HandleHandshakeStatus:329(173762759ns) {DropIncomingHandshake:442(864ns) DropOutgoingHandshake:457(933ns) SwitchToState:179(156ns) ProcessPendingSessionEvents:495(87192422ns) {ForwardSessionEventToSession:612(87173337ns) ForwardSessionEventToSession:612(3523ns) ForwardSessionEventToSession:612(1251ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(687ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(836ns)}}}
2025-12-24T02:10:47.474828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667875:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.506849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241179893565848:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(88055119ns) {HandleHandshakeStatus:329(88053657ns) {DropIncomingHandshake:442(565ns) DropOutgoingHandshake:457(332ns) SwitchToState:179(266ns) ProcessPendingSessionEvents:495(87862558ns) {ForwardSessionEventToSession:612(87856140ns) ForwardSessionEventToSession:612(2840ns)}}}
2025-12-24T02:10:47.507091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535483:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.512762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241180625360603:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:47.513065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043158:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.518956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.519197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360603:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.519251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241180625360510:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.522878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535479:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.522899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241179698535479:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.523589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241179698535479:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:47.523690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241179698535400:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.523720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:47.523734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241179698535400:2049] [node 3] ICP22 created new session: [1:7587241179698535491:2049]
2025-12-24T02:10:47.523742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241179698535491:2049] [node 3] ICS09 handshake done sender: [1:7587241179698535479:2115] self: [1:7587241179698535465:0] peer: [3:7587241180625360604:0] socket: 107 qp: -1
2025-12-24T02:10:47.523751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241179698535491:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:47.523763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:47.526912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241180625360510:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241180625360603:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.543208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241181704043078:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.543247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:47.543272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241181704043078:2050] [node 3] ICP22 created new session: [6:7587241181704043162:2050]
2025-12-24T02:10:47.543282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241181704043162:2050] [node 3] ICS09 handshake done sender: [6:7587241181704043148:2112] self: [6:7587241181704043154:0] peer: [3:7587241180625360582:0] socket: 60 qp: -1
2025-12-24T02:10:47.543294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241181704043162:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:47.543307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:47.543627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043161:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.543810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241181704043149:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.567187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241180625360605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:47.567420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241180625360603:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:47.567515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241180625360510:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.567542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:47.567553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241180625360510:2048] [node 1] ICP22 created new session: [3:7587241180625360606:2048]
2025-12-24T02:10:47.567563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241180625360606:2048] [node 1] ICS09 handshake done sender: [3:7587241180625360603:2117] self: [3:7587241180625360604:0] peer: [1:7587241179698535465:0] socket: 108 qp: -1
2025-12-24T02:10:47.567575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241180625360606:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:47.567587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:47.572340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241181704043161:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.572368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241181704043115:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.572389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:47.572404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241181704043115:2068] [node 4] ICP22 created new session: [6:7587241181704043165:2068]
2025-12-24T02:10:47.572410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241181704043165:2068] [node 4] ICS09 handshake done sender: [6:7587241181704043149:2113] self: [6:7587241181704043159:0] peer: [4:7587241179893565917:0] socket: 72 qp: -1
2025-12-24T02:10:47.572422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241181704043165:2068] [node 4] ICS10 traffic start
2025-12-24T02:10:47.572433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.576301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535480:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.576316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241179698535480:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.577455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241179698535480:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:47.577951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241179698535462:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.577963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.577968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241179698535462:2108] [node 6] ICP22 created new session: [1:7587241179698535495:2108]
2025-12-24T02:10:47.577974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241179698535495:2108] [node 6] ICS09 handshake done sender: [1:7587241179698535480:2116] self: [1:7587241179698535464:0] peer: [6:7587241181704043169:0] socket: 95 qp: -1
2025-12-24T02:10:47.577980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241179698535495:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:47.577986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:47.575333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043164:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.575578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241181704043091:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.576192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241181704043164:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.576215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241181704043076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.577344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043170:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.577648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241181704043164:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:47.577694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241181704043076:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.577706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:47.577712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241181704043076:2048] [node 1] ICP22 created new session: [6:7587241181704043171:2048]
2025-12-24T02:10:47.577719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241181704043171:2048] [node 1] ICS09 handshake done sender: [6:7587241181704043164:2121] self: [6:7587241181704043169:0] peer: [1:7587241179698535464:0] socket: 73 qp: -1
2025-12-24T02:10:47.577726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241181704043171:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:47.577735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:47.578858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241178751147729:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.578885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241178751147729:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.579587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241178751147729:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:47.579682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241178751147704:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.579705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.579716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241178751147704:2098] [node 6] ICP22 created new session: [5:7587241178751147740:2098]
2025-12-24T02:10:47.579725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241178751147740:2098] [node 6] ICS09 handshake done sender: [5:7587241178751147729:2118] self: [5:7587241178751147727:0] peer: [6:7587241181704043168:0] socket: 69 qp: -1
2025-12-24T02:10:47.579734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241178751147740:2098] [node 6] ICS10 traffic start
2025-12-24T02:10:47.579745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.627288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241181704043091:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32442) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241181704043161:2120] held: no
2025-12-24T02:10:47.627425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043173:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.632196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241181704043161:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:47.632341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241181704043091:2060] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.632364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:47.632377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241181704043091:2060] [node 5] ICP22 created new session: [6:7587241181704043174:2060]
2025-12-24T02:10:47.632385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241181704043174:2060] [node 5] ICS09 handshake done sender: [6:7587241181704043161:2120] self: [6:7587241181704043168:0] peer: [5:7587241178751147727:0] socket: 105 qp: -1
2025-12-24T02:10:47.632400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241181704043174:2060] [node 5] ICS10 traffic start
2025-12-24T02:10:47.632413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.716428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667856:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.716490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241181910667879:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:570
2025-12-24T02:10:47.724248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241181704043158:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.724322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241181704043077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.742989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241181910667866:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:47.767152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241181910667792:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.767199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:47.767216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241181910667792:2048] [node 1] ICP22 created new session: [2:7587241181910667880:2048]
2025-12-24T02:10:47.767224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241181910667880:2048] [node 1] ICS09 handshake done sender: [2:7587241181910667866:2115] self: [2:7587241181910667877:0] peer: [1:7587241179698535460:0] socket: 86 qp: -1
2025-12-24T02:10:47.767238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241181910667880:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:47.767250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:47.807216Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241181910667792:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40023720ns) {HandleHandshakeStatus:329(40021633ns) {DropIncomingHandshake:442(450ns) DropOutgoingHandshake:457(2769ns) SwitchToState:179(381ns) ProcessPendingSessionEvents:495(39795299ns) {ForwardSessionEventToSession:612(39773213ns) ForwardSessionEventToSession:612(4054ns) ForwardSessionEventToSession:612(1736ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(1736ns) ForwardSessionEventToSession:612(853ns) ForwardSessionEventToSession:612(955ns) ForwardSessionEventToSession:612(1056ns) ForwardSessionEventToSession:612(949ns)}}}
2025-12-24T02:10:47.807399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667870:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.807404Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241181910667870:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:47.859289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241181910667870:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.886957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241181704043177:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15436
2025-12-24T02:10:47.887393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241181704043158:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:47.887083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241181910667867:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:47.887120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:47.887132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241181910667867:2112] [node 6] ICP22 created new session: [2:7587241181910667885:2112]
2025-12-24T02:10:47.887142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241181910667885:2112] [node 6] ICS09 handshake done sender: [2:7587241181910667870:2116] self: [2:7587241181910667869:0] peer: [6:7587241181704043176:0] socket: 88 qp: -1
2025-12-24T02:10:47.887152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241181910667885:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:47.887165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:47.928859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535481:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.928908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241179698535462:2108] [node 6] ICP18 handshake (actor [1:7587241179698535481:2117]) is too old
2025-12-24T02:10:47.928652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241181704043077:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:47.928693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:47.928707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241181704043077:2049] [node 2] ICP22 created new session: [6:7587241181704043178:2049]
2025-12-24T02:10:47.928714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241181704043178:2049] [node 2] ICS09 handshake done sender: [6:7587241181704043158:2118] self: [6:7587241181704043176:0] peer: [2:7587241181910667869:0] socket: 101 qp: -1
2025-12-24T02:10:47.928727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241181704043178:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:47.928739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:47.942875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241181910667867:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(55738968ns) {HandleHandshakeStatus:329(55736756ns) {DropIncomingHandshake:442(8062ns) DropOutgoingHandshake:457(370ns) SwitchToState:179(290ns) ProcessPendingSessionEvents:495(55567691ns) {ForwardSessionEventToSession:612(55548190ns) ForwardSessionEventToSession:612(3678ns) ForwardSessionEventToSession:612(940ns) ForwardSessionEventToSession:612(2887ns) ForwardSessionEventToSession:612(1062ns) ForwardSessionEventToSession:612(867ns) ForwardSessionEventToSession:612(921ns) ForwardSessionEventToSession:612(864ns) ForwardSessionEventToSession:612(796ns) ForwardSessionEventToSession:612(1534ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:47.979371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241180625360608:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:47.994812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241180625360608:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.994880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241180625360511:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:47.995086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667872:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:47.995103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241181910667872:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.059728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241181910667872:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.063878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241184920327906:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15937
2025-12-24T02:10:48.064003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241181910667793:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.064037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.064044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241181910667793:2049] [node 3] ICP22 created new session: [2:7587241186205635185:2049]
2025-12-24T02:10:48.064053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241186205635185:2049] [node 3] ICS09 handshake done sender: [2:7587241181910667872:2117] self: [2:7587241181910667863:0] peer: [3:7587241180625360609:0] socket: 56 qp: -1
2025-12-24T02:10:48.064061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241186205635185:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:48.064074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.064570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241180625360608:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.065622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241180625360511:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14850) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241180625360608:2119] held: no
2025-12-24T02:10:48.065647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241180625360511:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.065657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.065662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241180625360511:2049] [node 2] ICP22 created new session: [3:7587241184920327907:2049]
2025-12-24T02:10:48.065668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241184920327907:2049] [node 2] ICS09 handshake done sender: [3:7587241180625360608:2119] self: [3:7587241180625360609:0] peer: [2:7587241181910667863:0] socket: 59 qp: -1
2025-12-24T02:10:48.065674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241184920327907:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.065682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.070739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241183046115038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:48.091050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241183046115038:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.091114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241178751147647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.091311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667876:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.091325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241181910667876:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.111545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241183046115040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:48.111750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241181910667876:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.112008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241183046115038:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.112137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241178751147647:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.112164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.112174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241178751147647:2049] [node 2] ICP22 created new session: [5:7587241183046115041:2049]
2025-12-24T02:10:48.112182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241183046115041:2049] [node 2] ICS09 handshake done sender: [5:7587241183046115038:2121] self: [5:7587241183046115039:0] peer: [2:7587241181910667874:0] socket: 116 qp: -1
2025-12-24T02:10:48.112190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241183046115041:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.112200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:48.113246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241181910667862:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.113271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.113278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241181910667862:2111] [node 5] ICP22 created new session: [2:7587241186205635189:2111]
2025-12-24T02:10:48.113284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241186205635189:2111] [node 5] ICS09 handshake done sender: [2:7587241181910667876:2119] self: [2:7587241181910667874:0] peer: [5:7587241183046115039:0] socket: 50 qp: -1
2025-12-24T02:10:48.113291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241186205635189:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:48.113301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.114800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565924:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.114867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241179893565846:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.118832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535472:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.118848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241179698535472:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.182852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241179698535472:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.183018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241179698535469:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.183044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.183055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241179698535469:2110] [node 4] ICP22 created new session: [1:7587241183993502794:2110]
2025-12-24T02:10:48.183062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241183993502794:2110] [node 4] ICS09 handshake done sender: [1:7587241179698535472:2113] self: [1:7587241179698535471:0] peer: [4:7587241184188533237:0] socket: 78 qp: -1
2025-12-24T02:10:48.183070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241183993502794:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:48.183083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.223635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241179698535469:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(40580003ns) {HandleHandshakeStatus:329(40578110ns) {DropIncomingHandshake:442(6353ns) DropOutgoingHandshake:457(308ns) SwitchToState:179(503ns) ProcessPendingSessionEvents:495(40459254ns) {ForwardSessionEventToSession:612(40443165ns) ForwardSessionEventToSession:612(3404ns) ForwardSessionEventToSession:612(2073ns) ForwardSessionEventToSession:612(886ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(1034ns) ForwardSessionEventToSession:612(606ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(699ns)}}}
2025-12-24T02:10:48.223215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184188533238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:48.223956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241179893565846:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241179893565924:2115] held: no
2025-12-24T02:10:48.224040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241179893565924:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.259607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535459:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.259654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241179698535399:2048] [node 2] ICP18 handshake (actor [1:7587241179698535459:2107]) is too old
2025-12-24T02:10:48.260073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241179893565846:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.260119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.260129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241179893565846:2048] [node 1] ICP22 created new session: [4:7587241184188533239:2048]
2025-12-24T02:10:48.260136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184188533239:2048] [node 1] ICS09 handshake done sender: [4:7587241179893565924:2115] self: [4:7587241184188533237:0] peer: [1:7587241179698535471:0] socket: 80 qp: -1
2025-12-24T02:10:48.260147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184188533239:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.260157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:48.340065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241179893565930:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.340132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241179893565847:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.342807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241181910667865:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.342825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241181910667865:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.395503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241181910667865:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.398739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241181910667861:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.419917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241179893565847:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14850) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241179893565930:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.424389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184188533242:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18249
2025-12-24T02:10:48.424565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241179893565930:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.424799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241179893565847:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.424833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.424838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241179893565847:2049] [node 2] ICP22 created new session: [4:7587241184188533243:2049]
2025-12-24T02:10:48.424843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184188533243:2049] [node 2] ICS09 handshake done sender: [4:7587241179893565930:2117] self: [4:7587241184188533241:0] peer: [2:7587241181910667864:0] socket: 94 qp: -1
2025-12-24T02:10:48.424847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184188533243:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.424854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:48.424055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.424086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241181910667861:2110] [node 4] ICP22 created new session: [2:7587241186205635192:2110]
2025-12-24T02:10:48.424103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241186205635192:2110] [node 4] ICS09 handshake done sender: [2:7587241181910667865:2114] self: [2:7587241181910667864:0] peer: [4:7587241184188533241:0] socket: 87 qp: -1
2025-12-24T02:10:48.424115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241186205635192:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:48.424132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:48.424651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241181910667861:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(25893719ns) {HandleHandshakeStatus:329(25892022ns) {DropIncomingHandshake:442(7463ns) DropOutgoingHandshake:457(212ns) SwitchToState:179(507ns) ProcessPendingSessionEvents:495(423176ns) {ForwardSessionEventToSession:612(418573ns) ForwardSessionEventToSession:612(1050ns) ForwardSessionEventToSession:612(825ns) ForwardSessionEventToSession:612(896ns)}}}
2025-12-24T02:10:48.466842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241178751147725:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.466917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241178751147646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.470825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241179698535474:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.470848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241179698535474:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.474820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241179698535474:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.478911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241183046115044:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47742
2025-12-24T02:10:48.482926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241178751147725:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.483016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241178751147646:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.483035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.483048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241178751147646:2048] [node 1] ICP22 created new session: [5:7587241183046115045:2048]
2025-12-24T02:10:48.483055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241183046115045:2048] [node 1] ICS09 handshake done sender: [5:7587241178751147725:2116] self: [5:7587241183046115043:0] peer: [1:7587241179698535473:0] socket: 83 qp: -1
2025-12-24T02:10:48.483064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241183046115045:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.483076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:48.490792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241179698535470:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.490821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.490830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241179698535470:2111] [node 5] ICP22 created new session: [1:7587241183993502797:2111]
2025-12-24T02:10:48.490836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241183993502797:2111] [node 5] ICS09 handshake done sender: [1:7587241179698535474:2114] self: [1:7587241179698535473:0] peer: [5:7587241183046115043:0] socket: 79 qp: -1
2025-12-24T02:10:48.490845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241183993502797:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:48.490867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.520858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:48.520971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241181910667880:2048] [node 1] ICS07 socket disconnect 86 reason# EndOfStream
2025-12-24T02:10:48.520979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241181910667880:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.521029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.521049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241181910667880:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:48.523538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:48.523631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241180625360606:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-12-24T02:10:48.523640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241180625360510:2048] [node 1] ICP30 unregister session Session# [3:7587241180625360606:2048] VirtualId# [3:7587241180625360604:0]
2025-12-24T02:10:48.523647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241180625360606:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.523693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.523721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.524578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:48.524687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241184188533239:2048] [node 1] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:10:48.524699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241179893565846:2048] [node 1] ICP30 unregister session Session# [4:7587241184188533239:2048] VirtualId# [4:7587241184188533237:0]
2025-12-24T02:10:48.524707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241184188533239:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.524793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.524825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.525254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241181910667792:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:48.525265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241181910667880:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:48.525592Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241181910667792:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:48.525599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241181910667880:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:48.525603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241181910667792:2048] [node 1] ICP30 unregister session Session# [2:7587241181910667880:2048] VirtualId# [2:7587241181910667877:0]
2025-12-24T02:10:48.525621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.525627Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241181910667792:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:48.525629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:48.525151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:48.525229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241181704043171:2048] [node 1] ICS01 socket: 73 reason# EndOfStream
2025-12-24T02:10:48.525237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241181704043076:2048] [node 1] ICP30 unregister session Session# [6:7587241181704043171:2048] VirtualId# [6:7587241181704043169:0]
2025-12-24T02:10:48.525242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241181704043171:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.525303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.525331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.530779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:48.530871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241183046115045:2048] [node 1] ICS01 socket: 83 reason# EndOfStream
2025-12-24T02:10:48.530879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241178751147646:2048] [node 1] ICP30 unregister session Session# [5:7587241183046115045:2048] VirtualId# [5:7587241183046115043:0]
2025-12-24T02:10:48.530887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241183046115045:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.530950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.530998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.534761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:48.534854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241184920327907:2049] [node 2] ICS01 socket: 59 reason# EndOfStream
2025-12-24T02:10:48.534862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241180625360511:2049] [node 2] ICP30 unregister session Session# [3:7587241184920327907:2049] VirtualId# [3:7587241180625360609:0]
2025-12-24T02:10:48.534868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241184920327907:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.534909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.534926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.539522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:48.539541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:48.539668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241184188533243:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-12-24T02:10:48.539678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241179893565847:2049] [node 2] ICP30 unregister session Session# [4:7587241184188533243:2049] VirtualId# [4:7587241184188533241:0]
2025-12-24T02:10:48.539686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241184188533243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.539756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.539788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.539914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241183046115041:2049] [node 2] ICS01 socket: 116 reason# EndOfStream
2025-12-24T02:10:48.539924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241178751147647:2049] [node 2] ICP30 unregister session Session# [5:7587241183046115041:2049] VirtualId# [5:7587241183046115039:0]
2025-12-24T02:10:48.539930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241183046115041:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.539968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.539992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.539589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:48.539668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241181704043178:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-12-24T02:10:48.539677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241181704043077:2049] [node 2] ICP30 unregister session Session# [6:7587241181704043178:2049] VirtualId# [6:7587241181704043176:0]
2025-12-24T02:10:48.539684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241181704043178:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.539738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.539767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.551883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:48.551961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241179893565939:2050] [node 3] ICS01 socket: 97 reason# EndOfStream
2025-12-24T02:10:48.551967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241179893565848:2050] [node 3] ICP30 unregister session Session# [4:7587241179893565939:2050] VirtualId# [4:7587241179893565933:0]
2025-12-24T02:10:48.551972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241179893565939:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.552010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:48.552037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:48.551364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:48.551482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241178751147736:2050] [node 3] ICS01 socket: 54 reason# EndOfStream
2025-12-24T02:10:48.551492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241178751147648:2050] [node 3] ICP30 unregister session Session# [5:7587241178751147736:2050] VirtualId# [5:7587241178751147724:0]
2025-12-24T02:10:48.551499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241178751147736:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.551573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:48.551603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:48.551573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:48.551697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241181704043162:2050] [node 3] ICS01 socket: 60 reason# EndOfStream
2025-12-24T02:10:48.551706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241181704043078:2050] [node 3] ICP30 unregister session Session# [6:7587241181704043162:2050] VirtualId# [6:7587241181704043154:0]
2025-12-24T02:10:48.551713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241181704043162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.551771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:48.551803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:48.573659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:48.573758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241178751147733:2092] [node 4] ICS01 socket: 67 reason# EndOfStream
2025-12-24T02:10:48.573768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241178751147692:2092] [node 4] ICP30 unregister session Session# [5:7587241178751147733:2092] VirtualId# [5:7587241178751147726:0]
2025-12-24T02:10:48.573775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241178751147733:2092] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.573842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:48.573871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:48.573917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:48.573989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241181704043165:2068] [node 4] ICS01 socket: 72 reason# EndOfStream
2025-12-24T02:10:48.573996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241181704043115:2068] [node 4] ICP30 unregister session Session# [6:7587241181704043165:2068] VirtualId# [6:7587241181704043159:0]
2025-12-24T02:10:48.574001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241181704043165:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.574037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:48.574060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:48.614919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241184194512749:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:48.614928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241184194512750:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:48.618802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241186364336976:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:48.618811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241186364336977:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:48.626798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241184602558680:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:48.626818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241184602558681:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:48.626017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241184641263531:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:48.626025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241184641263532:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:48.626029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241184641263533:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:48.638841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241182521572871:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:48.638852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241182521572872:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:48.638858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241182521572873:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:48.654969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241184602558716:2077] [node 6] ICP01 ready to work
2025-12-24T02:10:48.654995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241184602558715:2069] [node 4] ICP01 ready to work
2025-12-24T02:10:48.655058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241184194512797:2087] [node 6] ICP01 ready to work
2025-12-24T02:10:48.655119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241184602558720:2074] [node 5] ICP01 ready to work
2025-12-24T02:10:48.655320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241184194512800:2083] [node 5] ICP01 ready to work
2025-12-24T02:10:48.671235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241184602558680:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16189057ns) {Configure:104(16188487ns) {ProcessConfigured:135(15626193ns) {StartInitialHandshake:157(15625951ns) {DropHandshakes:470(445ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(1270ns) {SwitchToState:179(578ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.679196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241184602558716:2077] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24154963ns) {Configure:104(24154461ns) {ProcessConfigured:135(23371595ns) {StartInitialHandshake:157(23371212ns) {DropHandshakes:470(173ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:477(590ns) {SwitchToState:179(245ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.678770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241186364336976:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23712047ns) {Configure:104(23710206ns) {ProcessConfigured:135(23012284ns) {StartInitialHandshake:157(23012074ns) {DropHandshakes:470(251ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(30ns)}}}}}
2025-12-24T02:10:48.679364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241184641263531:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24295371ns) {Configure:104(24294125ns) {ProcessConfigured:135(23510775ns) {StartInitialHandshake:157(23510537ns) {DropHandshakes:470(200ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(794ns) {SwitchToState:179(32ns)}}}}}
2025-12-24T02:10:48.679052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337045:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.679067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337046:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.681250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241184602558753:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-12-24T02:10:48.687121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241182521572936:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.691195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241184602558720:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24398709ns) {Configure:104(24397741ns) {ProcessConfigured:135(24033970ns) {StartInitialHandshake:157(24033709ns) {DropHandshakes:470(1206ns) {DropIncomingHandshake:442(364ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(779ns) {SwitchToState:179(522ns)}}}}}
2025-12-24T02:10:48.690799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241184194512800:2083] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27902647ns) {Configure:104(27901304ns) {ProcessConfigured:135(27602140ns) {StartInitialHandshake:157(27601575ns) {DropHandshakes:470(712ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(363ns) PrepareNewSessionHandshake:477(1678ns) {SwitchToState:179(945ns)}}}}}
2025-12-24T02:10:48.690868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241184194512823:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64269
2025-12-24T02:10:48.690933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241184194512821:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64269
2025-12-24T02:10:48.691009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241184194512822:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64269
2025-12-24T02:10:48.691403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512821:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.694806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241182521572873:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27881410ns) {Configure:104(27880544ns) {ProcessConfigured:135(26643319ns) {StartInitialHandshake:157(26642492ns) {DropHandshakes:470(466ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(1608ns) {SwitchToState:179(721ns)}}}}}
2025-12-24T02:10:48.695475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572939:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.695656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241182521572871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28621738ns) {Configure:104(28620489ns) {ProcessConfigured:135(27402834ns) {StartInitialHandshake:157(27402519ns) {DropHandshakes:470(238ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(732ns) {SwitchToState:179(73ns)}}}}}
2025-12-24T02:10:48.698863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241184194512799:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.701045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337046:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.701159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241186364336977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.701241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337049:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.703235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558753:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.705005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512822:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.706082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241184194512749:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50952367ns) {Configure:104(50951840ns) {ProcessConfigured:135(50689736ns) {StartInitialHandshake:157(50689226ns) {DropHandshakes:470(666ns) {DropIncomingHandshake:442(115ns) DropOutgoingHandshake:457(155ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(642ns) {SwitchToState:179(204ns)}}}}}
2025-12-24T02:10:48.706973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572939:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.707018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241182521572871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.706294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241184602558760:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-12-24T02:10:48.707149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512826:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.707159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241184194512826:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.711250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241184641263561:2066] [node 6] ICP01 ready to work
2025-12-24T02:10:48.714794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558748:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.714812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241184602558748:2107] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.718603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241184602558761:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-12-24T02:10:48.715502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512806:2098] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.714722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241186364337009:2075] [node 4] ICP01 ready to work
2025-12-24T02:10:48.715521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241184194512806:2098] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.716688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241184194512806:2098] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:48.715017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337045:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.717782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241184194512750:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.715072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241186364336976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.717799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.716589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337054:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.716919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241186364337045:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.716980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241186364336976:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.717000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.717007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241186364336976:2048] [node 1] ICP22 created new session: [3:7587241186364337055:2048]
2025-12-24T02:10:48.717016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241186364337055:2048] [node 1] ICS09 handshake done sender: [3:7587241186364337045:2112] self: [3:7587241186364337052:0] peer: [1:7587241184194512798:0] socket: 55 qp: -1
2025-12-24T02:10:48.717024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241186364337055:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.717034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.717806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241184194512750:2049] [node 3] ICP22 created new session: [1:7587241184194512833:2049]
2025-12-24T02:10:48.717812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241184194512833:2049] [node 3] ICS09 handshake done sender: [1:7587241184194512806:2098] self: [1:7587241184194512798:0] peer: [3:7587241186364337052:0] socket: 51 qp: -1
2025-12-24T02:10:48.717819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241184194512833:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:48.717826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.720558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512830:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.720562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241184194512830:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.721428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241184194512830:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.718785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241184602558715:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(63612697ns) {Configure:104(63612182ns) {ProcessConfigured:135(62749839ns) {StartInitialHandshake:157(62748515ns) {DropHandshakes:470(107ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(39ns) PrepareNewSessionHandshake:477(1548ns) {SwitchToState:179(404ns)}}}}}
2025-12-24T02:10:48.720380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558761:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.720419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241184602558680:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.721291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241184602558764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-12-24T02:10:48.721593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241184602558761:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.721685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241184602558680:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.721700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.721708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241184602558680:2048] [node 1] ICP22 created new session: [2:7587241184602558765:2048]
2025-12-24T02:10:48.721713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241184602558765:2048] [node 1] ICS09 handshake done sender: [2:7587241184602558761:2116] self: [2:7587241184602558763:0] peer: [1:7587241184194512801:0] socket: 67 qp: -1
2025-12-24T02:10:48.721721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241184602558765:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.721753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.724274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337057:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.722830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241184194512837:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64269
2025-12-24T02:10:48.722918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241184194512799:2093] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23902307ns) {Configure:104(23889841ns) {ProcessConfigured:135(23560579ns) {StartInitialHandshake:157(23560117ns) {DropHandshakes:470(694ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(1039ns) {SwitchToState:179(181ns)}}}}}
2025-12-24T02:10:48.724216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241184194512826:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.724290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241184194512840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64269
2025-12-24T02:10:48.724419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241184194512749:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.724438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.724447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241184194512749:2048] [node 2] ICP22 created new session: [1:7587241184194512841:2048]
2025-12-24T02:10:48.724453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241184194512841:2048] [node 2] ICS09 handshake done sender: [1:7587241184194512830:2116] self: [1:7587241184194512801:0] peer: [2:7587241184602558763:0] socket: 59 qp: -1
2025-12-24T02:10:48.724460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241184194512841:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:48.724467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:48.698728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241185249469166:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:48.698778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241185249469168:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:48.723039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241185249469168:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16170750ns) {Configure:104(16169028ns) {ProcessConfigured:135(15779698ns) {StartInitialHandshake:157(15779539ns) {DropHandshakes:470(774ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:477(1092ns) {SwitchToState:179(188ns)}}}}}
2025-12-24T02:10:48.723509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469234:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.723572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469235:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.730916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241185249469238:2070] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.733071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263604:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.733095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263605:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.733675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241184602558748:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:48.730711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241186364337037:2077] [node 5] ICP01 ready to work
2025-12-24T02:10:48.734805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263605:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.734841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241184641263531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.734787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241184194512800:2083] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.734806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.734813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241184194512800:2083] [node 5] ICP22 created new session: [1:7587241184194512844:2083]
2025-12-24T02:10:48.734820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241184194512844:2083] [node 5] ICS09 handshake done sender: [1:7587241184194512826:2115] self: [1:7587241184194512807:0] peer: [5:7587241182521572946:0] socket: 74 qp: -1
2025-12-24T02:10:48.734826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241184194512844:2083] [node 5] ICS10 traffic start
2025-12-24T02:10:48.734839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.734968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512823:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.742147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512838:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.742172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241184194512838:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.738860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241184602558681:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.738894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.738904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241184602558681:2049] [node 3] ICP22 created new session: [2:7587241184602558769:2049]
2025-12-24T02:10:48.738912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241184602558769:2049] [node 3] ICS09 handshake done sender: [2:7587241184602558748:2107] self: [2:7587241184602558747:0] peer: [3:7587241186364337050:0] socket: 54 qp: -1
2025-12-24T02:10:48.738921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241184602558769:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:48.738931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.742101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241185249469167:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:48.742160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241185249469234:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.742179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241185249469235:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.742236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241185249469166:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.738796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241186364337009:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23957682ns) {Configure:104(23956658ns) {ProcessConfigured:135(23605861ns) {StartInitialHandshake:157(23605168ns) {DropHandshakes:470(1234ns) {DropIncomingHandshake:442(138ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:477(1161ns) {SwitchToState:179(388ns)}}}}}
2025-12-24T02:10:48.746445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241182521572872:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(59229208ns) {Configure:104(59224565ns) {ProcessConfigured:135(58885347ns) {StartInitialHandshake:157(58884637ns) {DropHandshakes:470(1350ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(1124ns) {SwitchToState:179(675ns)}}}}}
2025-12-24T02:10:48.742763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241185249469222:2061] [node 4] ICP01 ready to work
2025-12-24T02:10:48.743015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558760:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.746606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241186364336977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241186364337046:2113] held: no
2025-12-24T02:10:48.747643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337062:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.751206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512818:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.751225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241184194512818:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.754829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241186364337051:2082] [node 6] ICP01 ready to work
2025-12-24T02:10:48.754904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241186364337061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22876
2025-12-24T02:10:48.754970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241186364337046:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.754720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241184641263603:2071] [node 5] ICP01 ready to work
2025-12-24T02:10:48.754764Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241184641263533:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41493108ns) {Configure:104(41491892ns) {ProcessConfigured:135(40623636ns) {StartInitialHandshake:157(40619811ns) {DropHandshakes:470(565ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1020ns) {SwitchToState:179(374ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.758792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241186364337037:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27982325ns) {Configure:104(27981488ns) {ProcessConfigured:135(16681893ns) {StartInitialHandshake:157(16681176ns) {DropHandshakes:470(1107ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(1186ns) {SwitchToState:179(387ns)}}}}}
2025-12-24T02:10:48.758971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241186364336977:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.758998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.759005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241186364336977:2049] [node 2] ICP22 created new session: [3:7587241186364337064:2049]
2025-12-24T02:10:48.759013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241186364337064:2049] [node 2] ICS09 handshake done sender: [3:7587241186364337046:2113] self: [3:7587241186364337050:0] peer: [2:7587241184602558747:0] socket: 57 qp: -1
2025-12-24T02:10:48.759021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241186364337064:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.759040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.758863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241185249469238:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27878747ns) {Configure:104(27878148ns) {ProcessConfigured:135(16749646ns) {StartInitialHandshake:157(16749202ns) {DropHandshakes:470(1143ns) {DropIncomingHandshake:442(234ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(760ns)}}}}}
2025-12-24T02:10:48.760426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337061:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.761534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241184194512838:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.759375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572949:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.759446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.761639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241184194512799:2093] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.761659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.761666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241184194512799:2093] [node 4] ICP22 created new session: [1:7587241184194512848:2093]
2025-12-24T02:10:48.761672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241184194512848:2093] [node 4] ICS09 handshake done sender: [1:7587241184194512838:2118] self: [1:7587241184194512828:0] peer: [4:7587241184641263609:0] socket: 66 qp: -1
2025-12-24T02:10:48.761678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241184194512848:2093] [node 4] ICS10 traffic start
2025-12-24T02:10:48.761685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.763862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241182521572947:2059] [node 6] ICP01 ready to work
2025-12-24T02:10:48.766743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.766846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572949:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.766865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241182521572939:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.766912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241182521572872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.763212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241184641263531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241184641263605:2114] held: no
2025-12-24T02:10:48.766756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.767269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558758:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.767286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241184602558758:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.768299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241184602558758:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.768366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241184602558720:2074] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.768376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.768383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241184602558720:2074] [node 5] ICP22 created new session: [2:7587241184602558774:2074]
2025-12-24T02:10:48.768389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241184602558774:2074] [node 5] ICS09 handshake done sender: [2:7587241184602558758:2114] self: [2:7587241184602558745:0] peer: [5:7587241182521572953:0] socket: 87 qp: -1
2025-12-24T02:10:48.768396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241184602558774:2074] [node 5] ICS10 traffic start
2025-12-24T02:10:48.768404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.770806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263613:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.770825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572951:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.770848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241182521572871:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.770861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.770868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241182521572871:2048] [node 1] ICP22 created new session: [5:7587241182521572954:2048]
2025-12-24T02:10:48.770875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241182521572954:2048] [node 1] ICS09 handshake done sender: [5:7587241182521572939:2111] self: [5:7587241182521572946:0] peer: [1:7587241184194512807:0] socket: 75 qp: -1
2025-12-24T02:10:48.770881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241182521572954:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.770899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:48.771802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241184641263532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.774756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241185249469167:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.774806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241185249469167:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31991649ns) {Configure:104(31991191ns) {ProcessConfigured:135(31675185ns) {StartInitialHandshake:157(31646211ns) {DropHandshakes:470(457ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:477(879ns) {SwitchToState:179(223ns)}} IncomingHandshake:261(21673ns) {DropIncomingHandshake:442(67ns) PrepareNewSessionHandshake:477(238ns) {SwitchToState:179(204ns)} GenerateSessionVirtualId:988(214ns) IssueIncomingHandshakeReply:193(16605ns) {SwitchToState:179(39ns)}}}}}
2025-12-24T02:10:48.774814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241184194512818:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:48.774906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241184194512797:2087] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.774916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:48.774930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241184194512797:2087] [node 6] ICP22 created new session: [1:7587241184194512851:2087]
2025-12-24T02:10:48.774936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241184194512851:2087] [node 6] ICS09 handshake done sender: [1:7587241184194512818:2109] self: [1:7587241184194512803:0] peer: [6:7587241185249469240:0] socket: 52 qp: -1
2025-12-24T02:10:48.774942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241184194512851:2087] [node 6] ICS10 traffic start
2025-12-24T02:10:48.774951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:48.778538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263612:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.778819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558751:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.778833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241184602558751:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.782823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263612:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.782901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241184641263533:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.786911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241182521572873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.786816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337058:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.786832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241186364337058:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.788804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241186364337058:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.788904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241186364337009:2075] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.788923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.788931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241186364337009:2075] [node 4] ICP22 created new session: [3:7587241186364337072:2075]
2025-12-24T02:10:48.788938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241186364337072:2075] [node 4] ICS09 handshake done sender: [3:7587241186364337058:2117] self: [3:7587241186364337053:0] peer: [4:7587241184641263616:0] socket: 99 qp: -1
2025-12-24T02:10:48.788944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241186364337072:2075] [node 4] ICS10 traffic start
2025-12-24T02:10:48.788953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:48.790557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241184602558762:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.790599Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241184602558762:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.791941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572959:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.795012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241182521572949:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.794732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337063:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.794748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241186364337063:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.795034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337049:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.795123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241182521572872:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.795138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.795145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241182521572872:2049] [node 2] ICP22 created new session: [5:7587241182521572960:2049]
2025-12-24T02:10:48.795151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241182521572960:2049] [node 2] ICS09 handshake done sender: [5:7587241182521572949:2115] self: [5:7587241182521572953:0] peer: [2:7587241184602558745:0] socket: 94 qp: -1
2025-12-24T02:10:48.795157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241182521572960:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.795164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.794829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241184602558762:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.794971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241184602558751:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:48.795000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241184602558715:2069] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.795017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.795025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241184602558715:2069] [node 4] ICP22 created new session: [2:7587241184602558778:2069]
2025-12-24T02:10:48.795032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241184602558778:2069] [node 4] ICS09 handshake done sender: [2:7587241184602558762:2117] self: [2:7587241184602558727:0] peer: [4:7587241184641263615:0] socket: 100 qp: -1
2025-12-24T02:10:48.795037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241184602558778:2069] [node 4] ICS10 traffic start
2025-12-24T02:10:48.795047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:48.795436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241184602558716:2077] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.795443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:48.795448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241184602558716:2077] [node 6] ICP22 created new session: [2:7587241184602558780:2077]
2025-12-24T02:10:48.795454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241184602558780:2077] [node 6] ICS09 handshake done sender: [2:7587241184602558751:2108] self: [2:7587241184602558726:0] peer: [6:7587241185249469247:0] socket: 60 qp: -1
2025-12-24T02:10:48.795458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241184602558780:2077] [node 6] ICS10 traffic start
2025-12-24T02:10:48.795464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:48.795348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263617:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.795372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263618:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.795479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263619:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
2025-12-24T02:10:48.795525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241184641263620:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25397
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.799417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241184602558782:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49466
2025-12-24T02:10:48.801305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241184602558720:2074] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.808465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263604:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.808978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241184641263533:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241184641263612:2117] held: no
2025-12-24T02:10:48.809018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241184641263532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15041) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241184641263613:2118] held: no
2025-12-24T02:10:48.809047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241184641263612:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:48.809115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241184641263605:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.809167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241184641263613:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.809239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241184641263533:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.809255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.809260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241184641263533:2050] [node 3] ICP22 created new session: [4:7587241184641263622:2050]
2025-12-24T02:10:48.809266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184641263622:2050] [node 3] ICS09 handshake done sender: [4:7587241184641263612:2117] self: [4:7587241184641263616:0] peer: [3:7587241186364337053:0] socket: 101 qp: -1
2025-12-24T02:10:48.809272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184641263622:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:48.809278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.809391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241184641263531:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.809398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.809402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241184641263531:2048] [node 1] ICP22 created new session: [4:7587241184641263624:2048]
2025-12-24T02:10:48.809406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184641263624:2048] [node 1] ICS09 handshake done sender: [4:7587241184641263605:2114] self: [4:7587241184641263609:0] peer: [1:7587241184194512828:0] socket: 95 qp: -1
2025-12-24T02:10:48.809409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184641263624:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.809414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:48.809475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241184641263532:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.809480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.809483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241184641263532:2049] [node 2] ICP22 created new session: [4:7587241184641263626:2049]
2025-12-24T02:10:48.809486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184641263626:2049] [node 2] ICS09 handshake done sender: [4:7587241184641263613:2118] self: [4:7587241184641263615:0] peer: [2:7587241184602558727:0] socket: 107 qp: -1
2025-12-24T02:10:48.809488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184641263626:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.809492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:10:48.806760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337057:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.807594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469248:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.807662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469249:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.808187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469250:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.808221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469251:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.810801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241185249469248:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.810833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469252:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.810858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241185249469222:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.810922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241186364337063:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.811125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241186364337037:2077] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.811141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.811150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241186364337037:2077] [node 5] ICP22 created new session: [3:7587241186364337077:2077]
2025-12-24T02:10:48.811156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241186364337077:2077] [node 5] ICS09 handshake done sender: [3:7587241186364337063:2120] self: [3:7587241186364337060:0] peer: [5:7587241182521572957:0] socket: 104 qp: -1
2025-12-24T02:10:48.811169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241186364337077:2077] [node 5] ICS10 traffic start
2025-12-24T02:10:48.811176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.810966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.811354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241185249469252:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:10:48.811387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241185249469250:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.810923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241184194512840:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.811037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241184194512797:2087] [node 6] ICP18 handshake (actor [1:7587241184194512840:2119]) is too old
2025-12-24T02:10:48.811433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241185249469166:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241185249469234:2112] held: no
2025-12-24T02:10:48.811484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241185249469238:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.811494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241185249469168:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.814904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241186364337067:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.814914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241186364337067:2121] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.815016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263617:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.814884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241185249469234:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:48.814797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572956:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.814809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241182521572956:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.814818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572963:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.815196Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572964:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.815221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241182521572936:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:48.815347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241182521572951:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:48.815408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241182521572873:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.815418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.815421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241182521572956:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:48.815423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241182521572873:2050] [node 3] ICP22 created new session: [5:7587241182521572967:2050]
2025-12-24T02:10:48.815430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241182521572967:2050] [node 3] ICS09 handshake done sender: [5:7587241182521572951:2117] self: [5:7587241182521572957:0] peer: [3:7587241186364337060:0] socket: 68 qp: -1
2025-12-24T02:10:48.815437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241182521572967:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:48.815444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.815544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241182521572873:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:48.815648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241182521572947:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.815655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:48.815662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241182521572947:2059] [node 6] ICP22 created new session: [5:7587241182521572969:2059]
2025-12-24T02:10:48.815667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241182521572969:2059] [node 6] ICS09 handshake done sender: [5:7587241182521572956:2118] self: [5:7587241182521572952:0] peer: [6:7587241185249469254:0] socket: 64 qp: -1
2025-12-24T02:10:48.815672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241182521572969:2059] [node 6] ICS10 traffic start
2025-12-24T02:10:48.815678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:48.818735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241185249469235:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:48.818918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263607:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.818931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241184641263607:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.820499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241184641263607:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:48.819412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241185249469167:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.819428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:48.819434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241185249469167:2049] [node 2] ICP22 created new session: [6:7587241185249469256:2049]
2025-12-24T02:10:48.819440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241185249469256:2049] [node 2] ICS09 handshake done sender: [6:7587241185249469235:2113] self: [6:7587241185249469247:0] peer: [2:7587241184602558726:0] socket: 89 qp: -1
2025-12-24T02:10:48.819445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241185249469256:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:48.819465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:48.819597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241185249469166:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.819604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:48.819610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241185249469166:2048] [node 1] ICP22 created new session: [6:7587241185249469258:2048]
2025-12-24T02:10:48.819613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241185249469258:2048] [node 1] ICS09 handshake done sender: [6:7587241185249469234:2112] self: [6:7587241185249469240:0] peer: [1:7587241184194512803:0] socket: 88 qp: -1
2025-12-24T02:10:48.819617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241185249469258:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:48.819621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.822852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572971:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.820567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241184641263561:2066] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.820578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:48.820583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241184641263561:2066] [node 6] ICP22 created new session: [4:7587241184641263631:2066]
2025-12-24T02:10:48.820588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184641263631:2066] [node 6] ICS09 handshake done sender: [4:7587241184641263607:2115] self: [4:7587241184641263606:0] peer: [6:7587241185249469253:0] socket: 62 qp: -1
2025-12-24T02:10:48.820592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184641263631:2066] [node 6] ICS10 traffic start
2025-12-24T02:10:48.820598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:48.823030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241186364337067:2121] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:48.823148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241186364337051:2082] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.823168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:48.823173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241186364337051:2082] [node 6] ICP22 created new session: [3:7587241186364337080:2082]
2025-12-24T02:10:48.823178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241186364337080:2082] [node 6] ICS09 handshake done sender: [3:7587241186364337067:2121] self: [3:7587241186364337066:0] peer: [6:7587241185249469255:0] socket: 58 qp: -1
2025-12-24T02:10:48.823183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241186364337080:2082] [node 6] ICS10 traffic start
2025-12-24T02:10:48.823197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:48.828138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241185249469222:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241185249469248:2117] held: no
2025-12-24T02:10:48.828187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241185249469168:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241185249469250:2119] held: no
2025-12-24T02:10:48.830914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241184641263611:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.830931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241184641263611:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:48.831541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241184641263611:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.831606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241184641263603:2071] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:48.831622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.831630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241184641263603:2071] [node 5] ICP22 created new session: [4:7587241184641263634:2071]
2025-12-24T02:10:48.831636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241184641263634:2071] [node 5] ICS09 handshake done sender: [4:7587241184641263611:2116] self: [4:7587241184641263610:0] peer: [5:7587241182521572966:0] socket: 96 qp: -1
2025-12-24T02:10:48.831642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241184641263634:2071] [node 5] ICS10 traffic start
2025-12-24T02:10:48.831650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.834882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241182521572936:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241182521572964:2121] held: no
2025-12-24T02:10:48.834948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241182521572972:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28776
2025-12-24T02:10:48.835085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241182521572971:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:48.835118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241182521572947:2059] [node 6] ICP18 handshake (actor [5:7587241182521572971:2122]) is too old
2025-12-24T02:10:48.838758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241182521572964:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.838735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469261:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.838977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241185249469252:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:48.839087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241185249469238:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.839103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:48.839111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241185249469238:2070] [node 5] ICP22 created new session: [6:7587241185249469262:2070]
2025-12-24T02:10:48.839118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241185249469262:2070] [node 5] ICS09 handshake done sender: [6:7587241185249469252:2121] self: [6:7587241185249469254:0] peer: [5:7587241182521572952:0] socket: 119 qp: -1
2025-12-24T02:10:48.839125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241185249469262:2070] [node 5] ICS10 traffic start
2025-12-24T02:10:48.839136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:48.842868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469264:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:48.847105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241182521572936:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.847129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.847137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241182521572936:2063] [node 4] ICP22 created new session: [5:7587241182521572973:2063]
2025-12-24T02:10:48.847143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241182521572973:2063] [node 4] ICS09 handshake done sender: [5:7587241182521572964:2121] self: [5:7587241182521572966:0] peer: [4:7587241184641263610:0] socket: 121 qp: -1
2025-12-24T02:10:48.847149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241182521572973:2063] [node 4] ICS10 traffic start
2025-12-24T02:10:48.847157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:48.851049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241185249469248:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:48.854824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241185249469222:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.854846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241185249469265:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62560
2025-12-24T02:10:48.858852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:48.858866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241185249469222:2061] [node 4] ICP22 created new session: [6:7587241185249469266:2061]
2025-12-24T02:10:48.858875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241185249469266:2061] [node 4] ICS09 handshake done sender: [6:7587241185249469248:2117] self: [6:7587241185249469253:0] peer: [4:7587241184641263606:0] socket: 110 qp: -1
2025-12-24T02:10:48.858884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241185249469266:2061] [node 4] ICS10 traffic start
2025-12-24T02:10:48.858899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:48.862935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241185249469250:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:48.870784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241185249469168:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:48.870810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:48.870821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241185249469168:2050] [node 3] ICP22 created new session: [6:7587241185249469268:2050]
2025-12-24T02:10:48.870828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241185249469268:2050] [node 3] ICS09 handshake done sender: [6:7587241185249469250:2119] self: [6:7587241185249469255:0] peer: [3:7587241186364337066:0] socket: 80 qp: -1
2025-12-24T02:10:48.870838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241185249469268:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:48.870850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:48.883151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:48.883249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241184602558765:2048] [node 1] ICS01 socket: 67 reason# EndOfStream
2025-12-24T02:10:48.883260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241184602558680:2048] [node 1] ICP30 unregister session Session# [2:7587241184602558765:2048] VirtualId# [2:7587241184602558763:0]
2025-12-24T02:10:48.883267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241184602558765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.883309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.883331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.899101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:48.899207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241186364337055:2048] [node 1] ICS01 socket: 55 reason# EndOfStream
2025-12-24T02:10:48.899220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241186364336976:2048] [node 1] ICP30 unregister session Session# [3:7587241186364337055:2048] VirtualId# [3:7587241186364337052:0]
2025-12-24T02:10:48.899229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241186364337055:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.899266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.899289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.918771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:48.918872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241184641263624:2048] [node 1] ICS01 socket: 95 reason# EndOfStream
2025-12-24T02:10:48.918771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:48.918872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241185249469258:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-12-24T02:10:48.918883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241185249469166:2048] [node 1] ICP30 unregister session Session# [6:7587241185249469258:2048] VirtualId# [6:7587241185249469240:0]
2025-12-24T02:10:48.918892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241185249469258:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.918952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.918979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.918882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241184641263531:2048] [node 1] ICP30 unregister session Session# [4:7587241184641263624:2048] VirtualId# [4:7587241184641263609:0]
2025-12-24T02:10:48.918891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241184641263624:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.918943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.918967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.922763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:48.922884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241182521572954:2048] [node 1] ICS01 socket: 75 reason# EndOfStream
2025-12-24T02:10:48.922894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241182521572871:2048] [node 1] ICP30 unregister session Session# [5:7587241182521572954:2048] VirtualId# [5:7587241182521572946:0]
2025-12-24T02:10:48.922904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241182521572954:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.922967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:48.923006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:48.927249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:48.927332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241186364337064:2049] [node 2] ICS01 socket: 57 reason# EndOfStream
2025-12-24T02:10:48.927339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241186364336977:2049] [node 2] ICP30 unregister session Session# [3:7587241186364337064:2049] VirtualId# [3:7587241186364337050:0]
2025-12-24T02:10:48.927346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241186364337064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.927384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.927404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.930511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:48.930579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241182521572960:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-12-24T02:10:48.930587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241182521572872:2049] [node 2] ICP30 unregister session Session# [5:7587241182521572960:2049] VirtualId# [5:7587241182521572953:0]
2025-12-24T02:10:48.930592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241182521572960:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.930641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.930667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.930740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:48.930810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241184641263626:2049] [node 2] ICS01 socket: 107 reason# EndOfStream
2025-12-24T02:10:48.930816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241184641263532:2049] [node 2] ICP30 unregister session Session# [4:7587241184641263626:2049] VirtualId# [4:7587241184641263615:0]
2025-12-24T02:10:48.930821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241184641263626:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.930851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.930872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.930327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:48.930407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241185249469256:2049] [node 2] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:10:48.930414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241185249469167:2049] [node 2] ICP30 unregister session Session# [6:7587241185249469256:2049] VirtualId# [6:7587241185249469247:0]
2025-12-24T02:10:48.930421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241185249469256:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.930462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:48.930488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:48.958225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:48.958337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241185249469268:2050] [node 3] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:10:48.958348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241185249469168:2050] [node 3] ICP30 unregister session Session# [6:7587241185249469268:2050] VirtualId# [6:7587241185249469255:0]
2025-12-24T02:10:48.958357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241185249469268:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.958418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:48.958445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:48.958940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:48.959019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241182521572967:2050] [node 3] ICS01 socket: 68 reason# EndOfStream
2025-12-24T02:10:48.959027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241182521572873:2050] [node 3] ICP30 unregister session Session# [5:7587241182521572967:2050] VirtualId# [5:7587241182521572957:0]
2025-12-24T02:10:48.959032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241182521572967:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.959079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:48.959101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:48.990793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:48.990913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241185249469266:2061] [node 4] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:10:48.990925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241185249469222:2061] [node 4] ICP30 unregister session Session# [6:7587241185249469266:2061] VirtualId# [6:7587241185249469253:0]
2025-12-24T02:10:48.990933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241185249469266:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:48.991031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:48.991061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:49.082723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241189488489724:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:49.058823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241187793495493:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.082749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190363919497:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:49.090742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190363919496:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.058834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241187793495494:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.090713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241189488489723:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:49.098710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241188723995330:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.098763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241188723995331:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.111065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241188723995332:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.118818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241188336974276:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.118834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241188336974277:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.118839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241188336974278:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.154750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241188476445339:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.162711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241188476445340:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.162765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241188476445341:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.169008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190363919557:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:49.169628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241189488489786:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.170593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241187793495557:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:49.170142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995391:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.170091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190363919562:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:49.171474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241189488489792:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.174880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495559:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.175009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241188476445357:2063] [node 5] ICP01 ready to work
2025-12-24T02:10:49.174976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241189488489796:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.179103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495560:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.181266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974331:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:10:49.181289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974332:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.191390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241188476445404:2073] [node 4] ICP01 ready to work
2025-12-24T02:10:49.192256Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241188476445339:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21434217ns) {Configure:104(21433329ns) {ProcessConfigured:135(21090854ns) {StartInitialHandshake:157(21090116ns) {DropHandshakes:470(321ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(1124ns) {SwitchToState:179(573ns)}}}}}
2025-12-24T02:10:49.192389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241190363919496:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21382937ns) {Configure:104(21382214ns) {ProcessConfigured:135(21104185ns) {StartInitialHandshake:157(21103843ns) {DropHandshakes:470(370ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(104ns)}}}}}
2025-12-24T02:10:49.191213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241189488489800:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5980
2025-12-24T02:10:49.195345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241189488489792:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23780610ns) {Configure:104(23778023ns) {ProcessConfigured:135(23441640ns) {StartInitialHandshake:157(23440831ns) {DropHandshakes:470(935ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1568ns) {SwitchToState:179(868ns)}}}}}
2025-12-24T02:10:49.198513Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241189488489723:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26591578ns) {Configure:104(26590208ns) {ProcessConfigured:135(26415007ns) {StartInitialHandshake:157(26414349ns) {DropHandshakes:470(160ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(32ns) PrepareNewSessionHandshake:477(699ns) {SwitchToState:179(359ns)}}}}}
2025-12-24T02:10:49.202978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489800:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.203749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974331:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.204238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241188336974277:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.203545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241188476445340:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28472385ns) {Configure:104(28471895ns) {ProcessConfigured:135(28187092ns) {StartInitialHandshake:157(28186501ns) {DropHandshakes:470(322ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(853ns) {SwitchToState:179(420ns)}}}}}
2025-12-24T02:10:49.204516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919559:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.204531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190363919559:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.206208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495559:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.206251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241187793495493:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.206399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489785:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.206411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241189488489785:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.207026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241189488489785:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.207056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241189488489805:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5980
2025-12-24T02:10:49.207744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190363919572:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22315
2025-12-24T02:10:49.208017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190363919559:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.208159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190363919557:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.208173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.208179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190363919557:2107] [node 5] ICP22 created new session: [2:7587241190363919574:2107]
2025-12-24T02:10:49.208185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190363919574:2107] [node 5] ICS09 handshake done sender: [2:7587241190363919559:2108] self: [2:7587241190363919558:0] peer: [5:7587241188336974343:0] socket: 60 qp: -1
2025-12-24T02:10:49.208191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190363919574:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:49.208201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.208296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489805:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.211550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241189488489724:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.211578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.211589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241189488489724:2049] [node 3] ICP22 created new session: [1:7587241189488489807:2049]
2025-12-24T02:10:49.211597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241189488489807:2049] [node 3] ICS09 handshake done sender: [1:7587241189488489785:2108] self: [1:7587241189488489784:0] peer: [3:7587241187793495567:0] socket: 61 qp: -1
2025-12-24T02:10:49.211605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241189488489807:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:49.211615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.211028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241188723995398:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:49.211520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241188723995400:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:49.214772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919572:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.214823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241187793495557:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44168303ns) {Configure:104(44167729ns) {ProcessConfigured:135(43911627ns) {StartInitialHandshake:157(43911151ns) {DropHandshakes:470(412ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(1175ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.218900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445417:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.218982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445418:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.219113Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241190363919497:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18413732ns) {Configure:104(18412756ns) {ProcessConfigured:135(17650738ns) {StartInitialHandshake:157(17650146ns) {DropHandshakes:470(913ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(271ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1028ns) {SwitchToState:179(277ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.224767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241188476445418:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.224831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241188476445339:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.225086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241188476445417:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.225117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241188476445340:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.230773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241187793495493:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23575) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241187793495559:2109] held: no
2025-12-24T02:10:49.230901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495560:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.223533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974345:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:10:49.223992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241188336974326:2096] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.232435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241188336974276:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24411283ns) {Configure:104(24410761ns) {ProcessConfigured:135(24119574ns) {StartInitialHandshake:157(24118828ns) {DropHandshakes:470(505ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(1117ns) {SwitchToState:179(682ns)}}}}}
2025-12-24T02:10:49.232826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241188336974331:2101] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.232167Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241190363919557:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(23983863ns) {HandleHandshakeStatus:329(23982970ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(339ns) SwitchToState:179(217ns) ProcessPendingSessionEvents:495(63023ns) {ForwardSessionEventToSession:612(56178ns) ForwardSessionEventToSession:612(1006ns) ForwardSessionEventToSession:612(1965ns)}}}
2025-12-24T02:10:49.232888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241188723995330:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30095292ns) {Configure:104(30094386ns) {ProcessConfigured:135(24251725ns) {StartInitialHandshake:157(24251637ns) {DropHandshakes:470(748ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(827ns) {SwitchToState:179(260ns)}}}}}
2025-12-24T02:10:49.232959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995391:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.233053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995394:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.233120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995403:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.233159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241188723995330:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.239195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241188336974340:2099] [node 6] ICP01 ready to work
2025-12-24T02:10:49.239290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241188336974277:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.239305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.239312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241188336974277:2049] [node 2] ICP22 created new session: [5:7587241188336974353:2049]
2025-12-24T02:10:49.239318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241188336974353:2049] [node 2] ICS09 handshake done sender: [5:7587241188336974331:2101] self: [5:7587241188336974343:0] peer: [2:7587241190363919558:0] socket: 70 qp: -1
2025-12-24T02:10:49.239326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241188336974353:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.239361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.238836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241188723995398:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27673897ns) {Configure:104(27673121ns) {ProcessConfigured:135(27394956ns) {StartInitialHandshake:157(27394073ns) {DropHandshakes:470(485ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(756ns) {SwitchToState:179(205ns)}}}}}
2025-12-24T02:10:49.242813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489798:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.242835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241189488489798:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.242711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190363919570:2112] [node 4] ICP01 ready to work
2025-12-24T02:10:49.242713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919564:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.242723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190363919564:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.243018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974346:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:10:49.243362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974332:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.243527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241189488489724:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(31954122ns) {HandleHandshakeStatus:329(31953504ns) {DropIncomingHandshake:442(8150ns) DropOutgoingHandshake:457(166ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(19126340ns) {ForwardSessionEventToSession:612(19121980ns) ForwardSessionEventToSession:612(2258ns)}}}
2025-12-24T02:10:49.246091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489789:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.246103Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241189488489789:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.243868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995394:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.244151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241188723995400:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32561856ns) {Configure:104(32561258ns) {ProcessConfigured:135(15467810ns) {StartInitialHandshake:157(15465290ns) {DropHandshakes:470(927ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(1321ns) {SwitchToState:179(696ns)}}}}}
2025-12-24T02:10:49.245865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241188723995331:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42982557ns) {Configure:104(42980280ns) {ProcessConfigured:135(36998301ns) {StartInitialHandshake:157(36997412ns) {DropHandshakes:470(239ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(711ns) {SwitchToState:179(90ns)}}}}}
2025-12-24T02:10:49.246133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241187793495563:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:49.246794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995403:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.246940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241188723995332:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43118897ns) {Configure:104(43117243ns) {ProcessConfigured:135(38030106ns) {StartInitialHandshake:157(38028948ns) {DropHandshakes:470(416ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(857ns) {SwitchToState:179(467ns)}}}}}
2025-12-24T02:10:49.248317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241188723995332:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.252157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974346:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.252197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241188336974276:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.252201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495572:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.251885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190363919564:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.254354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241187793495568:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:49.251986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190363919562:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.251998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.252006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190363919562:2109] [node 6] ICP22 created new session: [2:7587241190363919581:2109]
2025-12-24T02:10:49.252013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190363919581:2109] [node 6] ICS09 handshake done sender: [2:7587241190363919564:2110] self: [2:7587241190363919563:0] peer: [6:7587241188476445420:0] socket: 66 qp: -1
2025-12-24T02:10:49.252021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190363919581:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:49.252048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.254766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241189488489810:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5980
2025-12-24T02:10:49.254802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241189488489798:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.255715Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241188336974277:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16394945ns) {HandleHandshakeStatus:329(16394227ns) {DropIncomingHandshake:442(290ns) DropOutgoingHandshake:457(55ns) SwitchToState:179(226ns) ProcessPendingSessionEvents:495(9473016ns) {ForwardSessionEventToSession:612(9461543ns) ForwardSessionEventToSession:612(1606ns) ForwardSessionEventToSession:612(1092ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(621ns) ForwardSessionEventToSession:612(1216ns) ForwardSessionEventToSession:612(691ns) ForwardSessionEventToSession:612(851ns) ForwardSessionEventToSession:612(597ns)}}}
2025-12-24T02:10:49.254847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495569:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.254853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241187793495569:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.256286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495574:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.256435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241189488489789:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.256530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241189488489786:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.256544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.256553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241189488489786:2107] [node 4] ICP22 created new session: [1:7587241189488489814:2107]
2025-12-24T02:10:49.256559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241189488489814:2107] [node 4] ICS09 handshake done sender: [1:7587241189488489789:2109] self: [1:7587241189488489788:0] peer: [4:7587241188723995405:0] socket: 63 qp: -1
2025-12-24T02:10:49.256566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241189488489814:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:49.256574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.258846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241187793495559:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.259084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995414:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.259126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995415:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.258910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241189488489816:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5980
2025-12-24T02:10:49.259413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190363919585:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22315
2025-12-24T02:10:49.262752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495573:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.262839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495574:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.264332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445421:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.264384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445422:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.264564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445423:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.264729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919585:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.264764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919583:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.264767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190363919583:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.265195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241190363919496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.263867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241187793495493:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.263883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.263892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241187793495493:2048] [node 1] ICP22 created new session: [3:7587241187793495581:2048]
2025-12-24T02:10:49.263898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241187793495581:2048] [node 1] ICS09 handshake done sender: [3:7587241187793495559:2109] self: [3:7587241187793495567:0] peer: [1:7587241189488489784:0] socket: 67 qp: -1
2025-12-24T02:10:49.263906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241187793495581:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.263915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.265406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190363919583:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.265479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190363919570:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.265492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.265497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190363919570:2112] [node 4] ICP22 created new session: [2:7587241190363919588:2112]
2025-12-24T02:10:49.265502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190363919588:2112] [node 4] ICS09 handshake done sender: [2:7587241190363919583:2115] self: [2:7587241190363919579:0] peer: [4:7587241188723995417:0] socket: 111 qp: -1
2025-12-24T02:10:49.265508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190363919588:2112] [node 4] ICS10 traffic start
2025-12-24T02:10:49.265513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.262785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489801:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.262799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241189488489801:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.262873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489816:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.262901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489810:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.263013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995414:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.263045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995415:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.263080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241188723995331:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.264478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241188336974340:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18044527ns) {Configure:104(18042388ns) {ProcessConfigured:135(9807393ns) {StartInitialHandshake:157(9806356ns) {DropHandshakes:470(461ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(655ns) {SwitchToState:179(31ns)}}}}}
2025-12-24T02:10:49.266849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241188476445417:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.266954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241188476445421:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.267005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241188476445404:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.267127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241188476445340:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.267141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.267156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241188476445340:2049] [node 2] ICP22 created new session: [6:7587241188476445425:2049]
2025-12-24T02:10:49.267160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241188476445425:2049] [node 2] ICS09 handshake done sender: [6:7587241188476445417:2115] self: [6:7587241188476445420:0] peer: [2:7587241190363919563:0] socket: 81 qp: -1
2025-12-24T02:10:49.267166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241188476445425:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.267180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.270800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495573:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.270863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241187793495569:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.270864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241187793495494:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.270807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241190363919562:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18797347ns) {HandleHandshakeStatus:329(18796637ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(18686874ns) {ForwardSessionEventToSession:612(18680277ns) ForwardSessionEventToSession:612(1620ns) ForwardSessionEventToSession:612(1832ns) ForwardSessionEventToSession:612(748ns)}}}
2025-12-24T02:10:49.271010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241187793495557:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.271024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.270906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241189488489796:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.270922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.274723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241189488489802:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.274737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241189488489802:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.275465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241189488489802:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.275534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241189488489723:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.275549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.275564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241189488489723:2048] [node 2] ICP22 created new session: [1:7587241189488489820:2048]
2025-12-24T02:10:49.275571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241189488489820:2048] [node 2] ICS09 handshake done sender: [1:7587241189488489802:2115] self: [1:7587241189488489794:0] peer: [2:7587241190363919587:0] socket: 112 qp: -1
2025-12-24T02:10:49.275577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241189488489820:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:49.275585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.278905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.280934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190363919576:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.280950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190363919576:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.281046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974362:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.282724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241190363919496:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23575) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241190363919585:2116] held: no
2025-12-24T02:10:49.282723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241189488489796:2111] [node 6] ICP22 created new session: [1:7587241189488489818:2111]
2025-12-24T02:10:49.282739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241189488489818:2111] [node 6] ICS09 handshake done sender: [1:7587241189488489798:2112] self: [1:7587241189488489797:0] peer: [6:7587241188476445419:0] socket: 72 qp: -1
2025-12-24T02:10:49.282747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241189488489818:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:49.282758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.282801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241189488489786:2107] [node 4] ICP18 handshake (actor [1:7587241189488489810:2117]) is too old
2025-12-24T02:10:49.282828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7587241189488489786:2107] [node 4] ICP35 event processing took too much time StateWork:294(19839703ns) {IncomingHandshake:261(19838893ns)}
2025-12-24T02:10:49.282718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241187793495557:2108] [node 4] ICP22 created new session: [3:7587241187793495585:2108]
2025-12-24T02:10:49.282745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241187793495585:2108] [node 4] ICS09 handshake done sender: [3:7587241187793495569:2114] self: [3:7587241187793495558:0] peer: [4:7587241188723995413:0] socket: 90 qp: -1
2025-12-24T02:10:49.282754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241187793495585:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:49.282772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.282949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974362:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.282887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241189488489801:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.283105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974361:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.283162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241188336974278:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.283176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241188336974326:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.282844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995408:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.282864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241188723995408:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.283327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241188723995406:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.283330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241188723995406:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.284494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241188723995408:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.284602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241188723995406:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.284657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241188723995400:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.284670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.284675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241188723995400:2110] [node 6] ICP22 created new session: [4:7587241188723995420:2110]
2025-12-24T02:10:49.284680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241188723995420:2110] [node 6] ICS09 handshake done sender: [4:7587241188723995408:2115] self: [4:7587241188723995401:0] peer: [6:7587241188476445424:0] socket: 100 qp: -1
2025-12-24T02:10:49.284685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241188723995420:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:49.284692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.285778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241188723995398:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.283750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241189488489792:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.285812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.283771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.283777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241189488489792:2110] [node 5] ICP22 created new session: [1:7587241189488489824:2110]
2025-12-24T02:10:49.283783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241189488489824:2110] [node 5] ICS09 handshake done sender: [1:7587241189488489801:2114] self: [1:7587241189488489793:0] peer: [5:7587241188336974357:0] socket: 89 qp: -1
2025-12-24T02:10:49.283788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241189488489824:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:49.283795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.286796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241188476445418:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.287996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241188476445340:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20814717ns) {HandleHandshakeStatus:329(20814254ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(3422ns) SwitchToState:179(44ns) ProcessPendingSessionEvents:495(17004291ns) {ForwardSessionEventToSession:612(16990864ns) ForwardSessionEventToSession:612(1837ns) ForwardSessionEventToSession:612(1042ns) ForwardSessionEventToSession:612(713ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(1199ns) ForwardSessionEventToSession:612(999ns) ForwardSessionEventToSession:612(708ns) ForwardSessionEventToSession:612(598ns) ForwardSessionEventToSession:612(1318ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.285827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241188723995398:2109] [node 5] ICP22 created new session: [4:7587241188723995423:2109]
2025-12-24T02:10:49.288406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495575:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.288421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241187793495575:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.285835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241188723995423:2109] [node 5] ICS09 handshake done sender: [4:7587241188723995406:2114] self: [4:7587241188723995399:0] peer: [5:7587241188336974364:0] socket: 96 qp: -1
2025-12-24T02:10:49.285847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241188723995423:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:49.285873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.290898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241187793495575:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.294739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190363919590:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22315
2025-12-24T02:10:49.295012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190363919585:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.295135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241190363919496:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.295152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.295161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190363919496:2048] [node 1] ICP22 created new session: [2:7587241190363919591:2048]
2025-12-24T02:10:49.295167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190363919591:2048] [node 1] ICS09 handshake done sender: [2:7587241190363919585:2116] self: [2:7587241190363919587:0] peer: [1:7587241189488489794:0] socket: 114 qp: -1
2025-12-24T02:10:49.295173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190363919591:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.295181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.295343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241188336974276:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23575) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241188336974346:2113] held: no
2025-12-24T02:10:49.295377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241188336974326:2096] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241188336974361:2118] held: no
2025-12-24T02:10:49.298891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241187793495563:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.298937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.298955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241187793495563:2111] [node 5] ICP22 created new session: [3:7587241187793495589:2111]
2025-12-24T02:10:49.298968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241187793495589:2111] [node 5] ICS09 handshake done sender: [3:7587241187793495575:2118] self: [3:7587241187793495571:0] peer: [5:7587241188336974363:0] socket: 116 qp: -1
2025-12-24T02:10:49.298980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241187793495589:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:49.299006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.302796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190363919576:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.302858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241187793495557:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(31808654ns) {HandleHandshakeStatus:329(31807502ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(142ns) SwitchToState:179(226ns) ProcessPendingSessionEvents:495(19992775ns) {ForwardSessionEventToSession:612(19977356ns) ForwardSessionEventToSession:612(1720ns) ForwardSessionEventToSession:612(1381ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(931ns) ForwardSessionEventToSession:612(1514ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(966ns)}}}
2025-12-24T02:10:49.303072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:10:49.307029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241189488489792:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(23233445ns) {HandleHandshakeStatus:329(23232734ns) {DropIncomingHandshake:442(3942ns) DropOutgoingHandshake:457(87ns) SwitchToState:179(230ns) ProcessPendingSessionEvents:495(23144453ns) {ForwardSessionEventToSession:612(23133129ns) ForwardSessionEventToSession:612(1102ns) ForwardSessionEventToSession:612(1326ns) ForwardSessionEventToSession:612(694ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(1032ns)}}}
2025-12-24T02:10:49.306873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241188336974346:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.306986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241188336974276:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.307009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.307017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241188336974276:2048] [node 1] ICP22 created new session: [5:7587241188336974367:2048]
2025-12-24T02:10:49.307023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241188336974367:2048] [node 1] ICS09 handshake done sender: [5:7587241188336974346:2113] self: [5:7587241188336974357:0] peer: [1:7587241189488489793:0] socket: 93 qp: -1
2025-12-24T02:10:49.307029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241188336974367:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.307055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.310843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445428:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.314765Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241188336974278:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241188336974362:2119] held: no
2025-12-24T02:10:49.314758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190363919497:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.314790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.314799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190363919497:2049] [node 3] ICP22 created new session: [2:7587241190363919594:2049]
2025-12-24T02:10:49.314806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190363919594:2049] [node 3] ICS09 handshake done sender: [2:7587241190363919576:2114] self: [2:7587241190363919571:0] peer: [3:7587241187793495584:0] socket: 92 qp: -1
2025-12-24T02:10:49.314813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190363919594:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:49.314823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.318866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241188476445404:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4275) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241188476445421:2117] held: no
2025-12-24T02:10:49.318955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241188476445428:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.319015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241188476445341:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.320225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241189488489796:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(65246388ns) {HandleHandshakeStatus:329(65245781ns) {DropIncomingHandshake:442(413ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(29443975ns) {ForwardSessionEventToSession:612(29430093ns) ForwardSessionEventToSession:612(1753ns) ForwardSessionEventToSession:612(1658ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(1272ns) ForwardSessionEventToSession:612(725ns) ForwardSessionEventToSession:612(1038ns)}}}
2025-12-24T02:10:49.322833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241187793495577:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.322852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241187793495577:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.326848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241188723995330:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23575) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241188723995391:2108] held: no
2025-12-24T02:10:49.326940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995427:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.334929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241187793495494:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11095) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241187793495573:2116] held: no
2025-12-24T02:10:49.335290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241188476445339:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.335326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.335340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241188476445339:2048] [node 1] ICP22 created new session: [6:7587241188476445430:2048]
2025-12-24T02:10:49.335350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241188476445430:2048] [node 1] ICS09 handshake done sender: [6:7587241188476445418:2116] self: [6:7587241188476445419:0] peer: [1:7587241189488489797:0] socket: 91 qp: -1
2025-12-24T02:10:49.335361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241188476445430:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.335390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.338949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495591:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.339146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241187793495563:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(40227276ns) {HandleHandshakeStatus:329(40226054ns) {DropIncomingHandshake:442(15797ns) DropOutgoingHandshake:457(306ns) SwitchToState:179(177ns) ProcessPendingSessionEvents:495(24252800ns) {ForwardSessionEventToSession:612(24239897ns) ForwardSessionEventToSession:612(1100ns) ForwardSessionEventToSession:612(1389ns) ForwardSessionEventToSession:612(895ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(1117ns) ForwardSessionEventToSession:612(809ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(713ns) ForwardSessionEventToSession:612(925ns)}}}
2025-12-24T02:10:49.339295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241190363919497:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24522307ns) {HandleHandshakeStatus:329(24521121ns) {DropIncomingHandshake:442(7773ns) DropOutgoingHandshake:457(174ns) SwitchToState:179(246ns) ProcessPendingSessionEvents:495(24402810ns) {ForwardSessionEventToSession:612(24396833ns) ForwardSessionEventToSession:612(1780ns) ForwardSessionEventToSession:612(757ns) ForwardSessionEventToSession:612(921ns) ForwardSessionEventToSession:612(851ns)}}}
2025-12-24T02:10:49.343055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241188723995415:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.342892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241187793495573:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.343377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241187793495577:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.343487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241187793495494:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.343505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.343516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241187793495494:2049] [node 2] ICP22 created new session: [3:7587241187793495593:2049]
2025-12-24T02:10:49.343524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241187793495593:2049] [node 2] ICS09 handshake done sender: [3:7587241187793495573:2116] self: [3:7587241187793495584:0] peer: [2:7587241190363919571:0] socket: 109 qp: -1
2025-12-24T02:10:49.343532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241187793495593:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.343543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.343691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241187793495568:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.343706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.343711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241187793495568:2112] [node 6] ICP22 created new session: [3:7587241187793495595:2112]
2025-12-24T02:10:49.343715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241187793495595:2112] [node 6] ICS09 handshake done sender: [3:7587241187793495577:2119] self: [3:7587241187793495576:0] peer: [6:7587241188476445429:0] socket: 121 qp: -1
2025-12-24T02:10:49.343720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241187793495595:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:49.343727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.351084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995426:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.351360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241188723995331:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.351374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.351385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241188723995331:2049] [node 2] ICP22 created new session: [4:7587241188723995428:2049]
2025-12-24T02:10:49.351392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241188723995428:2049] [node 2] ICS09 handshake done sender: [4:7587241188723995415:2119] self: [4:7587241188723995417:0] peer: [2:7587241190363919579:0] socket: 65 qp: -1
2025-12-24T02:10:49.351401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241188723995428:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.351410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.351613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241188723995391:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.351685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241188723995330:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.351692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.351697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241188723995330:2048] [node 1] ICP22 created new session: [4:7587241188723995430:2048]
2025-12-24T02:10:49.351702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241188723995430:2048] [node 1] ICS09 handshake done sender: [4:7587241188723995391:2108] self: [4:7587241188723995405:0] peer: [1:7587241189488489788:0] socket: 64 qp: -1
2025-12-24T02:10:49.351707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241188723995430:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.351713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.359412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241188476445341:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241188476445428:2120] held: no
2025-12-24T02:10:49.362937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241188476445339:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(73441662ns) {HandleHandshakeStatus:329(73440563ns) {DropIncomingHandshake:442(926ns) DropOutgoingHandshake:457(2448ns) SwitchToState:179(232ns) ProcessPendingSessionEvents:495(16114672ns) {ForwardSessionEventToSession:612(16099407ns) ForwardSessionEventToSession:612(1866ns) ForwardSessionEventToSession:612(1212ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(763ns) ForwardSessionEventToSession:612(2006ns) ForwardSessionEventToSession:612(772ns) ForwardSessionEventToSession:612(762ns) ForwardSessionEventToSession:612(1232ns) ForwardSessionEventToSession:612(1238ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.374912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974369:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:10:49.378763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241188336974370:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.378983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241188336974362:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.379102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241188336974278:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.379121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.379131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241188336974278:2050] [node 3] ICP22 created new session: [5:7587241188336974371:2050]
2025-12-24T02:10:49.379138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241188336974371:2050] [node 3] ICS09 handshake done sender: [5:7587241188336974362:2119] self: [5:7587241188336974363:0] peer: [3:7587241187793495571:0] socket: 123 qp: -1
2025-12-24T02:10:49.379147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241188336974371:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.379157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:10:49.379033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445432:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.382756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241188336974361:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.382778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445434:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.382862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241188336974326:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.382882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.382893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241188336974326:2096] [node 4] ICP22 created new session: [5:7587241188336974373:2096]
2025-12-24T02:10:49.382901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241188336974373:2096] [node 4] ICS09 handshake done sender: [5:7587241188336974361:2118] self: [5:7587241188336974364:0] peer: [4:7587241188723995399:0] socket: 99 qp: -1
2025-12-24T02:10:49.382910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241188336974373:2096] [node 4] ICS10 traffic start
2025-12-24T02:10:49.382923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.382919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241188476445421:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.383120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241188476445428:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.383471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241188476445341:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.383484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.383490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241188476445341:2050] [node 3] ICP22 created new session: [6:7587241188476445435:2050]
2025-12-24T02:10:49.383497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241188476445435:2050] [node 3] ICS09 handshake done sender: [6:7587241188476445428:2120] self: [6:7587241188476445429:0] peer: [3:7587241187793495576:0] socket: 125 qp: -1
2025-12-24T02:10:49.383502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241188476445435:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.383509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.383711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241188476445404:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.383732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.383737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241188476445404:2073] [node 4] ICP22 created new session: [6:7587241188476445437:2073]
2025-12-24T02:10:49.383742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241188476445437:2073] [node 4] ICS09 handshake done sender: [6:7587241188476445421:2117] self: [6:7587241188476445424:0] peer: [4:7587241188723995401:0] socket: 101 qp: -1
2025-12-24T02:10:49.383747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241188476445437:2073] [node 4] ICS10 traffic start
2025-12-24T02:10:49.383767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.385601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241187793495597:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63045
2025-12-24T02:10:49.383243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241188723995432:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45840
2025-12-24T02:10:49.383262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190363919596:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22315
2025-12-24T02:10:49.386868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445433:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.386964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241188723995403:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.385773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241189488489826:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5980
2025-12-24T02:10:49.391465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241188476445433:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.391521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241188476445357:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.390716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241188723995332:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.399108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.399131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241188723995332:2050] [node 3] ICP22 created new session: [4:7587241188723995433:2050]
2025-12-24T02:10:49.399142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241188723995433:2050] [node 3] ICS09 handshake done sender: [4:7587241188723995403:2113] self: [4:7587241188723995413:0] peer: [3:7587241187793495558:0] socket: 71 qp: -1
2025-12-24T02:10:49.399152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241188723995433:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.399167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.402736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241188336974359:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.402753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241188336974359:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.403588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241188336974359:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.403672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241188336974340:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.403703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.403711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241188336974340:2099] [node 6] ICP22 created new session: [5:7587241188336974376:2099]
2025-12-24T02:10:49.403717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241188336974376:2099] [node 6] ICS09 handshake done sender: [5:7587241188336974359:2117] self: [5:7587241188336974358:0] peer: [6:7587241188476445439:0] socket: 95 qp: -1
2025-12-24T02:10:49.403722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241188336974376:2099] [node 6] ICS10 traffic start
2025-12-24T02:10:49.403733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.403908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241188476445440:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3332
2025-12-24T02:10:49.404145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241188476445433:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.404475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241188476445357:2063] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.404496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.404502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241188476445357:2063] [node 5] ICP22 created new session: [6:7587241188476445441:2063]
2025-12-24T02:10:49.404508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241188476445441:2063] [node 5] ICS09 handshake done sender: [6:7587241188476445433:2122] self: [6:7587241188476445439:0] peer: [5:7587241188336974358:0] socket: 77 qp: -1
2025-12-24T02:10:49.404514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241188476445441:2063] [node 5] ICS10 traffic start
2025-12-24T02:10:49.404527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.407306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241188723995332:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16555062ns) {HandleHandshakeStatus:329(16554891ns) {DropIncomingHandshake:442(154ns) DropOutgoingHandshake:457(2681ns) SwitchToState:179(101ns) ProcessPendingSessionEvents:495(8063747ns) {ForwardSessionEventToSession:612(8052465ns) ForwardSessionEventToSession:612(1089ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(1689ns) ForwardSessionEventToSession:612(890ns) ForwardSessionEventToSession:612(681ns) ForwardSessionEventToSession:612(752ns) ForwardSessionEventToSession:612(695ns)}}}
2025-12-24T02:10:49.418981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241187793495557:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:49.455326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:49.455466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241190363919591:2048] [node 1] ICS01 socket: 114 reason# EndOfStream
2025-12-24T02:10:49.455480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241190363919496:2048] [node 1] ICP30 unregister session Session# [2:7587241190363919591:2048] VirtualId# [2:7587241190363919587:0]
2025-12-24T02:10:49.455492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241190363919591:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.455538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.455561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.462777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:49.462895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241187793495581:2048] [node 1] ICS01 socket: 67 reason# EndOfStream
2025-12-24T02:10:49.462904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241187793495493:2048] [node 1] ICP30 unregister session Session# [3:7587241187793495581:2048] VirtualId# [3:7587241187793495567:0]
2025-12-24T02:10:49.462914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241187793495581:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.462989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.463044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.466749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:49.466855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241188723995430:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-12-24T02:10:49.466865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241188723995330:2048] [node 1] ICP30 unregister session Session# [4:7587241188723995430:2048] VirtualId# [4:7587241188723995405:0]
2025-12-24T02:10:49.466873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241188723995430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.466930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.466954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.466944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:49.467024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241188336974367:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-12-24T02:10:49.467033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241188336974276:2048] [node 1] ICP30 unregister session Session# [5:7587241188336974367:2048] VirtualId# [5:7587241188336974357:0]
2025-12-24T02:10:49.467039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241188336974367:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.467082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.467106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.467769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:49.467863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241188476445430:2048] [node 1] ICS01 socket: 91 reason# EndOfStream
2025-12-24T02:10:49.467870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241188476445339:2048] [node 1] ICP30 unregister session Session# [6:7587241188476445430:2048] VirtualId# [6:7587241188476445419:0]
2025-12-24T02:10:49.467877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241188476445430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.467928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.467951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.470783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:49.470882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241187793495593:2049] [node 2] ICS01 socket: 109 reason# EndOfStream
2025-12-24T02:10:49.470892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241187793495494:2049] [node 2] ICP30 unregister session Session# [3:7587241187793495593:2049] VirtualId# [3:7587241187793495584:0]
2025-12-24T02:10:49.470900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241187793495593:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.470956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:49.470982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:49.482762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:49.482858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241188336974353:2049] [node 2] ICS01 socket: 70 reason# EndOfStream
2025-12-24T02:10:49.482800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:49.482890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241188723995428:2049] [node 2] ICS01 socket: 65 reason# EndOfStream
2025-12-24T02:10:49.482869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241188336974277:2049] [node 2] ICP30 unregister session Session# [5:7587241188336974353:2049] VirtualId# [5:7587241188336974343:0]
2025-12-24T02:10:49.482878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241188336974353:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.482944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:49.482971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:49.482897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241188723995331:2049] [node 2] ICP30 unregister session Session# [4:7587241188723995428:2049] VirtualId# [4:7587241188723995417:0]
2025-12-24T02:10:49.482904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241188723995428:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.482961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:49.482985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:49.485771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:49.485886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241188476445425:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-12-24T02:10:49.485896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241188476445340:2049] [node 2] ICP30 unregister session Session# [6:7587241188476445425:2049] VirtualId# [6:7587241188476445420:0]
2025-12-24T02:10:49.485904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241188476445425:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.485969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:49.485997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:49.506754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:49.506856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241188336974371:2050] [node 3] ICS01 socket: 123 reason# EndOfStream
2025-12-24T02:10:49.506864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241188336974278:2050] [node 3] ICP30 unregister session Session# [5:7587241188336974371:2050] VirtualId# [5:7587241188336974363:0]
2025-12-24T02:10:49.506874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241188336974371:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.506922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:49.506947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:49.507719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:49.507813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241188476445435:2050] [node 3] ICS01 socket: 125 reason# EndOfStream
2025-12-24T02:10:49.507821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241188476445341:2050] [node 3] ICP30 unregister session Session# [6:7587241188476445435:2050] VirtualId# [6:7587241188476445429:0]
2025-12-24T02:10:49.507827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241188476445435:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.507880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:49.507909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:49.519559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:49.519700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241188476445437:2073] [node 4] ICS01 socket: 101 reason# EndOfStream
2025-12-24T02:10:49.519712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241188476445404:2073] [node 4] ICP30 unregister session Session# [6:7587241188476445437:2073] VirtualId# [6:7587241188476445424:0]
2025-12-24T02:10:49.519721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241188476445437:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.519784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:49.519811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:49.582873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241190713599314:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:49.582888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241190713599315:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:49.594811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241189798167164:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.594829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241189798167165:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:49.628824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241189417823169:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.628858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241189417823170:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.628864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241189417823171:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.631016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190446158649:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.631034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190446158650:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.631040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190446158651:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.630769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241190127404082:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.630803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241190127404083:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.646774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241186473620290:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:49.646791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241186473620291:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:49.646797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241186473620292:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:49.699374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241190713599376:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:49.699472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241190713599377:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.700044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241189798167224:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31337
2025-12-24T02:10:49.704312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241190713599388:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.711902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241190713599389:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63764
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.712283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241186473620324:2072] [node 5] ICP01 ready to work
2025-12-24T02:10:49.712384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241186473620325:2076] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.718731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404142:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.724400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404143:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.727303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241189798167227:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:49.727399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241189798167228:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.729026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241190713599394:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63764
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.732818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620365:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.735026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599389:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.751549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404144:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.751954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241190127404148:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:49.751998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404143:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.752079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241190127404149:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:49.752153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241190127404150:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.755793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190446158720:2068] [node 4] ICP01 ready to work
2025-12-24T02:10:49.755927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190446158721:2070] [node 6] ICP01 ready to work
2025-12-24T02:10:49.757008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158718:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.757058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158719:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.762921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241189798167239:2113] [node 5] ICP01 ready to work
2025-12-24T02:10:49.766801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241186473620365:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.766858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241186473620291:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.774579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599394:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.774729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241190713599396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63764
2025-12-24T02:10:49.775331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167234:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.775348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241189798167234:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.778902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241189798167234:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.779021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241189798167228:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.779038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.779049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241189798167228:2109] [node 6] ICP22 created new session: [2:7587241189798167245:2109]
2025-12-24T02:10:49.779057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241189798167245:2109] [node 6] ICS09 handshake done sender: [2:7587241189798167234:2112] self: [2:7587241189798167233:0] peer: [6:7587241186473620369:0] socket: 84 qp: -1
2025-12-24T02:10:49.779067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241189798167245:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:49.779080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.785099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158718:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.785222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190446158649:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.783047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241189417823234:2092] [node 6] ICP01 ready to work
2025-12-24T02:10:49.783151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241189417823235:2095] [node 5] ICP01 ready to work
2025-12-24T02:10:49.783231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823232:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.783303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823233:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.786970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167224:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.787061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241189798167248:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.786964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158730:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.790786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823232:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.790854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823233:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.790913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241189417823169:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.790928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241189417823170:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.794837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241190127404083:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(42598686ns) {Configure:104(42596716ns) {ProcessConfigured:135(42136614ns) {StartInitialHandshake:157(42135406ns) {DropHandshakes:470(1566ns) {DropIncomingHandshake:442(240ns) DropOutgoingHandshake:457(223ns)} GenerateSessionVirtualId:988(332ns) PrepareNewSessionHandshake:477(1856ns) {SwitchToState:179(220ns)}}}}}
2025-12-24T02:10:49.795566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241190127404082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.796024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404142:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.796046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404144:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.796339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.805282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599379:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.805319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241190713599379:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.805977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599396:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.806047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599383:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.806050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241190713599383:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.810923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404162:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.822948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.830714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241189798167250:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31337
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.838426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620370:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.838441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158719:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.842781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167232:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.842807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241189798167232:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.843500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241189798167232:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.843573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241189798167227:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.843590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.843603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241189798167227:2108] [node 4] ICP22 created new session: [2:7587241189798167252:2108]
2025-12-24T02:10:49.843610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241189798167252:2108] [node 4] ICS09 handshake done sender: [2:7587241189798167232:2111] self: [2:7587241189798167231:0] peer: [4:7587241189417823250:0] socket: 83 qp: -1
2025-12-24T02:10:49.843619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241189798167252:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:49.843632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.846937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158733:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.847176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158733:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.847211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190446158650:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.847839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158735:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.848098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190446158733:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.848147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190446158650:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.848164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.848171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190446158650:2049] [node 2] ICP22 created new session: [5:7587241190446158736:2049]
2025-12-24T02:10:49.848178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190446158736:2049] [node 2] ICS09 handshake done sender: [5:7587241190446158733:2119] self: [5:7587241190446158734:0] peer: [2:7587241189798167242:0] socket: 114 qp: -1
2025-12-24T02:10:49.848185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190446158736:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.848193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.847312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167243:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.847322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241189798167243:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.847931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241189798167243:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.850184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241189798167239:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.850200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.850206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241189798167239:2113] [node 5] ICP22 created new session: [2:7587241189798167256:2113]
2025-12-24T02:10:49.850216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241189798167256:2113] [node 5] ICS09 handshake done sender: [2:7587241189798167243:2115] self: [2:7587241189798167242:0] peer: [5:7587241190446158734:0] socket: 113 qp: -1
2025-12-24T02:10:49.850223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241189798167256:2113] [node 5] ICS10 traffic start
2025-12-24T02:10:49.850231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.850867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599381:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.850891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241190713599381:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.859737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241190127404083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.859891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.859941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241186473620365:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.862868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167250:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.862918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167241:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.862922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241189798167241:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.866759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241189798167241:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.867018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158731:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.867067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158739:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.862751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241190713599398:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63764
2025-12-24T02:10:49.862869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241190713599379:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.863001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241190713599383:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.866876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241190713599381:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.871515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158730:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.871545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190446158651:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.871561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190446158718:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.871304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.871833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404169:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.871683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190446158649:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.871703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.871709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190446158649:2048] [node 1] ICP22 created new session: [5:7587241190446158741:2048]
2025-12-24T02:10:49.871714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190446158741:2048] [node 1] ICS09 handshake done sender: [5:7587241190446158718:2112] self: [5:7587241190446158729:0] peer: [1:7587241190713599378:0] socket: 90 qp: -1
2025-12-24T02:10:49.871722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190446158741:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.871731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.871858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190446158720:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.871939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241186473620291:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.871962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.871969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241186473620291:2049] [node 2] ICP22 created new session: [6:7587241186473620372:2049]
2025-12-24T02:10:49.871977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241186473620372:2049] [node 2] ICS09 handshake done sender: [6:7587241186473620365:2114] self: [6:7587241186473620369:0] peer: [2:7587241189798167233:0] socket: 87 qp: -1
2025-12-24T02:10:49.871984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241186473620372:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.871992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.878772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241189798167165:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.878810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.878820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241189798167165:2049] [node 3] ICP22 created new session: [2:7587241189798167260:2049]
2025-12-24T02:10:49.878828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241189798167260:2049] [node 3] ICS09 handshake done sender: [2:7587241189798167241:2114] self: [2:7587241189798167240:0] peer: [3:7587241190127404168:0] socket: 96 qp: -1
2025-12-24T02:10:49.878838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241189798167260:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:49.878848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.883141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241190713599376:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.883176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.883185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241190713599376:2107] [node 5] ICP22 created new session: [1:7587241190713599403:2107]
2025-12-24T02:10:49.883193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241190713599403:2107] [node 5] ICS09 handshake done sender: [1:7587241190713599379:2109] self: [1:7587241190713599378:0] peer: [5:7587241190446158729:0] socket: 70 qp: -1
2025-12-24T02:10:49.883201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241190713599403:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:49.883211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.883448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241190713599315:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.883456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.883461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241190713599315:2049] [node 3] ICP22 created new session: [1:7587241190713599405:2049]
2025-12-24T02:10:49.883466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241190713599405:2049] [node 3] ICS09 handshake done sender: [1:7587241190713599383:2111] self: [1:7587241190713599382:0] peer: [3:7587241190127404157:0] socket: 72 qp: -1
2025-12-24T02:10:49.883470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241190713599405:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:49.883476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.883603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599398:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.883612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241190713599377:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.883618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.883621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241190713599377:2108] [node 4] ICP22 created new session: [1:7587241190713599407:2108]
2025-12-24T02:10:49.883625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241190713599407:2108] [node 4] ICS09 handshake done sender: [1:7587241190713599381:2110] self: [1:7587241190713599380:0] peer: [4:7587241189417823249:0] socket: 71 qp: -1
2025-12-24T02:10:49.883630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241190713599407:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:49.883636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.890941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.891018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823253:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.898721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404154:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.898751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241190127404154:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.903188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241190713599315:2049] [node 3] ICP18 handshake (actor [1:7587241190713599398:2118]) is too old
2025-12-24T02:10:49.903303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241186473620291:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31329979ns) {HandleHandshakeStatus:329(31329161ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(1952ns) SwitchToState:179(105ns) ProcessPendingSessionEvents:495(24281943ns) {ForwardSessionEventToSession:612(24268570ns) ForwardSessionEventToSession:612(2217ns) ForwardSessionEventToSession:612(1036ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(1087ns) ForwardSessionEventToSession:612(750ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(1196ns)}}}
2025-12-24T02:10:49.918830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241190127404143:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.922724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823239:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.922748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241189417823239:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.922914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241186473620371:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.931625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823252:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.931732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241189417823233:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.935497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241189417823232:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.935277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620374:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.935332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.935624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167248:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.935378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241186473620325:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.937028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241186473620374:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.937057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241186473620324:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.935713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241189798167227:2108] [node 4] ICP18 handshake (actor [2:7587241189798167248:2116]) is too old
2025-12-24T02:10:49.938778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241190127404083:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27002) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241190127404162:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.938823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190446158727:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.938836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241190446158727:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.939423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241189417823170:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.939445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.939453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241189417823170:2049] [node 2] ICP22 created new session: [4:7587241189417823255:2049]
2025-12-24T02:10:49.939460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241189417823255:2049] [node 2] ICS09 handshake done sender: [4:7587241189417823233:2111] self: [4:7587241189417823250:0] peer: [2:7587241189798167231:0] socket: 89 qp: -1
2025-12-24T02:10:49.939469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241189417823255:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.939478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.939719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823237:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.939723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241189417823237:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.940123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241189417823169:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.940136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.940144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241189417823169:2048] [node 1] ICP22 created new session: [4:7587241189417823258:2048]
2025-12-24T02:10:49.940149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241189417823258:2048] [node 1] ICS09 handshake done sender: [4:7587241189417823232:2110] self: [4:7587241189417823249:0] peer: [1:7587241190713599380:0] socket: 74 qp: -1
2025-12-24T02:10:49.940155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241189417823258:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.940161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.940996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241189798167262:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31337
2025-12-24T02:10:49.941813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190446158727:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.942226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241190446158721:2070] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.942243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.942248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190446158721:2070] [node 6] ICP22 created new session: [5:7587241190446158748:2070]
2025-12-24T02:10:49.942255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190446158748:2070] [node 6] ICS09 handshake done sender: [5:7587241190446158727:2116] self: [5:7587241190446158726:0] peer: [6:7587241186473620378:0] socket: 92 qp: -1
2025-12-24T02:10:49.942260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190446158748:2070] [node 6] ICS10 traffic start
2025-12-24T02:10:49.942267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.942056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404170:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.942618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241190127404082:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.942630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.942636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241190127404082:2048] [node 1] ICP22 created new session: [3:7587241190127404172:2048]
2025-12-24T02:10:49.942642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241190127404172:2048] [node 1] ICS09 handshake done sender: [3:7587241190127404143:2108] self: [3:7587241190127404157:0] peer: [1:7587241190713599382:0] socket: 80 qp: -1
2025-12-24T02:10:49.942648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241190127404172:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.942654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.942736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241190127404082:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.942866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241186473620375:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.942905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.943090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241186473620290:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.943103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241186473620374:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.943040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158746:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.943068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190446158747:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48999
2025-12-24T02:10:49.943159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241186473620324:2072] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.943177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.943183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241186473620324:2072] [node 5] ICP22 created new session: [6:7587241186473620381:2072]
2025-12-24T02:10:49.943188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241186473620381:2072] [node 5] ICS09 handshake done sender: [6:7587241186473620374:2118] self: [6:7587241186473620378:0] peer: [5:7587241190446158726:0] socket: 110 qp: -1
2025-12-24T02:10:49.943194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241186473620381:2072] [node 5] ICS10 traffic start
2025-12-24T02:10:49.943199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.943883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241186473620324:2072] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:49.942905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241190127404162:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.943320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241190127404154:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.946793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190446158731:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.946984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599391:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.946998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241190713599391:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.946837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241190127404083:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.946853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.946861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241190127404083:2049] [node 2] ICP22 created new session: [3:7587241190127404176:2049]
2025-12-24T02:10:49.946868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241190127404176:2049] [node 2] ICS09 handshake done sender: [3:7587241190127404162:2118] self: [3:7587241190127404168:0] peer: [2:7587241189798167240:0] socket: 103 qp: -1
2025-12-24T02:10:49.946875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241190127404176:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:49.946883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.947079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241190127404148:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.947089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.947095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241190127404148:2110] [node 5] ICP22 created new session: [3:7587241190127404178:2110]
2025-12-24T02:10:49.947100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241190127404178:2110] [node 5] ICS09 handshake done sender: [3:7587241190127404154:2114] self: [3:7587241190127404153:0] peer: [5:7587241190446158740:0] socket: 105 qp: -1
2025-12-24T02:10:49.947103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241190127404178:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:49.947110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.951206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823257:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.951310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823262:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.951454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241189798167262:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.951484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241189798167164:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.952086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190446158651:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.952104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.952109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190446158651:2050] [node 3] ICP22 created new session: [5:7587241190446158750:2050]
2025-12-24T02:10:49.952115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190446158750:2050] [node 3] ICS09 handshake done sender: [5:7587241190446158731:2118] self: [5:7587241190446158740:0] peer: [3:7587241190127404153:0] socket: 107 qp: -1
2025-12-24T02:10:49.952120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190446158750:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.952127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.951998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620384:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.952020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620385:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.952030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620383:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.954790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241190713599391:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.954799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241189417823237:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.955416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241190127404180:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49968
2025-12-24T02:10:49.954957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241189417823234:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.954969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.954975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241189417823234:2092] [node 6] ICP22 created new session: [4:7587241189417823264:2092]
2025-12-24T02:10:49.954981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241189417823264:2092] [node 6] ICS09 handshake done sender: [4:7587241189417823237:2112] self: [4:7587241189417823236:0] peer: [6:7587241186473620377:0] socket: 97 qp: -1
2025-12-24T02:10:49.954987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241189417823264:2092] [node 6] ICS10 traffic start
2025-12-24T02:10:49.954996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.956270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241189417823239:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:49.956452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823257:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.956466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241189417823235:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.956479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:49.956483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241189417823235:2095] [node 5] ICP22 created new session: [4:7587241189417823266:2095]
2025-12-24T02:10:49.956488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241189417823266:2095] [node 5] ICS09 handshake done sender: [4:7587241189417823239:2113] self: [4:7587241189417823238:0] peer: [5:7587241190446158743:0] socket: 98 qp: -1
2025-12-24T02:10:49.956492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241189417823266:2095] [node 5] ICS10 traffic start
2025-12-24T02:10:49.956498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:49.956888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241189417823234:2092] [node 6] ICP18 handshake (actor [4:7587241189417823257:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.955610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241190713599393:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.955622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241190713599393:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.955897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241190713599388:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.955911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.955916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241190713599388:2112] [node 6] ICP22 created new session: [1:7587241190713599412:2112]
2025-12-24T02:10:49.955921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241190713599412:2112] [node 6] ICS09 handshake done sender: [1:7587241190713599391:2114] self: [1:7587241190713599390:0] peer: [6:7587241186473620380:0] socket: 93 qp: -1
2025-12-24T02:10:49.955926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241190713599412:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:49.955932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.958082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241190713599393:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:49.958145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241190713599314:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.958156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:49.958161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241190713599314:2048] [node 2] ICP22 created new session: [1:7587241190713599415:2048]
2025-12-24T02:10:49.958166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241190713599415:2048] [node 2] ICS09 handshake done sender: [1:7587241190713599393:2115] self: [1:7587241190713599392:0] peer: [2:7587241189798167263:0] socket: 109 qp: -1
2025-12-24T02:10:49.958170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241190713599415:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:49.958177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:49.958653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241189798167264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31337
2025-12-24T02:10:49.959617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241189798167262:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.959677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241189798167164:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.959692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.959697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241189798167164:2048] [node 1] ICP22 created new session: [2:7587241189798167265:2048]
2025-12-24T02:10:49.959704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241189798167265:2048] [node 1] ICS09 handshake done sender: [2:7587241189798167262:2118] self: [2:7587241189798167263:0] peer: [1:7587241190713599392:0] socket: 104 qp: -1
2025-12-24T02:10:49.959709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241189798167265:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.959716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.960498Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241190446158720:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241190446158730:2117] held: no
2025-12-24T02:10:49.960539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190446158730:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.960614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190446158720:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.960626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.960631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190446158720:2068] [node 4] ICP22 created new session: [5:7587241190446158752:2068]
2025-12-24T02:10:49.960636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190446158752:2068] [node 4] ICS09 handshake done sender: [5:7587241190446158730:2117] self: [5:7587241190446158743:0] peer: [4:7587241189417823238:0] socket: 100 qp: -1
2025-12-24T02:10:49.960642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190446158752:2068] [node 4] ICS10 traffic start
2025-12-24T02:10:49.960648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.961032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404161:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.961041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241190127404161:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:49.961626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241190127404161:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:49.961705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241190127404150:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.961722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:49.961727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241190127404150:2112] [node 6] ICP22 created new session: [3:7587241190127404182:2112]
2025-12-24T02:10:49.961732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241190127404182:2112] [node 6] ICS09 handshake done sender: [3:7587241190127404161:2117] self: [3:7587241190127404160:0] peer: [6:7587241186473620390:0] socket: 88 qp: -1
2025-12-24T02:10:49.961737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241190127404182:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:49.961744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:49.956661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241186473620375:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:49.956718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241186473620371:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.956757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241186473620290:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.956765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:49.956770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241186473620290:2048] [node 1] ICP22 created new session: [6:7587241186473620386:2048]
2025-12-24T02:10:49.956776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241186473620386:2048] [node 1] ICS09 handshake done sender: [6:7587241186473620375:2119] self: [6:7587241186473620380:0] peer: [1:7587241190713599390:0] socket: 126 qp: -1
2025-12-24T02:10:49.956780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241186473620386:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:49.956787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:49.957129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241186473620325:2076] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.957158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.957162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241186473620325:2076] [node 4] ICP22 created new session: [6:7587241186473620388:2076]
2025-12-24T02:10:49.957167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241186473620388:2076] [node 4] ICS09 handshake done sender: [6:7587241186473620371:2117] self: [6:7587241186473620377:0] peer: [4:7587241189417823236:0] socket: 111 qp: -1
2025-12-24T02:10:49.957171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241186473620388:2076] [node 4] ICS10 traffic start
2025-12-24T02:10:49.957176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.958846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241186473620383:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.960924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241186473620292:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.967089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241189417823262:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.967190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241189417823171:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:49.967963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404180:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.968050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241190127404159:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:49.968054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241190127404159:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.969412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241190127404159:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:49.969553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241190127404149:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:49.969570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:49.969576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241190127404149:2111] [node 4] ICP22 created new session: [3:7587241190127404186:2111]
2025-12-24T02:10:49.969581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241190127404186:2111] [node 4] ICS09 handshake done sender: [3:7587241190127404159:2116] self: [3:7587241190127404158:0] peer: [4:7587241189417823268:0] socket: 91 qp: -1
2025-12-24T02:10:49.969586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241190127404186:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:49.969593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:49.968837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241186473620292:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241186473620383:2121] held: no
2025-12-24T02:10:49.968926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241186473620391:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64250
2025-12-24T02:10:49.969060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241186473620383:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.969105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241186473620292:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.969116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.969123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241186473620292:2050] [node 3] ICP22 created new session: [6:7587241186473620392:2050]
2025-12-24T02:10:49.969128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241186473620392:2050] [node 3] ICS09 handshake done sender: [6:7587241186473620383:2121] self: [6:7587241186473620390:0] peer: [3:7587241190127404160:0] socket: 133 qp: -1
2025-12-24T02:10:49.969135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241186473620392:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.969144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:49.971417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241189417823171:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241189417823262:2121] held: no
2025-12-24T02:10:49.971479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241189417823269:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62582
2025-12-24T02:10:49.971615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241189417823262:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:49.971667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241189417823171:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:49.971679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:49.971684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241189417823171:2050] [node 3] ICP22 created new session: [4:7587241189417823270:2050]
2025-12-24T02:10:49.971688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241189417823270:2050] [node 3] ICS09 handshake done sender: [4:7587241189417823262:2121] self: [4:7587241189417823268:0] peer: [3:7587241190127404158:0] socket: 99 qp: -1
2025-12-24T02:10:49.971694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241189417823270:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:49.971700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:49.979959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:49.980057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241189798167265:2048] [node 1] ICS01 socket: 104 reason# EndOfStream
2025-12-24T02:10:49.980067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241189798167164:2048] [node 1] ICP30 unregister session Session# [2:7587241189798167265:2048] VirtualId# [2:7587241189798167263:0]
2025-12-24T02:10:49.980075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241189798167265:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.980109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.980129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.980523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:49.980587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241190127404172:2048] [node 1] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:10:49.980591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241190127404082:2048] [node 1] ICP30 unregister session Session# [3:7587241190127404172:2048] VirtualId# [3:7587241190127404157:0]
2025-12-24T02:10:49.980595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241190127404172:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.980619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.980635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.986893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:49.986982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190446158741:2048] [node 1] ICS01 socket: 90 reason# EndOfStream
2025-12-24T02:10:49.986990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190446158649:2048] [node 1] ICP30 unregister session Session# [5:7587241190446158741:2048] VirtualId# [5:7587241190446158729:0]
2025-12-24T02:10:49.986998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190446158741:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.987064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.987089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.987130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:49.987191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241186473620386:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-12-24T02:10:49.987196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241186473620290:2048] [node 1] ICP30 unregister session Session# [6:7587241186473620386:2048] VirtualId# [6:7587241186473620380:0]
2025-12-24T02:10:49.987200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241186473620386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.987228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.987246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.990740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:49.990826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241189417823258:2048] [node 1] ICS01 socket: 74 reason# EndOfStream
2025-12-24T02:10:49.990834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241189417823169:2048] [node 1] ICP30 unregister session Session# [4:7587241189417823258:2048] VirtualId# [4:7587241189417823249:0]
2025-12-24T02:10:49.990841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241189417823258:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.990883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:49.990905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:49.991804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:49.991922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241190127404176:2049] [node 2] ICS01 socket: 103 reason# EndOfStream
2025-12-24T02:10:49.991932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241190127404083:2049] [node 2] ICP30 unregister session Session# [3:7587241190127404176:2049] VirtualId# [3:7587241190127404168:0]
2025-12-24T02:10:49.991939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241190127404176:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:49.992008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:49.992034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.002770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:50.002854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241189417823255:2049] [node 2] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:10:50.002863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241189417823170:2049] [node 2] ICP30 unregister session Session# [4:7587241189417823255:2049] VirtualId# [4:7587241189417823250:0]
2025-12-24T02:10:50.002871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241189417823255:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.002915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.002940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.002862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:50.002949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241186473620372:2049] [node 2] ICS07 socket disconnect 87 reason# EndOfStream
2025-12-24T02:10:50.002955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241186473620372:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.003019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.003044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241186473620372:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:50.004127Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:50.004235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190446158736:2049] [node 2] ICS01 socket: 114 reason# EndOfStream
2025-12-24T02:10:50.004245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190446158650:2049] [node 2] ICP30 unregister session Session# [5:7587241190446158736:2049] VirtualId# [5:7587241190446158734:0]
2025-12-24T02:10:50.004252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190446158736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.004317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.004347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.010946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241186473620291:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27002) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:50.010969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241186473620372:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:50.010975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241186473620291:2049] [node 2] ICP30 unregister session Session# [6:7587241186473620372:2049] VirtualId# [6:7587241186473620369:0]
2025-12-24T02:10:50.010999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.011018Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241186473620291:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.011021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27002) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.013374Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:50.013486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190446158750:2050] [node 3] ICS01 socket: 107 reason# EndOfStream
2025-12-24T02:10:50.013496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190446158651:2050] [node 3] ICP30 unregister session Session# [5:7587241190446158750:2050] VirtualId# [5:7587241190446158740:0]
2025-12-24T02:10:50.013503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190446158750:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.013553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.013582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.014777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:50.014865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241189417823270:2050] [node 3] ICS01 socket: 99 reason# EndOfStream
2025-12-24T02:10:50.014872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241189417823171:2050] [node 3] ICP30 unregister session Session# [4:7587241189417823270:2050] VirtualId# [4:7587241189417823268:0]
2025-12-24T02:10:50.014879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241189417823270:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.014912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.014939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.014839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:50.014918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241186473620392:2050] [node 3] ICS01 socket: 133 reason# EndOfStream
2025-12-24T02:10:50.014924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241186473620292:2050] [node 3] ICP30 unregister session Session# [6:7587241186473620392:2050] VirtualId# [6:7587241186473620390:0]
2025-12-24T02:10:50.014930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241186473620392:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.014981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.015013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.022753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241186473620291:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:50.037808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:50.037941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190446158752:2068] [node 4] ICS01 socket: 100 reason# EndOfStream
2025-12-24T02:10:50.037954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190446158720:2068] [node 4] ICP30 unregister session Session# [5:7587241190446158752:2068] VirtualId# [5:7587241190446158743:0]
2025-12-24T02:10:50.037961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190446158752:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.038032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.038065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:50.038869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:50.038968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241186473620388:2076] [node 4] ICS01 socket: 111 reason# EndOfStream
2025-12-24T02:10:50.038976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241186473620325:2076] [node 4] ICP30 unregister session Session# [6:7587241186473620388:2076] VirtualId# [6:7587241186473620377:0]
2025-12-24T02:10:50.038984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241186473620388:2076] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.039062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.039087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:50.092519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192851778486:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.092529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192851778487:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.098795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241194289003279:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.098816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241194289003280:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.094776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241194630570942:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.094795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241194630570943:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.094804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241194630570944:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.106008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192227885446:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.106138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192227885447:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.106144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192227885448:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.114728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241193471634351:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:50.122727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241193471634352:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.122753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241194425805647:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.122778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241194425805648:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.122787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241194425805649:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.164475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571003:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.162968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241194425805691:2080] [node 4] ICP01 ready to work
2025-12-24T02:10:50.163513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241194425805699:2088] [node 5] ICP01 ready to work
2025-12-24T02:10:50.164360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778546:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.167997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778547:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.168017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778548:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.168029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778549:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.169884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778546:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.170031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192851778551:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:50.169939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241194289003346:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:50.170039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241194289003347:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:50.170040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241193471634418:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:50.170428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241193471634422:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.171019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192851778554:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:50.171457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241192851778554:2112] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.171081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192227885512:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:50.171466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885511:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.170980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571003:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.171039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571010:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.171166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241194630571012:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.171684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241194425805698:2096] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.171694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241194425805698:2096] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.172193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805719:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
2025-12-24T02:10:50.172366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241194425805698:2096] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.175112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241194630571012:2112] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.175248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241194630571015:2113] [node 5] ICP01 ready to work
2025-12-24T02:10:50.174799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241194289003345:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:50.177319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241194425805649:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.177339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.177349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241194425805649:2050] [node 3] ICP22 created new session: [6:7587241194425805726:2050]
2025-12-24T02:10:50.177356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241194425805726:2050] [node 3] ICS09 handshake done sender: [6:7587241194425805698:2096] self: [6:7587241194425805697:0] peer: [3:7587241192851778557:0] socket: 72 qp: -1
2025-12-24T02:10:50.177362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241194425805726:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.177383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.177550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241194425805701:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.177555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241194425805701:2097] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.178666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.179406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192851778561:2114] [node 4] ICP01 ready to work
2025-12-24T02:10:50.178817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241194425805701:2097] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.182955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241194425805691:2080] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.182971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.182979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241194425805691:2080] [node 4] ICP22 created new session: [6:7587241194425805730:2080]
2025-12-24T02:10:50.182987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241194425805730:2080] [node 4] ICS09 handshake done sender: [6:7587241194425805701:2097] self: [6:7587241194425805700:0] peer: [4:7587241194630571014:0] socket: 73 qp: -1
2025-12-24T02:10:50.182993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241194425805730:2080] [node 4] ICS10 traffic start
2025-12-24T02:10:50.183002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.183779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241193471634427:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.210946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778566:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.215319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003356:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.216366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241194425805647:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26107491ns) {Configure:104(26105855ns) {ProcessConfigured:135(25777847ns) {StartInitialHandshake:157(25776644ns) {DropHandshakes:470(884ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:477(1264ns) {SwitchToState:179(742ns)}}}}}
2025-12-24T02:10:50.215715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241194630571003:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.235278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241194425805719:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:10:50.235330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241194425805699:2088] [node 5] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.234985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241192227885446:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18895768ns) {Configure:104(18894659ns) {ProcessConfigured:135(18562523ns) {StartInitialHandshake:157(18561724ns) {DropHandshakes:470(1072ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(1356ns) {SwitchToState:179(396ns)}}}}}
2025-12-24T02:10:50.235241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885511:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.235406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192227885522:2113] [node 4] ICP01 ready to work
2025-12-24T02:10:50.235449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885516:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.235480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885521:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.236120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241192227885447:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(57115903ns) {Configure:104(57110633ns) {ProcessConfigured:135(56740591ns) {StartInitialHandshake:157(56740217ns) {DropHandshakes:470(476ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(815ns) {SwitchToState:179(31ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.239599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241193471634428:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33886
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.247216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778549:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.247537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192851778546:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.246962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241193471634432:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33886
2025-12-24T02:10:50.246987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241193471634433:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33886
2025-12-24T02:10:50.247002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241193471634431:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33886
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.249923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805736:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
2025-12-24T02:10:50.249137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885514:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.249172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241192227885514:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.249790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.250141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192227885514:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.250264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241192227885512:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.250284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.250291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192227885512:2109] [node 6] ICP22 created new session: [5:7587241192227885530:2109]
2025-12-24T02:10:50.250297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192227885530:2109] [node 6] ICS09 handshake done sender: [5:7587241192227885514:2111] self: [5:7587241192227885513:0] peer: [6:7587241194425805735:0] socket: 94 qp: -1
2025-12-24T02:10:50.250304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192227885530:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:50.250325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.252123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241194630571012:2112] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.252142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.252147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241194630571012:2112] [node 6] ICP22 created new session: [4:7587241194630571025:2112]
2025-12-24T02:10:50.252154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241194630571025:2112] [node 6] ICS09 handshake done sender: [4:7587241194630571003:2108] self: [4:7587241194630571014:0] peer: [6:7587241194425805700:0] socket: 76 qp: -1
2025-12-24T02:10:50.252160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241194630571025:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:50.252176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.252356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571010:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.252464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571022:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.252476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571023:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.252488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571024:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.252537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241194630570942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.254784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634431:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.258742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634421:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.250915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778548:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.255194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003360:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.255244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003359:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.259272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571024:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.259314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571017:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.259317Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241194630571017:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.258991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003356:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.259110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241194289003279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.259671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003359:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.258749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241193471634421:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.259432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634428:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.259476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241193471634421:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.258829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885516:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.258965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192227885522:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.259190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885521:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.259208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192227885447:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.266819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634415:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.266841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241193471634415:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.268694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003349:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.268718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241194289003349:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.269716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241194425805719:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.269829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241194425805699:2088] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241194425805719:2110] held: no
2025-12-24T02:10:50.269852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241194425805699:2088] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.269865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.269874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241194425805699:2088] [node 5] ICP22 created new session: [6:7587241194425805737:2088]
2025-12-24T02:10:50.269881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241194425805737:2088] [node 5] ICS09 handshake done sender: [6:7587241194425805719:2110] self: [6:7587241194425805735:0] peer: [5:7587241192227885513:0] socket: 96 qp: -1
2025-12-24T02:10:50.269892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241194425805737:2088] [node 5] ICS10 traffic start
2025-12-24T02:10:50.269904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.271517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003360:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.274794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241193471634418:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.274816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.274826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241193471634418:2109] [node 4] ICP22 created new session: [1:7587241193471634436:2109]
2025-12-24T02:10:50.274833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241193471634436:2109] [node 4] ICS09 handshake done sender: [1:7587241193471634421:2111] self: [1:7587241193471634420:0] peer: [4:7587241194630571027:0] socket: 90 qp: -1
2025-12-24T02:10:50.274841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241193471634436:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:50.274850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.275027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634432:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.275103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241193471634418:2109] [node 4] ICP18 handshake (actor [1:7587241193471634428:2114]) is too old
2025-12-24T02:10:50.275704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571022:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.275840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241194630570943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.276029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241194630571017:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.276278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241194630571015:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.276290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.276297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241194630571015:2113] [node 5] ICP22 created new session: [4:7587241194630571031:2113]
2025-12-24T02:10:50.276303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241194630571031:2113] [node 5] ICS09 handshake done sender: [4:7587241194630571017:2114] self: [4:7587241194630571016:0] peer: [5:7587241192227885533:0] socket: 104 qp: -1
2025-12-24T02:10:50.276308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241194630571031:2113] [node 5] ICS10 traffic start
2025-12-24T02:10:50.276314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.278219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241194630570942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241194630571010:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.279082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241192851778554:2112] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.279105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.279113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192851778554:2112] [node 6] ICP22 created new session: [3:7587241192851778570:2112]
2025-12-24T02:10:50.279124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192851778570:2112] [node 6] ICS09 handshake done sender: [3:7587241192851778546:2107] self: [3:7587241192851778557:0] peer: [6:7587241194425805697:0] socket: 74 qp: -1
2025-12-24T02:10:50.279131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192851778570:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:50.279141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.279077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
2025-12-24T02:10:50.279309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634433:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.279186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241192227885522:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25343) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241192227885516:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.282814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241194425805739:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.282862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241194425805647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.282862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571034:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.285530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241193471634415:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.285639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241193471634351:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.285655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.285664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241193471634351:2048] [node 2] ICP22 created new session: [1:7587241193471634442:2048]
2025-12-24T02:10:50.285671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241193471634442:2048] [node 2] ICS09 handshake done sender: [1:7587241193471634415:2108] self: [1:7587241193471634414:0] peer: [2:7587241194289003361:0] socket: 87 qp: -1
2025-12-24T02:10:50.285679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241193471634442:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:50.285687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.291465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003364:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.298728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885535:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.298814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634424:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.298831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241193471634424:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.300141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805741:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
2025-12-24T02:10:50.298877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241194630571010:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.299596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241192851778554:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(20484933ns) {HandleHandshakeStatus:329(20484061ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(73ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(20374453ns) {ForwardSessionEventToSession:612(20361318ns) ForwardSessionEventToSession:612(2254ns) ForwardSessionEventToSession:612(956ns) ForwardSessionEventToSession:612(906ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(618ns) ForwardSessionEventToSession:612(890ns) ForwardSessionEventToSession:612(1321ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(487ns)}}}
2025-12-24T02:10:50.299752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778547:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.299819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241192851778486:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.299915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.299929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241192851778487:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.302925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241193471634424:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.298734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003351:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.298760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241194289003351:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.304653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241194425805739:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.304368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003340:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.304380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241194289003340:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.304914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241194289003349:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.302976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241194630570942:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.302995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.303007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241194630570942:2048] [node 1] ICP22 created new session: [4:7587241194630571035:2048]
2025-12-24T02:10:50.303023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241194630571035:2048] [node 1] ICS09 handshake done sender: [4:7587241194630571010:2111] self: [4:7587241194630571027:0] peer: [1:7587241193471634420:0] socket: 92 qp: -1
2025-12-24T02:10:50.303034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241194630571035:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.303045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.305104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192227885516:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.305180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192227885522:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.305190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.305198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192227885522:2113] [node 4] ICP22 created new session: [5:7587241192227885536:2113]
2025-12-24T02:10:50.305204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192227885536:2113] [node 4] ICS09 handshake done sender: [5:7587241192227885516:2112] self: [5:7587241192227885533:0] peer: [4:7587241194630571016:0] socket: 105 qp: -1
2025-12-24T02:10:50.305211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192227885536:2113] [node 4] ICS10 traffic start
2025-12-24T02:10:50.305218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.307120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778572:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.307299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778563:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.307302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241192851778563:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.307847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192851778563:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.308791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192851778560:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.308794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241192851778560:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.309127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241192851778561:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.309138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.309142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192851778561:2114] [node 4] ICP22 created new session: [3:7587241192851778577:2114]
2025-12-24T02:10:50.309147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192851778577:2114] [node 4] ICS09 handshake done sender: [3:7587241192851778563:2116] self: [3:7587241192851778562:0] peer: [4:7587241194630571037:0] socket: 118 qp: -1
2025-12-24T02:10:50.309152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192851778577:2114] [node 4] ICS10 traffic start
2025-12-24T02:10:50.309157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.310045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192851778560:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.308569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885528:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.308603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192227885448:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.307202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241194630571023:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.307226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241194630570944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.307753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.307986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241194630571023:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.308059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241194630570944:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.308070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.308076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241194630570944:2050] [node 3] ICP22 created new session: [4:7587241194630571039:2050]
2025-12-24T02:10:50.308081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241194630571039:2050] [node 3] ICS09 handshake done sender: [4:7587241194630571023:2118] self: [4:7587241194630571037:0] peer: [3:7587241192851778562:0] socket: 123 qp: -1
2025-12-24T02:10:50.308087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241194630571039:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.308093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.310298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241192851778551:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.310315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.310321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192851778551:2111] [node 5] ICP22 created new session: [3:7587241192851778581:2111]
2025-12-24T02:10:50.310325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192851778581:2111] [node 5] ICS09 handshake done sender: [3:7587241192851778560:2115] self: [3:7587241192851778553:0] peer: [5:7587241192227885539:0] socket: 117 qp: -1
2025-12-24T02:10:50.310330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192851778581:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:50.310336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.314837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241194289003356:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.315103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241194289003347:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.315115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.315122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241194289003347:2110] [node 5] ICP22 created new session: [2:7587241194289003370:2110]
2025-12-24T02:10:50.315127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241194289003370:2110] [node 5] ICS09 handshake done sender: [2:7587241194289003349:2111] self: [2:7587241194289003348:0] peer: [5:7587241192227885534:0] socket: 109 qp: -1
2025-12-24T02:10:50.315134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241194289003370:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:50.315140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.315243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241194289003279:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.315250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.315256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241194289003279:2048] [node 1] ICP22 created new session: [2:7587241194289003372:2048]
2025-12-24T02:10:50.315260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241194289003372:2048] [node 1] ICS09 handshake done sender: [2:7587241194289003356:2115] self: [2:7587241194289003361:0] peer: [1:7587241193471634414:0] socket: 88 qp: -1
2025-12-24T02:10:50.315264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241194289003372:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.315268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.315662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003368:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.315702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241194289003369:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21286
2025-12-24T02:10:50.315752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241194289003351:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.315831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241194289003345:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.315844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.315848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241194289003345:2108] [node 4] ICP22 created new session: [2:7587241194289003374:2108]
2025-12-24T02:10:50.315852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241194289003374:2108] [node 4] ICS09 handshake done sender: [2:7587241194289003351:2112] self: [2:7587241194289003350:0] peer: [4:7587241194630571030:0] socket: 108 qp: -1
2025-12-24T02:10:50.315856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241194289003374:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:50.315861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.317127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.317174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885542:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.317234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241194425805647:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.317252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.317259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241194425805647:2048] [node 1] ICP22 created new session: [6:7587241194425805742:2048]
2025-12-24T02:10:50.317264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241194425805742:2048] [node 1] ICS09 handshake done sender: [6:7587241194425805739:2118] self: [6:7587241194425805740:0] peer: [1:7587241193471634423:0] socket: 99 qp: -1
2025-12-24T02:10:50.317270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241194425805742:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.317276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.317400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241194289003340:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.314846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241193471634351:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(29157439ns) {HandleHandshakeStatus:329(29156429ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(224ns) SwitchToState:179(501ns) ProcessPendingSessionEvents:495(19745591ns) {ForwardSessionEventToSession:612(19733354ns) ForwardSessionEventToSession:612(2241ns) ForwardSessionEventToSession:612(1214ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(1039ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(479ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(767ns)}}}
2025-12-24T02:10:50.316378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241193471634422:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.316392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.316399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241193471634422:2110] [node 6] ICP22 created new session: [1:7587241193471634447:2110]
2025-12-24T02:10:50.316405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241193471634447:2110] [node 6] ICS09 handshake done sender: [1:7587241193471634424:2112] self: [1:7587241193471634423:0] peer: [6:7587241194425805740:0] socket: 75 qp: -1
2025-12-24T02:10:50.316410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241193471634447:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:50.316416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.318802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634412:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.318816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241193471634412:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.318863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.318899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241194630570943:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241194630571022:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.322196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778583:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.322723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192851778547:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.322952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805744:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.323131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003368:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.323205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241194289003347:2110] [node 5] ICP18 handshake (actor [2:7587241194289003368:2119]) is too old
2025-12-24T02:10:50.324024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241194289003280:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.324044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.324052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241194289003280:2049] [node 3] ICP22 created new session: [2:7587241194289003378:2049]
2025-12-24T02:10:50.324060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241194289003378:2049] [node 3] ICS09 handshake done sender: [2:7587241194289003340:2107] self: [2:7587241194289003339:0] peer: [3:7587241192851778574:0] socket: 78 qp: -1
2025-12-24T02:10:50.324066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241194289003378:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.324075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.323458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241193471634446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33886
2025-12-24T02:10:50.325865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241192851778487:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.325888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.325897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192851778487:2049] [node 2] ICP22 created new session: [3:7587241192851778584:2049]
2025-12-24T02:10:50.325903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192851778584:2049] [node 2] ICS09 handshake done sender: [3:7587241192851778547:2108] self: [3:7587241192851778574:0] peer: [2:7587241194289003339:0] socket: 83 qp: -1
2025-12-24T02:10:50.325908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192851778584:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.325916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.327327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003369:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.330369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241194630571041:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65378
2025-12-24T02:10:50.331593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241194289003280:2049] [node 3] ICP18 handshake (actor [2:7587241194289003369:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.334857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241194630571022:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.334943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241194630570943:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.335142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241194425805744:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.335191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241194425805648:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.334957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.334972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241194630570943:2049] [node 2] ICP22 created new session: [4:7587241194630571042:2049]
2025-12-24T02:10:50.334979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241194630571042:2049] [node 2] ICS09 handshake done sender: [4:7587241194630571022:2117] self: [4:7587241194630571030:0] peer: [2:7587241194289003350:0] socket: 122 qp: -1
2025-12-24T02:10:50.334986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241194630571042:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.334999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.338754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192851778586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61815
2025-12-24T02:10:50.338853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241193471634412:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.338978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192851778548:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.338997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241193471634352:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.339032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.339039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241193471634352:2049] [node 3] ICP22 created new session: [1:7587241193471634450:2049]
2025-12-24T02:10:50.339046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241193471634450:2049] [node 3] ICS09 handshake done sender: [1:7587241193471634412:2107] self: [1:7587241193471634411:0] peer: [3:7587241192851778573:0] socket: 80 qp: -1
2025-12-24T02:10:50.339053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241193471634450:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.339062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.342744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241194289003355:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.342775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241194289003355:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.344102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192227885541:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.344127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192227885521:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.344297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241194289003355:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.344391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241194289003346:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.344416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.344428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241194289003346:2109] [node 6] ICP22 created new session: [2:7587241194289003381:2109]
2025-12-24T02:10:50.344436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241194289003381:2109] [node 6] ICS09 handshake done sender: [2:7587241194289003355:2114] self: [2:7587241194289003354:0] peer: [6:7587241194425805745:0] socket: 145 qp: -1
2025-12-24T02:10:50.344445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241194289003381:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:50.344456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.346869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241192851778486:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.346894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.346903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192851778486:2048] [node 1] ICP22 created new session: [3:7587241192851778587:2048]
2025-12-24T02:10:50.346910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192851778587:2048] [node 1] ICS09 handshake done sender: [3:7587241192851778548:2109] self: [3:7587241192851778573:0] peer: [1:7587241193471634411:0] socket: 84 qp: -1
2025-12-24T02:10:50.346918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192851778587:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.346927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.358853Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241193471634352:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19808004ns) {HandleHandshakeStatus:329(19806404ns) {DropIncomingHandshake:442(5232ns) DropOutgoingHandshake:457(188ns) SwitchToState:179(120ns) ProcessPendingSessionEvents:495(19701575ns) {ForwardSessionEventToSession:612(19685204ns) ForwardSessionEventToSession:612(3692ns) ForwardSessionEventToSession:612(687ns) ForwardSessionEventToSession:612(900ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(1667ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(457ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(693ns)}}}
2025-12-24T02:10:50.359386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192227885528:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.359451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241192227885448:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30705) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241192227885528:2118] held: no
2025-12-24T02:10:50.359491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241192227885447:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241192227885521:2115] held: no
2025-12-24T02:10:50.359504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192227885447:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.359517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.359524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192227885447:2049] [node 2] ICP22 created new session: [5:7587241192227885543:2049]
2025-12-24T02:10:50.359534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192227885543:2049] [node 2] ICS09 handshake done sender: [5:7587241192227885521:2115] self: [5:7587241192227885534:0] peer: [2:7587241194289003348:0] socket: 113 qp: -1
2025-12-24T02:10:50.359541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192227885543:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.359549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.359735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192227885446:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.359963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241193471634430:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.359972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241193471634430:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.360824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192227885448:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.360843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.360848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192227885448:2050] [node 3] ICP22 created new session: [5:7587241192227885547:2050]
2025-12-24T02:10:50.360855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192227885547:2050] [node 3] ICS09 handshake done sender: [5:7587241192227885528:2118] self: [5:7587241192227885539:0] peer: [3:7587241192851778553:0] socket: 119 qp: -1
2025-12-24T02:10:50.360864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192227885547:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.360872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.361099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241193471634430:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.361175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192227885546:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43587
2025-12-24T02:10:50.361375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192227885541:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.361448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241193471634427:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.361468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.361473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241193471634427:2113] [node 5] ICP22 created new session: [1:7587241193471634453:2113]
2025-12-24T02:10:50.361478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241193471634453:2113] [node 5] ICS09 handshake done sender: [1:7587241193471634430:2115] self: [1:7587241193471634429:0] peer: [5:7587241192227885545:0] socket: 132 qp: -1
2025-12-24T02:10:50.361484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241193471634453:2113] [node 5] ICS10 traffic start
2025-12-24T02:10:50.361490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.361786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192227885446:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.361801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.361806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192227885446:2048] [node 1] ICP22 created new session: [5:7587241192227885549:2048]
2025-12-24T02:10:50.361811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192227885549:2048] [node 1] ICS09 handshake done sender: [5:7587241192227885541:2121] self: [5:7587241192227885545:0] peer: [1:7587241193471634429:0] socket: 142 qp: -1
2025-12-24T02:10:50.361818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192227885549:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.361824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.359235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241194425805746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60776
2025-12-24T02:10:50.363050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241194289003346:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18614113ns) {HandleHandshakeStatus:329(18612513ns) {DropIncomingHandshake:442(4527ns) DropOutgoingHandshake:457(152ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(18489702ns) {ForwardSessionEventToSession:612(18479735ns) ForwardSessionEventToSession:612(1691ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(1126ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(482ns) ForwardSessionEventToSession:612(478ns) ForwardSessionEventToSession:612(717ns)}}}
2025-12-24T02:10:50.364123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241194425805648:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241194425805744:2120] held: no
2025-12-24T02:10:50.364180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241194425805744:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.364296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241194425805648:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.364308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.364314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241194425805648:2049] [node 2] ICP22 created new session: [6:7587241194425805747:2049]
2025-12-24T02:10:50.364320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241194425805747:2049] [node 2] ICS09 handshake done sender: [6:7587241194425805744:2120] self: [6:7587241194425805745:0] peer: [2:7587241194289003354:0] socket: 115 qp: -1
2025-12-24T02:10:50.364327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241194425805747:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.364335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.370781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:50.370885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241194289003372:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-12-24T02:10:50.370895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241194289003279:2048] [node 1] ICP30 unregister session Session# [2:7587241194289003372:2048] VirtualId# [2:7587241194289003361:0]
2025-12-24T02:10:50.370902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241194289003372:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.370957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.370982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.372425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:50.372511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241192851778587:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-12-24T02:10:50.372520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241192851778486:2048] [node 1] ICP30 unregister session Session# [3:7587241192851778587:2048] VirtualId# [3:7587241192851778573:0]
2025-12-24T02:10:50.372527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241192851778587:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.372572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.372592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.373968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:50.374046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241194630571035:2048] [node 1] ICS01 socket: 92 reason# EndOfStream
2025-12-24T02:10:50.374053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241194630570942:2048] [node 1] ICP30 unregister session Session# [4:7587241194630571035:2048] VirtualId# [4:7587241194630571027:0]
2025-12-24T02:10:50.374059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241194630571035:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.374102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.374124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.374163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:50.374230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241194425805742:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-12-24T02:10:50.374235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241194425805647:2048] [node 1] ICP30 unregister session Session# [6:7587241194425805742:2048] VirtualId# [6:7587241194425805740:0]
2025-12-24T02:10:50.374239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241194425805742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.374264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.374281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.375478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:50.375558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192227885549:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:10:50.375566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192227885446:2048] [node 1] ICP30 unregister session Session# [5:7587241192227885549:2048] VirtualId# [5:7587241192227885545:0]
2025-12-24T02:10:50.375571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192227885549:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.375630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.375658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.380866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:50.380955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241192851778584:2049] [node 2] ICS01 socket: 83 reason# EndOfStream
2025-12-24T02:10:50.380964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241192851778487:2049] [node 2] ICP30 unregister session Session# [3:7587241192851778584:2049] VirtualId# [3:7587241192851778574:0]
2025-12-24T02:10:50.380969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241192851778584:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.381014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.381035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.382718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:50.382806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241194630571042:2049] [node 2] ICS01 socket: 122 reason# EndOfStream
2025-12-24T02:10:50.382814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241194630570943:2049] [node 2] ICP30 unregister session Session# [4:7587241194630571042:2049] VirtualId# [4:7587241194630571030:0]
2025-12-24T02:10:50.382820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241194630571042:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.382868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.382892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.382931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:50.382999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241194425805747:2049] [node 2] ICS01 socket: 115 reason# EndOfStream
2025-12-24T02:10:50.383005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241194425805648:2049] [node 2] ICP30 unregister session Session# [6:7587241194425805747:2049] VirtualId# [6:7587241194425805745:0]
2025-12-24T02:10:50.383023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241194425805747:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.383188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.383209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.386756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:50.386861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192227885543:2049] [node 2] ICS01 socket: 113 reason# EndOfStream
2025-12-24T02:10:50.386869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192227885447:2049] [node 2] ICP30 unregister session Session# [5:7587241192227885543:2049] VirtualId# [5:7587241192227885534:0]
2025-12-24T02:10:50.386877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192227885543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.386949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.386977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.402243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:50.402356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241194425805726:2050] [node 3] ICS01 socket: 72 reason# EndOfStream
2025-12-24T02:10:50.402365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241194425805649:2050] [node 3] ICP30 unregister session Session# [6:7587241194425805726:2050] VirtualId# [6:7587241194425805697:0]
2025-12-24T02:10:50.402371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241194425805726:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.402444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.402471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.402778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:50.402873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192227885547:2050] [node 3] ICS01 socket: 119 reason# EndOfStream
2025-12-24T02:10:50.402881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192227885448:2050] [node 3] ICP30 unregister session Session# [5:7587241192227885547:2050] VirtualId# [5:7587241192227885539:0]
2025-12-24T02:10:50.402886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192227885547:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.402945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.402976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.407820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:50.407878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192227885536:2113] [node 4] ICS01 socket: 105 reason# EndOfStream
2025-12-24T02:10:50.407884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192227885522:2113] [node 4] ICP30 unregister session Session# [5:7587241192227885536:2113] VirtualId# [5:7587241192227885533:0]
2025-12-24T02:10:50.407887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192227885536:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.407916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.407934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:50.407437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:50.407523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241194425805730:2080] [node 4] ICS01 socket: 73 reason# EndOfStream
2025-12-24T02:10:50.407532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241194425805691:2080] [node 4] ICP30 unregister session Session# [6:7587241194425805730:2080] VirtualId# [6:7587241194425805700:0]
2025-12-24T02:10:50.407538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241194425805730:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.407590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.407614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:50.413740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:50.413840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241194425805737:2088] [node 5] ICS01 socket: 96 reason# EndOfStream
2025-12-24T02:10:50.413849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241194425805699:2088] [node 5] ICP30 unregister session Session# [6:7587241194425805737:2088] VirtualId# [6:7587241194425805735:0]
2025-12-24T02:10:50.413856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241194425805737:2088] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.413907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:50.413931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:50.429537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241194359010475:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.429545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241194359010476:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.433291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241192888707850:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.433299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241192888707851:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.433304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241192888707852:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.427829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190890042583:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.427839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190890042584:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.425928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241194531162042:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:50.425943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241194531162043:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.432929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192233842472:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.432936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192233842473:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.432941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192233842474:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.436826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192410673786:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.436836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192410673787:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.436910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192410673785:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.450030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010533:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.450507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241194531162109:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:50.450857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241194531162113:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:50.452557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241194359010538:2105] [node 5] ICP01 ready to work
2025-12-24T02:10:50.452664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241194359010539:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:50.447814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190890042642:2105] [node 5] ICP01 ready to work
2025-12-24T02:10:50.448320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190890042647:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:50.448710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190890042650:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26376
2025-12-24T02:10:50.450833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190890042653:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26376
2025-12-24T02:10:50.447653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192410673847:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:50.451209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241194531162118:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:50.454634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241190890042656:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.455046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241192888707915:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:50.455136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241192888707916:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:50.455766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707922:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.457352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042653:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.456268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241194531162120:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43525
2025-12-24T02:10:50.458434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241194531162121:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43525
2025-12-24T02:10:50.458144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192410673860:2113] [node 4] ICP01 ready to work
2025-12-24T02:10:50.462939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241194359010549:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.463709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192233842535:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.468340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241190890042583:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.469353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842538:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.469483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842539:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.469520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842540:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.469554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842541:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.470805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842540:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.470870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192233842472:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.474960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162120:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.475064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707929:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.475215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707922:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.475257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241192888707851:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.474802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241192410673860:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16584825ns) {Configure:104(16584202ns) {ProcessConfigured:135(16216643ns) {StartInitialHandshake:157(16216137ns) {DropHandshakes:470(454ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(112ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(748ns)}}}}}
2025-12-24T02:10:50.478371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162121:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.478483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042649:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.478496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190890042649:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.479129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190890042649:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.482871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673864:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.481411Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241190890042584:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26638407ns) {Configure:104(26635201ns) {ProcessConfigured:135(26291839ns) {StartInitialHandshake:157(26291191ns) {DropHandshakes:470(463ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(100ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1008ns) {SwitchToState:179(386ns)}}}}}
2025-12-24T02:10:50.482092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190890042647:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.482104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.482115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190890042647:2107] [node 4] ICP22 created new session: [2:7587241190890042667:2107]
2025-12-24T02:10:50.482121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190890042667:2107] [node 4] ICS09 handshake done sender: [2:7587241190890042649:2109] self: [2:7587241190890042648:0] peer: [4:7587241192888707930:0] socket: 83 qp: -1
2025-12-24T02:10:50.482130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190890042667:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:50.482139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.487008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010533:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.487317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162104:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.487333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241194531162104:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.488324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241194359010538:2105] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35610418ns) {Configure:104(35609864ns) {ProcessConfigured:135(35336891ns) {StartInitialHandshake:157(35336216ns) {DropHandshakes:470(459ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(1143ns) {SwitchToState:179(400ns)}}}}}
2025-12-24T02:10:50.488578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241194359010475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.488606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010550:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
2025-12-24T02:10:50.486957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673862:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.487025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241192233842536:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:50.488605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192410673864:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.488643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192410673786:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.487154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842539:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.487176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842538:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.488795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241192233842473:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25022219ns) {Configure:104(25021505ns) {ProcessConfigured:135(24570178ns) {StartInitialHandshake:157(24569598ns) {DropHandshakes:470(832ns) {DropIncomingHandshake:442(101ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(823ns) {SwitchToState:179(510ns)}}}}}
2025-12-24T02:10:50.488821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192233842473:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.488158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190890042663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26376
2025-12-24T02:10:50.493250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162112:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.493267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241194531162112:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.493747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162102:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.493752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241194531162102:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.494448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241194531162104:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.494563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241194531162102:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.490847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042650:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.490892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042661:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.490896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190890042661:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.491269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042646:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.494629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241194531162042:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.494648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.494655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241194531162042:2048] [node 2] ICP22 created new session: [1:7587241194531162127:2048]
2025-12-24T02:10:50.494661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241194531162127:2048] [node 2] ICS09 handshake done sender: [1:7587241194531162104:2107] self: [1:7587241194531162103:0] peer: [2:7587241190890042664:0] socket: 86 qp: -1
2025-12-24T02:10:50.494667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241194531162127:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:50.491272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190890042646:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.491644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190890042661:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.494731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190890042656:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.494743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.494751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190890042656:2113] [node 6] ICP22 created new session: [2:7587241190890042672:2113]
2025-12-24T02:10:50.494758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190890042672:2113] [node 6] ICS09 handshake done sender: [2:7587241190890042661:2115] self: [2:7587241190890042660:0] peer: [6:7587241192410673865:0] socket: 116 qp: -1
2025-12-24T02:10:50.494765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190890042672:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:50.494774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.494675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.494797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241194531162043:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.494807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.494814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241194531162043:2049] [node 3] ICP22 created new session: [1:7587241194531162131:2049]
2025-12-24T02:10:50.494819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241194531162131:2049] [node 3] ICS09 handshake done sender: [1:7587241194531162102:2106] self: [1:7587241194531162101:0] peer: [3:7587241194359010553:0] socket: 85 qp: -1
2025-12-24T02:10:50.494825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241194531162131:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.494830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.499407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673866:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.499277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241190890042674:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26376
2025-12-24T02:10:50.501107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190890042653:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.503542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010550:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.502931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192410673864:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.503866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192410673786:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.503889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.503896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192410673786:2049] [node 2] ICP22 created new session: [6:7587241192410673867:2049]
2025-12-24T02:10:50.503903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192410673867:2049] [node 2] ICS09 handshake done sender: [6:7587241192410673864:2116] self: [6:7587241192410673865:0] peer: [2:7587241190890042660:0] socket: 117 qp: -1
2025-12-24T02:10:50.503913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192410673867:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.503923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.503232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707931:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.503654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707932:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.506856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241194359010549:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19780272ns) {Configure:104(19778690ns) {ProcessConfigured:135(13585581ns) {StartInitialHandshake:157(13585084ns) {DropHandshakes:470(289ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:477(796ns) {SwitchToState:179(375ns)}}}}}
2025-12-24T02:10:50.507357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010548:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.507363Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241194359010548:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.507030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707932:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.507126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707933:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.507170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241192888707852:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.507367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241192888707922:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.507670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707933:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.507679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241192888707851:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.507690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.507696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241192888707851:2049] [node 2] ICP22 created new session: [4:7587241192888707935:2049]
2025-12-24T02:10:50.507701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241192888707935:2049] [node 2] ICS09 handshake done sender: [4:7587241192888707922:2112] self: [4:7587241192888707930:0] peer: [2:7587241190890042648:0] socket: 95 qp: -1
2025-12-24T02:10:50.507707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241192888707935:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.507724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.508651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673870:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.507478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190890042646:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.508171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241194531162129:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43525
2025-12-24T02:10:50.508386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241194531162130:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43525
2025-12-24T02:10:50.508527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842558:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.509270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.511122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162130:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.508790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241194359010548:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.508926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241194359010539:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.508936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.508943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241194359010539:2107] [node 4] ICP22 created new session: [3:7587241194359010559:2107]
2025-12-24T02:10:50.508948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241194359010559:2107] [node 4] ICS09 handshake done sender: [3:7587241194359010548:2113] self: [3:7587241194359010547:0] peer: [4:7587241192888707934:0] socket: 121 qp: -1
2025-12-24T02:10:50.508953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241194359010559:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:50.508959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.519946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.522275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241190890042583:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.522303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.522313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190890042583:2048] [node 1] ICP22 created new session: [2:7587241190890042677:2048]
2025-12-24T02:10:50.522321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190890042677:2048] [node 1] ICS09 handshake done sender: [2:7587241190890042653:2111] self: [2:7587241190890042664:0] peer: [1:7587241194531162103:0] socket: 89 qp: -1
2025-12-24T02:10:50.522331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190890042677:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.522354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.522528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190890042642:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.522534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.522541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190890042642:2105] [node 5] ICP22 created new session: [2:7587241190890042679:2105]
2025-12-24T02:10:50.522546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190890042679:2105] [node 5] ICS09 handshake done sender: [2:7587241190890042646:2108] self: [2:7587241190890042645:0] peer: [5:7587241192233842553:0] socket: 82 qp: -1
2025-12-24T02:10:50.522551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190890042679:2105] [node 5] ICS10 traffic start
2025-12-24T02:10:50.522555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.526590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241194531162043:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(31754564ns) {HandleHandshakeStatus:329(31754054ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(142ns) SwitchToState:179(358ns) ProcessPendingSessionEvents:495(25507051ns) {ForwardSessionEventToSession:612(25494189ns) ForwardSessionEventToSession:612(2354ns) ForwardSessionEventToSession:612(1091ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(844ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(790ns)}}}
2025-12-24T02:10:50.522972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192233842539:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.526745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010562:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.528257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.526981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010563:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
2025-12-24T02:10:50.527168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241194359010533:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.527276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241194359010475:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.527291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.527299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241194359010475:2048] [node 1] ICP22 created new session: [3:7587241194359010565:2048]
2025-12-24T02:10:50.527306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241194359010565:2048] [node 1] ICS09 handshake done sender: [3:7587241194359010533:2103] self: [3:7587241194359010553:0] peer: [1:7587241194531162101:0] socket: 88 qp: -1
2025-12-24T02:10:50.527314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241194359010565:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.527322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.532198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010556:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.532210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241194359010556:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.532866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241194359010556:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.530998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192233842473:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.530959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192410673871:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.531047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.531000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192410673787:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.531055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192233842473:2049] [node 2] ICP22 created new session: [5:7587241192233842560:2049]
2025-12-24T02:10:50.531061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192233842560:2049] [node 2] ICS09 handshake done sender: [5:7587241192233842539:2111] self: [5:7587241192233842553:0] peer: [2:7587241190890042645:0] socket: 106 qp: -1
2025-12-24T02:10:50.531067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192233842560:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.531076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.532752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241194531162112:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.534788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241192888707851:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27074544ns) {HandleHandshakeStatus:329(27073987ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(1174ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(15761389ns) {ForwardSessionEventToSession:612(15748763ns) ForwardSessionEventToSession:612(1847ns) ForwardSessionEventToSession:612(726ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(842ns) ForwardSessionEventToSession:612(803ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(440ns) ForwardSessionEventToSession:612(785ns)}}}
2025-12-24T02:10:50.535058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707929:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.535709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707920:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.535714Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241192888707920:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.536210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241192888707920:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.534799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010564:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
2025-12-24T02:10:50.534855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241194359010549:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.534870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.534877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241194359010549:2108] [node 6] ICP22 created new session: [3:7587241194359010569:2108]
2025-12-24T02:10:50.534883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241194359010569:2108] [node 6] ICS09 handshake done sender: [3:7587241194359010556:2116] self: [3:7587241194359010555:0] peer: [6:7587241192410673872:0] socket: 120 qp: -1
2025-12-24T02:10:50.534888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241194359010569:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:50.534912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.536309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241192888707916:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.536329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.536335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241192888707916:2108] [node 6] ICP22 created new session: [4:7587241192888707941:2108]
2025-12-24T02:10:50.536341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241192888707941:2108] [node 6] ICS09 handshake done sender: [4:7587241192888707920:2111] self: [4:7587241192888707919:0] peer: [6:7587241192410673874:0] socket: 99 qp: -1
2025-12-24T02:10:50.536346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241192888707941:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:50.536363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.535468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.535579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192410673862:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.535601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192410673860:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.536975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192410673873:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.536988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192410673860:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15553) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192410673862:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.537027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707918:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.537034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241192888707918:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.538505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241192888707918:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.535081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842559:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.536605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842541:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.536627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192233842474:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.536639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192233842540:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.536735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241192233842535:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.536828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192233842472:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.536837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.536842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192233842472:2048] [node 1] ICP22 created new session: [5:7587241192233842565:2048]
2025-12-24T02:10:50.536847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192233842565:2048] [node 1] ICS09 handshake done sender: [5:7587241192233842540:2112] self: [5:7587241192233842551:0] peer: [1:7587241194531162111:0] socket: 107 qp: -1
2025-12-24T02:10:50.536852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192233842565:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.536859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.538410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.538672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192233842541:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.538734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192233842535:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.538742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.538749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192233842535:2108] [node 4] ICP22 created new session: [5:7587241192233842568:2108]
2025-12-24T02:10:50.538754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192233842568:2108] [node 4] ICS09 handshake done sender: [5:7587241192233842541:2113] self: [5:7587241192233842564:0] peer: [4:7587241192888707917:0] socket: 108 qp: -1
2025-12-24T02:10:50.538760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192233842568:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:50.538766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.535243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241194531162043:2049] [node 3] ICP18 handshake (actor [1:7587241194531162130:2118]) is too old
2025-12-24T02:10:50.538784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162129:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.538796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241194531162109:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.538811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.538819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241194531162109:2109] [node 5] ICP22 created new session: [1:7587241194531162136:2109]
2025-12-24T02:10:50.538826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241194531162136:2109] [node 5] ICS09 handshake done sender: [1:7587241194531162112:2111] self: [1:7587241194531162111:0] peer: [5:7587241192233842551:0] socket: 91 qp: -1
2025-12-24T02:10:50.538833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241194531162136:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:50.538840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.539097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010562:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.539185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010564:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.543044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241192888707915:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.543065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.543071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241192888707915:2107] [node 5] ICP22 created new session: [4:7587241192888707945:2107]
2025-12-24T02:10:50.543077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241192888707945:2107] [node 5] ICS09 handshake done sender: [4:7587241192888707918:2110] self: [4:7587241192888707917:0] peer: [5:7587241192233842564:0] socket: 98 qp: -1
2025-12-24T02:10:50.543084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241192888707945:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:50.543092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.542876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192410673847:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.550734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241194359010551:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.550755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241194359010551:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.550983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241194359010476:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.551032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673876:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.552369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192410673870:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.552410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192410673787:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2244) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192410673871:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.554808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192410673871:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.554851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192410673785:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.558042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241192233842535:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19272823ns) {HandleHandshakeStatus:329(19272432ns) {DropIncomingHandshake:442(126ns) DropOutgoingHandshake:457(659ns) SwitchToState:179(94ns) ProcessPendingSessionEvents:495(19210647ns) {ForwardSessionEventToSession:612(19198271ns) ForwardSessionEventToSession:612(1814ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(1327ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(461ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(456ns) ForwardSessionEventToSession:612(413ns) ForwardSessionEventToSession:612(937ns)}}}
2025-12-24T02:10:50.554941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241190890042666:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.554956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241190890042666:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.555077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241194531162109:2109] [node 5] ICP18 handshake (actor [1:7587241194531162129:2117]) is too old
2025-12-24T02:10:50.557179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241194531162138:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43525
2025-12-24T02:10:50.558360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241194359010551:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.558439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241194359010538:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.558452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.558456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241194359010538:2105] [node 5] ICP22 created new session: [3:7587241194359010574:2105]
2025-12-24T02:10:50.558460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241194359010574:2105] [node 5] ICS09 handshake done sender: [3:7587241194359010551:2115] self: [3:7587241194359010542:0] peer: [5:7587241192233842563:0] socket: 140 qp: -1
2025-12-24T02:10:50.558464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241194359010574:2105] [node 5] ICS10 traffic start
2025-12-24T02:10:50.558469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.556635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707947:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.558978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241192888707932:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.558251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192410673787:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.558268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.558274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192410673787:2050] [node 3] ICP22 created new session: [6:7587241192410673879:2050]
2025-12-24T02:10:50.558279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192410673879:2050] [node 3] ICS09 handshake done sender: [6:7587241192410673871:2119] self: [6:7587241192410673872:0] peer: [3:7587241194359010555:0] socket: 145 qp: -1
2025-12-24T02:10:50.558286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192410673879:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.558303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.558753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673877:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.559489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192410673862:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.559556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673881:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.559564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192410673860:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.559572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.559579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192410673860:2113] [node 4] ICP22 created new session: [6:7587241192410673882:2113]
2025-12-24T02:10:50.559584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192410673882:2113] [node 4] ICS09 handshake done sender: [6:7587241192410673862:2114] self: [6:7587241192410673874:0] peer: [4:7587241192888707919:0] socket: 104 qp: -1
2025-12-24T02:10:50.559589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192410673882:2113] [node 4] ICS10 traffic start
2025-12-24T02:10:50.559595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.558791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162122:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.558803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241194531162122:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.559074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162138:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.563086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192410673870:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.562901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241194531162122:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.563160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241194531162118:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.563172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.563177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241194531162118:2112] [node 6] ICP22 created new session: [1:7587241194531162141:2112]
2025-12-24T02:10:50.563182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241194531162141:2112] [node 6] ICS09 handshake done sender: [1:7587241194531162122:2116] self: [1:7587241194531162119:0] peer: [6:7587241192410673878:0] socket: 137 qp: -1
2025-12-24T02:10:50.563187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241194531162141:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:50.563194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.563475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192410673785:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.563488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.563494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192410673785:2048] [node 1] ICP22 created new session: [6:7587241192410673884:2048]
2025-12-24T02:10:50.563498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192410673884:2048] [node 1] ICS09 handshake done sender: [6:7587241192410673870:2118] self: [6:7587241192410673878:0] peer: [1:7587241194531162119:0] socket: 139 qp: -1
2025-12-24T02:10:50.563503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192410673884:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.563508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.567099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241192233842556:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.567118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241192233842556:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.570727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707948:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.570804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241192888707852:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.570817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.570824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241192888707852:2050] [node 3] ICP22 created new session: [4:7587241192888707949:2050]
2025-12-24T02:10:50.570830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241192888707949:2050] [node 3] ICS09 handshake done sender: [4:7587241192888707932:2118] self: [4:7587241192888707934:0] peer: [3:7587241194359010547:0] socket: 131 qp: -1
2025-12-24T02:10:50.570837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241192888707949:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.570845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.574943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241194359010538:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(16470235ns) {HandleHandshakeStatus:329(16469792ns) {DropIncomingHandshake:442(2358ns) DropOutgoingHandshake:457(173ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(16414482ns) {ForwardSessionEventToSession:612(16404987ns) ForwardSessionEventToSession:612(1485ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(485ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1226ns) ForwardSessionEventToSession:612(712ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(373ns) ForwardSessionEventToSession:612(392ns)}}}
2025-12-24T02:10:50.575254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241192233842570:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57388
2025-12-24T02:10:50.578898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241192888707948:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.578966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241192888707850:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.579216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192233842559:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.579275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241192233842474:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.579288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.579294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192233842474:2050] [node 3] ICP22 created new session: [5:7587241192233842571:2050]
2025-12-24T02:10:50.579299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192233842571:2050] [node 3] ICS09 handshake done sender: [5:7587241192233842559:2121] self: [5:7587241192233842563:0] peer: [3:7587241194359010542:0] socket: 141 qp: -1
2025-12-24T02:10:50.579307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192233842571:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.579314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.584224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241190890042666:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.584304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241190890042584:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.584319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.584325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241190890042584:2049] [node 3] ICP22 created new session: [2:7587241190890042682:2049]
2025-12-24T02:10:50.584331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241190890042682:2049] [node 3] ICS09 handshake done sender: [2:7587241190890042666:2117] self: [2:7587241190890042659:0] peer: [3:7587241194359010571:0] socket: 128 qp: -1
2025-12-24T02:10:50.584337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241190890042682:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.584343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.584779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241194531162117:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.584795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241194531162117:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.585812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241194531162117:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.585885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241194531162113:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.585900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.585906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241194531162113:2110] [node 4] ICP22 created new session: [1:7587241194531162144:2110]
2025-12-24T02:10:50.585911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241194531162144:2110] [node 4] ICS09 handshake done sender: [1:7587241194531162117:2113] self: [1:7587241194531162116:0] peer: [4:7587241192888707951:0] socket: 136 qp: -1
2025-12-24T02:10:50.585916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241194531162144:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:50.585922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.583224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010576:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
2025-12-24T02:10:50.586787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241194359010577:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50184
2025-12-24T02:10:50.586976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241192233842556:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.587146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241194359010539:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:50.587167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241194359010564:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.587261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241192233842536:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.587278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.587284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241192233842536:2109] [node 6] ICP22 created new session: [5:7587241192233842574:2109]
2025-12-24T02:10:50.587289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241192233842574:2109] [node 6] ICS09 handshake done sender: [5:7587241192233842556:2119] self: [5:7587241192233842554:0] peer: [6:7587241192410673875:0] socket: 146 qp: -1
2025-12-24T02:10:50.587297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241192233842574:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:50.587304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.587660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241194359010476:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.587675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.587680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241194359010476:2049] [node 2] ICP22 created new session: [3:7587241194359010578:2049]
2025-12-24T02:10:50.587685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241194359010578:2049] [node 2] ICS09 handshake done sender: [3:7587241194359010564:2119] self: [3:7587241194359010571:0] peer: [2:7587241190890042659:0] socket: 143 qp: -1
2025-12-24T02:10:50.587691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241194359010578:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.587697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.585517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192410673886:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1355
2025-12-24T02:10:50.587776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192410673873:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.588378Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192410673847:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192410673873:2120] held: no
2025-12-24T02:10:50.588485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192410673847:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.588492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.588498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192410673847:2108] [node 5] ICP22 created new session: [6:7587241192410673887:2108]
2025-12-24T02:10:50.588504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192410673887:2108] [node 5] ICS09 handshake done sender: [6:7587241192410673873:2120] self: [6:7587241192410673875:0] peer: [5:7587241192233842554:0] socket: 103 qp: -1
2025-12-24T02:10:50.588513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192410673887:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:50.588519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.595555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241192888707850:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241192888707948:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.595908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241192888707952:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49468
2025-12-24T02:10:50.596133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241192888707948:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.599317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241192888707850:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.599336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.599343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241192888707850:2048] [node 1] ICP22 created new session: [4:7587241192888707953:2048]
2025-12-24T02:10:50.599350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241192888707953:2048] [node 1] ICS09 handshake done sender: [4:7587241192888707948:2121] self: [4:7587241192888707951:0] peer: [1:7587241194531162116:0] socket: 97 qp: -1
2025-12-24T02:10:50.599357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241192888707953:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.599378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.612775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:50.612878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241190890042677:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:10:50.612886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241190890042583:2048] [node 1] ICP30 unregister session Session# [2:7587241190890042677:2048] VirtualId# [2:7587241190890042664:0]
2025-12-24T02:10:50.612893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241190890042677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.612941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.612972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.618768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:50.618881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241194359010565:2048] [node 1] ICS01 socket: 88 reason# EndOfStream
2025-12-24T02:10:50.618890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241194359010475:2048] [node 1] ICP30 unregister session Session# [3:7587241194359010565:2048] VirtualId# [3:7587241194359010553:0]
2025-12-24T02:10:50.618896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241194359010565:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.618944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.618968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.622986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:50.623084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192233842565:2048] [node 1] ICS01 socket: 107 reason# EndOfStream
2025-12-24T02:10:50.623096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192233842472:2048] [node 1] ICP30 unregister session Session# [5:7587241192233842565:2048] VirtualId# [5:7587241192233842551:0]
2025-12-24T02:10:50.623102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192233842565:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.623153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.623180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.623216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:50.623278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192410673884:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-12-24T02:10:50.623282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192410673785:2048] [node 1] ICP30 unregister session Session# [6:7587241192410673884:2048] VirtualId# [6:7587241192410673878:0]
2025-12-24T02:10:50.623285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192410673884:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.623308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.623328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.624889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:50.624964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241192888707953:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-12-24T02:10:50.624969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241192888707850:2048] [node 1] ICP30 unregister session Session# [4:7587241192888707953:2048] VirtualId# [4:7587241192888707951:0]
2025-12-24T02:10:50.624974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241192888707953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.624984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.625025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.630755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:50.630837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241194359010578:2049] [node 2] ICS07 socket disconnect 143 reason# EndOfStream
2025-12-24T02:10:50.630843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241194359010578:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.630886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.630906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241194359010578:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:50.633331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:50.633406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241192888707935:2049] [node 2] ICS07 socket disconnect 95 reason# EndOfStream
2025-12-24T02:10:50.633410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241192888707935:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.633452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.633473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241192888707935:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:50.633531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:50.633622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192233842560:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-12-24T02:10:50.633630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192233842473:2049] [node 2] ICP30 unregister session Session# [5:7587241192233842560:2049] VirtualId# [5:7587241192233842553:0]
2025-12-24T02:10:50.633636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192233842560:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.633682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.633707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.635947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241192888707851:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2151) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:50.635962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241192888707935:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:50.635967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241192888707851:2049] [node 2] ICP30 unregister session Session# [4:7587241192888707935:2049] VirtualId# [4:7587241192888707930:0]
2025-12-24T02:10:50.635980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.635986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241192888707851:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2151) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.635989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2151) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.636588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:50.636662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192410673867:2049] [node 2] ICS01 socket: 117 reason# EndOfStream
2025-12-24T02:10:50.636667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192410673786:2049] [node 2] ICP30 unregister session Session# [6:7587241192410673867:2049] VirtualId# [6:7587241192410673865:0]
2025-12-24T02:10:50.636672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192410673867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.636719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.636740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.644330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:50.644438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241192233842571:2050] [node 3] ICS01 socket: 141 reason# EndOfStream
2025-12-24T02:10:50.644446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241192233842474:2050] [node 3] ICP30 unregister session Session# [5:7587241192233842571:2050] VirtualId# [5:7587241192233842563:0]
2025-12-24T02:10:50.644453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192233842571:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.644503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.644528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.646753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:50.646839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192410673879:2050] [node 3] ICS01 socket: 145 reason# EndOfStream
2025-12-24T02:10:50.646846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192410673787:2050] [node 3] ICP30 unregister session Session# [6:7587241192410673879:2050] VirtualId# [6:7587241192410673872:0]
2025-12-24T02:10:50.646850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192410673879:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.646898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.646921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.663623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:50.663723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241192233842568:2108] [node 4] ICS07 socket disconnect 108 reason# EndOfStream
2025-12-24T02:10:50.663730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241192233842568:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.663792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.663826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241192233842568:2108] [node 4] ICS15 start handshake
2025-12-24T02:10:50.666781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:50.666864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192410673882:2113] [node 4] ICS01 socket: 104 reason# EndOfStream
2025-12-24T02:10:50.666870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192410673860:2113] [node 4] ICP30 unregister session Session# [6:7587241192410673882:2113] VirtualId# [6:7587241192410673874:0]
2025-12-24T02:10:50.666876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192410673882:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.666924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.666945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:50.670758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:50.670830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192410673887:2108] [node 5] ICS01 socket: 103 reason# EndOfStream
2025-12-24T02:10:50.670837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192410673847:2108] [node 5] ICP30 unregister session Session# [6:7587241192410673887:2108] VirtualId# [6:7587241192410673875:0]
2025-12-24T02:10:50.670842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192410673887:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.670886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:50.670907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:50.691863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241191290483794:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.691872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241191290483795:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.693475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192903752872:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.693494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192903752873:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.689924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241192548036170:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:50.689935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241192548036171:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:50.703366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241193791057841:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.703381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241193791057843:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.703393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241193791057842:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.705262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190762891922:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.705272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190762891924:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:50.705298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190762891923:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.714745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192293298238:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:50.714835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192293298237:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:50.718851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192293298239:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.743842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483854:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-12-24T02:10:50.743318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192293298299:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.745494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762891983:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.747415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483854:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.747799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241191290483794:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.748656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241192548036236:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:50.748784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241192548036237:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:50.748991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036231:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.749001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241192548036231:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.751040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241192548036246:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.754889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057902:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.755652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057902:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.756103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241193791057841:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.756400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762891984:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.756616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190762891983:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.756800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190762891986:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:50.757176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190762891986:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.756938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241191290483861:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:50.757832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192293298301:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.757851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241192293298301:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.755589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241192548036231:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.755685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241192548036170:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.755697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.755703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241192548036170:2048] [node 2] ICP22 created new session: [1:7587241192548036250:2048]
2025-12-24T02:10:50.755709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241192548036250:2048] [node 2] ICS09 handshake done sender: [1:7587241192548036231:2107] self: [1:7587241192548036230:0] peer: [2:7587241191290483855:0] socket: 84 qp: -1
2025-12-24T02:10:50.755715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241192548036250:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:50.755722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.757130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036240:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.757135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241192548036240:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.758811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241192548036240:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.759005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241192548036236:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.759034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.759041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241192548036236:2108] [node 4] ICP22 created new session: [1:7587241192548036255:2108]
2025-12-24T02:10:50.759048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241192548036255:2108] [node 4] ICS09 handshake done sender: [1:7587241192548036240:2111] self: [1:7587241192548036239:0] peer: [4:7587241193791057903:0] socket: 91 qp: -1
2025-12-24T02:10:50.759056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241192548036255:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:50.759064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.759509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752932:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.760096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752932:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.762816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192293298301:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.762896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190762891984:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.763207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483860:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-12-24T02:10:50.763381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036242:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.763390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241192548036242:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.763238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190762891922:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.763746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241192293298299:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.763767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.763774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192293298299:2108] [node 5] ICP22 created new session: [6:7587241192293298304:2108]
2025-12-24T02:10:50.763779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192293298304:2108] [node 5] ICS09 handshake done sender: [6:7587241192293298301:2109] self: [6:7587241192293298300:0] peer: [5:7587241190762891987:0] socket: 86 qp: -1
2025-12-24T02:10:50.763785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192293298304:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:50.763804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.763969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241191290483867:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:50.764979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241191290483854:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.765070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241191290483794:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.765083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.765090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241191290483794:2048] [node 1] ICP22 created new session: [2:7587241191290483872:2048]
2025-12-24T02:10:50.765095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241191290483872:2048] [node 1] ICS09 handshake done sender: [2:7587241191290483854:2107] self: [2:7587241191290483855:0] peer: [1:7587241192548036230:0] socket: 85 qp: -1
2025-12-24T02:10:50.765102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241191290483872:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.765119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.775192Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241192903752872:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16635351ns) {Configure:104(16634714ns) {ProcessConfigured:135(16218722ns) {StartInitialHandshake:157(16218321ns) {DropHandshakes:470(539ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1489ns) {SwitchToState:179(161ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.775443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762891992:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.775654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190762891983:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.775654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298311:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.775709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298312:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.779330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241193791057909:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:50.776717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241191290483865:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:50.782916Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241192903752873:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19625484ns) {Configure:104(19624552ns) {ProcessConfigured:135(19331128ns) {StartInitialHandshake:157(19330436ns) {DropHandshakes:470(660ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(287ns) {SwitchToState:179(59ns)}}}}}
2025-12-24T02:10:50.783077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190762891986:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.783096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.783104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190762891986:2110] [node 6] ICP22 created new session: [5:7587241190762891994:2110]
2025-12-24T02:10:50.783111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190762891994:2110] [node 6] ICS09 handshake done sender: [5:7587241190762891983:2108] self: [5:7587241190762891987:0] peer: [6:7587241192293298300:0] socket: 88 qp: -1
2025-12-24T02:10:50.783118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190762891994:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:50.783136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.786631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241190762891924:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17432440ns) {Configure:104(17431960ns) {ProcessConfigured:135(17195482ns) {StartInitialHandshake:157(17194684ns) {DropHandshakes:470(413ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(1269ns) {SwitchToState:179(483ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.787350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057907:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.787974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483875:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-12-24T02:10:50.788071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241193791057902:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.786843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762891999:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.786853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192293298311:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.786908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192293298237:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.791225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057908:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.791271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241193791057841:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.791283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.791291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241193791057841:2048] [node 1] ICP22 created new session: [4:7587241193791057918:2048]
2025-12-24T02:10:50.791298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241193791057918:2048] [node 1] ICS09 handshake done sender: [4:7587241193791057902:2108] self: [4:7587241193791057903:0] peer: [1:7587241192548036239:0] socket: 94 qp: -1
2025-12-24T02:10:50.791304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241193791057918:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.791325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.799919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241192548036242:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.800151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241192548036237:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.800164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.800171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241192548036237:2109] [node 5] ICP22 created new session: [1:7587241192548036258:2109]
2025-12-24T02:10:50.800178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241192548036258:2109] [node 5] ICS09 handshake done sender: [1:7587241192548036242:2112] self: [1:7587241192548036241:0] peer: [5:7587241190762891988:0] socket: 96 qp: -1
2025-12-24T02:10:50.800187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241192548036258:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:50.800195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.799253Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241191290483865:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22447553ns) {Configure:104(22446903ns) {ProcessConfigured:135(22156824ns) {StartInitialHandshake:157(22156115ns) {DropHandshakes:470(635ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(1307ns) {SwitchToState:179(594ns)}}}}}
2025-12-24T02:10:50.799475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892001:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.802758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190762891984:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.803178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192903752942:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:50.803232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241192903752872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.803701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241193791057920:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:50.805561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036248:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.805581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241192548036248:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.806761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192903752939:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.804948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241192293298314:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:50.807066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190762891922:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.807088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.807095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190762891922:2048] [node 1] ICP22 created new session: [5:7587241190762892002:2048]
2025-12-24T02:10:50.807102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190762892002:2048] [node 1] ICS09 handshake done sender: [5:7587241190762891984:2109] self: [5:7587241190762891988:0] peer: [1:7587241192548036241:0] socket: 97 qp: -1
2025-12-24T02:10:50.807109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190762892002:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.807132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.807491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036238:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.807501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241192548036238:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.808399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241192548036238:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.808471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241192548036171:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.808479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.808485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241192548036171:2049] [node 3] ICP22 created new session: [1:7587241192548036261:2049]
2025-12-24T02:10:50.808490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241192548036261:2049] [node 3] ICS09 handshake done sender: [1:7587241192548036238:2110] self: [1:7587241192548036234:0] peer: [3:7587241192903752943:0] socket: 90 qp: -1
2025-12-24T02:10:50.808496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241192548036261:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.808502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.810982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241192548036263:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20050
2025-12-24T02:10:50.813715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241192548036263:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.815584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752947:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.815647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752948:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.815660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752949:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.815671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752950:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.816026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241192903752951:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:50.815838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483877:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-12-24T02:10:50.817799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241190762892000:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.820654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241192548036265:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20050
2025-12-24T02:10:50.818796Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752940:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.827176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241192903752942:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23871371ns) {Configure:104(23870075ns) {ProcessConfigured:135(23562697ns) {StartInitialHandshake:157(23562295ns) {DropHandshakes:470(571ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1350ns) {SwitchToState:179(778ns)}}}}}
2025-12-24T02:10:50.827246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752947:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.827327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192903752932:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:50.826837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241192293298238:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55464923ns) {Configure:104(55463852ns) {ProcessConfigured:135(55200193ns) {StartInitialHandshake:157(55199486ns) {DropHandshakes:470(434ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(1649ns) {SwitchToState:179(957ns)}}}}}
2025-12-24T02:10:50.827749Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241192293298314:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22723179ns) {Configure:104(22721900ns) {ProcessConfigured:135(22424673ns) {StartInitialHandshake:157(22424401ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(636ns) {SwitchToState:179(353ns)}}}}}
2025-12-24T02:10:50.831415Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241193791057841:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40100626ns) {HandleHandshakeStatus:329(40099464ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(138ns) SwitchToState:179(185ns) ProcessPendingSessionEvents:495(24509159ns) {ForwardSessionEventToSession:612(24496793ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(1253ns) ForwardSessionEventToSession:612(746ns) ForwardSessionEventToSession:612(709ns) ForwardSessionEventToSession:612(1128ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(459ns) ForwardSessionEventToSession:612(662ns) ForwardSessionEventToSession:612(853ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.832699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298322:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.832734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298323:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.834848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241192903752872:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.834875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.834885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192903752872:2048] [node 1] ICP22 created new session: [3:7587241192903752957:2048]
2025-12-24T02:10:50.834891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192903752957:2048] [node 1] ICS09 handshake done sender: [3:7587241192903752932:2107] self: [3:7587241192903752943:0] peer: [1:7587241192548036234:0] socket: 92 qp: -1
2025-12-24T02:10:50.834901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192903752957:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.834923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:50.840191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241192548036248:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.840809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483879:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
2025-12-24T02:10:50.841076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241191290483878:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.841198Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241190762892000:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23311708ns) {Configure:104(23311107ns) {ProcessConfigured:135(23011744ns) {StartInitialHandshake:157(23011270ns) {DropHandshakes:470(693ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(824ns) {SwitchToState:179(515ns)}}}}}
2025-12-24T02:10:50.841495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057923:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.840527Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241192903752939:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24440919ns) {Configure:104(24439898ns) {ProcessConfigured:135(24164852ns) {StartInitialHandshake:157(24163865ns) {DropHandshakes:470(562ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1249ns) {SwitchToState:179(651ns)}}}}}
2025-12-24T02:10:50.841733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241192548036246:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.841762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.841770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241192548036246:2113] [node 6] ICP22 created new session: [1:7587241192548036267:2113]
2025-12-24T02:10:50.841777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241192548036267:2113] [node 6] ICS09 handshake done sender: [1:7587241192548036248:2114] self: [1:7587241192548036247:0] peer: [6:7587241192293298317:0] socket: 103 qp: -1
2025-12-24T02:10:50.841784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241192548036267:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:50.841794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.842317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192293298311:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.842502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892007:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.843007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483879:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.842628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192293298237:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.842647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:50.842652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192293298237:2048] [node 1] ICP22 created new session: [6:7587241192293298324:2048]
2025-12-24T02:10:50.842658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192293298324:2048] [node 1] ICS09 handshake done sender: [6:7587241192293298311:2112] self: [6:7587241192293298317:0] peer: [1:7587241192548036247:0] socket: 106 qp: -1
2025-12-24T02:10:50.842664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192293298324:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:50.842676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.846684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.847075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057924:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.849965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057908:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.850073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192293298322:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.850125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192293298239:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.852832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483866:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.852843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241191290483866:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.853511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483878:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.853542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483875:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.854532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241191290483866:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.851429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241193791057842:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.854220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057914:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.854648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483877:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.854224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241193791057914:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.854660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241191290483861:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.854670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.854678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241191290483861:2109] [node 4] ICP22 created new session: [2:7587241191290483884:2109]
2025-12-24T02:10:50.854684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241191290483884:2109] [node 4] ICS09 handshake done sender: [2:7587241191290483866:2113] self: [2:7587241191290483862:0] peer: [4:7587241193791057926:0] socket: 107 qp: -1
2025-12-24T02:10:50.854705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241191290483884:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:50.854715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.851666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752954:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.851681Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241192903752954:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.852368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192903752954:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.852474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241192903752951:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.852492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.852500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192903752951:2110] [node 6] ICP22 created new session: [3:7587241192903752963:2110]
2025-12-24T02:10:50.852507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192903752963:2110] [node 6] ICS09 handshake done sender: [3:7587241192903752954:2118] self: [3:7587241192903752953:0] peer: [6:7587241192293298327:0] socket: 137 qp: -1
2025-12-24T02:10:50.852514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192903752963:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:50.852531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.852938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752948:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.853824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190762891999:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.853874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190762892007:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.853906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190762892000:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.853918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190762891924:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.855053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192293298239:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192293298322:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.863158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752949:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.863065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892010:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.864321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752955:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.864338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241192903752955:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.866752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.866881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752940:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.866921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241192903752873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.867124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192293298312:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.867163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192293298238:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.867424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483857:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.867436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241191290483857:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.868053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241191290483861:2109] [node 4] ICP18 handshake (actor [2:7587241191290483877:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.868394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241192903752969:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4704
2025-12-24T02:10:50.866994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
2025-12-24T02:10:50.867322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241193791057908:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.868521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241193791057914:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.868649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057923:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.868674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057927:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.868681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241193791057909:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.868690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.868698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241193791057909:2109] [node 5] ICP22 created new session: [4:7587241193791057930:2109]
2025-12-24T02:10:50.868706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241193791057930:2109] [node 5] ICS09 handshake done sender: [4:7587241193791057914:2115] self: [4:7587241193791057913:0] peer: [5:7587241190762892008:0] socket: 116 qp: -1
2025-12-24T02:10:50.868712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241193791057930:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:50.868719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.868846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241193791057909:2109] [node 5] ICP18 handshake (actor [4:7587241193791057923:2117]) is too old
2025-12-24T02:10:50.868866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241193791057843:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.869208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241193791057842:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.869220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.869225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241193791057842:2049] [node 2] ICP22 created new session: [4:7587241193791057934:2049]
2025-12-24T02:10:50.869230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241193791057934:2049] [node 2] ICS09 handshake done sender: [4:7587241193791057908:2112] self: [4:7587241193791057926:0] peer: [2:7587241191290483862:0] socket: 109 qp: -1
2025-12-24T02:10:50.869236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241193791057934:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.869240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.869977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241192903752960:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.869985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241192903752960:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.870559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192903752960:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.870637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241192903752939:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.870651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.870657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192903752939:2108] [node 4] ICP22 created new session: [3:7587241192903752971:2108]
2025-12-24T02:10:50.870663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192903752971:2108] [node 4] ICS09 handshake done sender: [3:7587241192903752960:2120] self: [3:7587241192903752952:0] peer: [4:7587241193791057933:0] socket: 112 qp: -1
2025-12-24T02:10:50.870668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192903752971:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:50.870673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.871250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241191290483857:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.871477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192903752940:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.871932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241190762892000:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241190762891999:2115] held: no
2025-12-24T02:10:50.871558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241192903752873:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.871575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.871583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192903752873:2049] [node 2] ICP22 created new session: [3:7587241192903752973:2049]
2025-12-24T02:10:50.871591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192903752973:2049] [node 2] ICS09 handshake done sender: [3:7587241192903752940:2112] self: [3:7587241192903752968:0] peer: [2:7587241191290483856:0] socket: 101 qp: -1
2025-12-24T02:10:50.871597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192903752973:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.871604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.872191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241191290483795:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.872210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.872215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241191290483795:2049] [node 3] ICP22 created new session: [2:7587241191290483888:2049]
2025-12-24T02:10:50.872220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241191290483888:2049] [node 3] ICS09 handshake done sender: [2:7587241191290483857:2108] self: [2:7587241191290483856:0] peer: [3:7587241192903752968:0] socket: 93 qp: -1
2025-12-24T02:10:50.872225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241191290483888:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:50.872231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.874932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241193791057843:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241193791057927:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.882844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298330:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.883780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483869:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.883800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241191290483869:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.883945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298329:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.886832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241192903752955:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.887126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192293298322:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.887267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241193791057936:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4172
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.888597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892011:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.888600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892012:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.888711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241191290483869:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.888769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241191290483867:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.888792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.888804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241191290483867:2112] [node 6] ICP22 created new session: [2:7587241191290483891:2112]
2025-12-24T02:10:50.888811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241191290483891:2112] [node 6] ICS09 handshake done sender: [2:7587241191290483869:2114] self: [2:7587241191290483868:0] peer: [6:7587241192293298328:0] socket: 108 qp: -1
2025-12-24T02:10:50.888817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241191290483891:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:50.888825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:50.889006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241193791057927:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.889160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241190762892010:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.889194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241190762891923:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.889449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241192903752942:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.889468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.889474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241192903752942:2109] [node 5] ICP22 created new session: [3:7587241192903752976:2109]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.889479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241192903752976:2109] [node 5] ICS09 handshake done sender: [3:7587241192903752955:2119] self: [3:7587241192903752944:0] peer: [5:7587241190762892009:0] socket: 142 qp: -1
2025-12-24T02:10:50.889484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241192903752976:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:50.889492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.889633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241192293298330:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.889668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298331:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
2025-12-24T02:10:50.889555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190762891999:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.889714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241193791057843:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.889727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.889734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241193791057843:2050] [node 3] ICP22 created new session: [4:7587241193791057937:2050]
2025-12-24T02:10:50.889739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241193791057937:2050] [node 3] ICS09 handshake done sender: [4:7587241193791057927:2119] self: [4:7587241193791057933:0] peer: [3:7587241192903752952:0] socket: 131 qp: -1
2025-12-24T02:10:50.889744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241193791057937:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.889753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.889981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192293298239:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.889994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.889999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192293298239:2050] [node 3] ICP22 created new session: [6:7587241192293298332:2050]
2025-12-24T02:10:50.890004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192293298332:2050] [node 3] ICS09 handshake done sender: [6:7587241192293298322:2117] self: [6:7587241192293298327:0] peer: [3:7587241192903752953:0] socket: 138 qp: -1
2025-12-24T02:10:50.890009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192293298332:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.890014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.890091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241192293298314:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:50.890329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241193791057922:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.890337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241193791057922:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.890911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241193791057922:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:50.891014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241193791057920:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.891039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:50.891045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241193791057920:2110] [node 6] ICP22 created new session: [4:7587241193791057940:2110]
2025-12-24T02:10:50.891051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241193791057940:2110] [node 6] ICS09 handshake done sender: [4:7587241193791057922:2116] self: [4:7587241193791057921:0] peer: [6:7587241192293298334:0] socket: 155 qp: -1
2025-12-24T02:10:50.891056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241193791057940:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:50.891062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.890231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241191290483876:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:50.890243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241191290483876:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:50.894824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192293298238:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192293298312:2113] held: no
2025-12-24T02:10:50.894824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192293298312:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.894893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192293298238:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.894903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.894911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192293298238:2049] [node 2] ICP22 created new session: [6:7587241192293298335:2049]
2025-12-24T02:10:50.894916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192293298335:2049] [node 2] ICS09 handshake done sender: [6:7587241192293298312:2113] self: [6:7587241192293298328:0] peer: [2:7587241191290483868:0] socket: 110 qp: -1
2025-12-24T02:10:50.894923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192293298335:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.894930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.895061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241192293298314:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241192293298330:2120] held: no
2025-12-24T02:10:50.899575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241190762891924:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241190762892007:2118] held: no
2025-12-24T02:10:50.899620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190762892000:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.899629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.899635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190762892000:2111] [node 4] ICP22 created new session: [5:7587241190762892014:2111]
2025-12-24T02:10:50.899641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190762892014:2111] [node 4] ICS09 handshake done sender: [5:7587241190762891999:2115] self: [5:7587241190762892008:0] peer: [4:7587241193791057913:0] socket: 118 qp: -1
2025-12-24T02:10:50.899648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190762892014:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:50.899654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.899752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190762892007:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:50.899915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241191290483876:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:50.900364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241191290483865:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:50.900375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:50.900380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241191290483865:2111] [node 5] ICP22 created new session: [2:7587241191290483894:2111]
2025-12-24T02:10:50.900385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241191290483894:2111] [node 5] ICS09 handshake done sender: [2:7587241191290483876:2116] self: [2:7587241191290483874:0] peer: [5:7587241190762892013:0] socket: 149 qp: -1
2025-12-24T02:10:50.900390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241191290483894:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:50.900394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:50.902899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241192293298336:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:50.903009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190762891924:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.903038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:50.903046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190762891924:2050] [node 3] ICP22 created new session: [5:7587241190762892016:2050]
2025-12-24T02:10:50.903052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190762892016:2050] [node 3] ICS09 handshake done sender: [5:7587241190762892007:2118] self: [5:7587241190762892009:0] peer: [3:7587241192903752944:0] socket: 145 qp: -1
2025-12-24T02:10:50.903058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190762892016:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:50.903064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:50.903355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241190762892018:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38437
2025-12-24T02:10:50.907647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241190762891923:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241190762892010:2119] held: no
2025-12-24T02:10:50.906883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241192293298330:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:50.911072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241192293298314:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.911088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:50.911095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241192293298314:2110] [node 4] ICP22 created new session: [6:7587241192293298338:2110]
2025-12-24T02:10:50.911103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241192293298338:2110] [node 4] ICS09 handshake done sender: [6:7587241192293298330:2120] self: [6:7587241192293298334:0] peer: [4:7587241193791057921:0] socket: 157 qp: -1
2025-12-24T02:10:50.911111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241192293298338:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:50.911124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:50.910896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241190762892010:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:50.915047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241190762891923:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:50.915065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:50.915073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241190762891923:2049] [node 2] ICP22 created new session: [5:7587241190762892019:2049]
2025-12-24T02:10:50.915079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241190762892019:2049] [node 2] ICS09 handshake done sender: [5:7587241190762892010:2119] self: [5:7587241190762892013:0] peer: [2:7587241191290483874:0] socket: 150 qp: -1
2025-12-24T02:10:50.915086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241190762892019:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:50.915112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:50.923802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:50.923892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241191290483872:2048] [node 1] ICS01 socket: 85 reason# EndOfStream
2025-12-24T02:10:50.923899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241191290483794:2048] [node 1] ICP30 unregister session Session# [2:7587241191290483872:2048] VirtualId# [2:7587241191290483855:0]
2025-12-24T02:10:50.923905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241191290483872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.923938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.923958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.927281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:50.927373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241192903752957:2048] [node 1] ICS01 socket: 92 reason# EndOfStream
2025-12-24T02:10:50.927382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241192903752872:2048] [node 1] ICP30 unregister session Session# [3:7587241192903752957:2048] VirtualId# [3:7587241192903752943:0]
2025-12-24T02:10:50.927388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241192903752957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.927425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.927447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.942793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:50.942892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192293298324:2048] [node 1] ICS01 socket: 106 reason# EndOfStream
2025-12-24T02:10:50.942900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192293298237:2048] [node 1] ICP30 unregister session Session# [6:7587241192293298324:2048] VirtualId# [6:7587241192293298317:0]
2025-12-24T02:10:50.942908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192293298324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.942973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.943003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.943060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:50.943129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241193791057918:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-12-24T02:10:50.943133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241193791057918:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.943167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.943189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241193791057918:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:50.943658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241193791057841:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21070) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:50.943672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241193791057918:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:50.943676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241193791057841:2048] [node 1] ICP30 unregister session Session# [4:7587241193791057918:2048] VirtualId# [4:7587241193791057903:0]
2025-12-24T02:10:50.943691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.943696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241193791057841:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21070) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.943699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21070) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:50.948416Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:50.948509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190762892002:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-12-24T02:10:50.948517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190762891922:2048] [node 1] ICP30 unregister session Session# [5:7587241190762892002:2048] VirtualId# [5:7587241190762891988:0]
2025-12-24T02:10:50.948524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190762892002:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.948587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:50.948612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:50.949688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:50.949763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241192903752973:2049] [node 2] ICS01 socket: 101 reason# EndOfStream
2025-12-24T02:10:50.949770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241192903752873:2049] [node 2] ICP30 unregister session Session# [3:7587241192903752973:2049] VirtualId# [3:7587241192903752968:0]
2025-12-24T02:10:50.949775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241192903752973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.949825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.949845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.955822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241193791057841:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:50.962856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:50.962940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190762892019:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-12-24T02:10:50.962949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190762891923:2049] [node 2] ICP30 unregister session Session# [5:7587241190762892019:2049] VirtualId# [5:7587241190762892013:0]
2025-12-24T02:10:50.962955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190762892019:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.963010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.963055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.962841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:50.962941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241193791057934:2049] [node 2] ICS01 socket: 109 reason# EndOfStream
2025-12-24T02:10:50.962949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241193791057842:2049] [node 2] ICP30 unregister session Session# [4:7587241193791057934:2049] VirtualId# [4:7587241193791057926:0]
2025-12-24T02:10:50.962955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241193791057934:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.963011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.963052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.963096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:50.963170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192293298335:2049] [node 2] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:10:50.963175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192293298238:2049] [node 2] ICP30 unregister session Session# [6:7587241192293298335:2049] VirtualId# [6:7587241192293298328:0]
2025-12-24T02:10:50.963180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192293298335:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.963219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:50.963240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:50.978768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:50.978847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241190762892016:2050] [node 3] ICS01 socket: 145 reason# EndOfStream
2025-12-24T02:10:50.978854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241190762891924:2050] [node 3] ICP30 unregister session Session# [5:7587241190762892016:2050] VirtualId# [5:7587241190762892009:0]
2025-12-24T02:10:50.978858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241190762892016:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.978908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.978928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.978561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:50.978665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192293298332:2050] [node 3] ICS01 socket: 138 reason# EndOfStream
2025-12-24T02:10:50.978675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192293298239:2050] [node 3] ICP30 unregister session Session# [6:7587241192293298332:2050] VirtualId# [6:7587241192293298327:0]
2025-12-24T02:10:50.978681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192293298332:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.978749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:50.978775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:50.998620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:50.998731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241192293298338:2110] [node 4] ICS01 socket: 157 reason# EndOfStream
2025-12-24T02:10:50.998740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241192293298314:2110] [node 4] ICP30 unregister session Session# [6:7587241192293298338:2110] VirtualId# [6:7587241192293298334:0]
2025-12-24T02:10:50.998749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241192293298338:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:50.998817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:50.998845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:51.046661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241197685987201:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:51.046670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241197685987202:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.042825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199273203352:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.042839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199273203353:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.041695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241196039913006:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.041705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241196039913007:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.048197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241196545971829:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.048208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241196545971830:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.048212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241196545971831:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.051695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241196986675215:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.051704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241196986675216:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.051709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241196986675217:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.054579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241198658858235:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.054742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241198658858236:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.066760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241198658858237:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.103439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241196545971874:2091] [node 5] ICP01 ready to work
2025-12-24T02:10:51.103899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241196545971879:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.104615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858248:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.106886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241198658858262:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.107044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241197685987259:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30768
2025-12-24T02:10:51.107245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675233:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.108299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241196986675238:2064] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.111199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858276:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.111389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241198658858248:2060] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.111544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858277:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.111325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199273203413:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:51.110945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241197685987265:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:51.113570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987259:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.115583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199273203418:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:51.127091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913072:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
2025-12-24T02:10:51.128164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913072:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.132996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241196039913077:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:51.135031Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241199273203418:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19373386ns) {Configure:104(19372283ns) {ProcessConfigured:135(19076333ns) {StartInitialHandshake:157(19075732ns) {DropHandshakes:470(1157ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(115ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(1047ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.135684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241196986675217:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25451824ns) {Configure:104(25450780ns) {ProcessConfigured:135(25214303ns) {StartInitialHandshake:157(25214007ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(110ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(1208ns) {SwitchToState:179(1023ns)}}}}}
2025-12-24T02:10:51.135901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675283:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.135791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199273203427:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.136383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913079:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
2025-12-24T02:10:51.136519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241198658858262:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20192756ns) {Configure:104(20191178ns) {ProcessConfigured:135(19960189ns) {StartInitialHandshake:157(19959717ns) {DropHandshakes:470(205ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(918ns) {SwitchToState:179(443ns)}}}}}
2025-12-24T02:10:51.136753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241198658858235:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17976761ns) {Configure:104(17975806ns) {ProcessConfigured:135(17638786ns) {StartInitialHandshake:157(17638488ns) {DropHandshakes:470(604ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(532ns) {SwitchToState:179(176ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.138901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675284:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.139115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241196545971830:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19975648ns) {Configure:104(19975385ns) {ProcessConfigured:135(19703562ns) {StartInitialHandshake:157(19703219ns) {DropHandshakes:470(175ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(134ns) PrepareNewSessionHandshake:477(858ns) {SwitchToState:179(70ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.138846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241197685987202:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19870530ns) {Configure:104(19870064ns) {ProcessConfigured:135(19530111ns) {StartInitialHandshake:157(19506760ns) {DropHandshakes:470(510ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(430ns) PrepareNewSessionHandshake:477(1139ns) {SwitchToState:179(490ns)}} IncomingHandshake:261(10069ns) {DropIncomingHandshake:442(187ns) PrepareNewSessionHandshake:477(231ns) {SwitchToState:179(164ns)} GenerateSessionVirtualId:988(407ns) IssueIncomingHandshakeReply:193(483ns)}}}}
2025-12-24T02:10:51.139427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241197685987277:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30768
2025-12-24T02:10:51.140454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913080:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
2025-12-24T02:10:51.146346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675284:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.147162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241197685987279:2115] [node 4] ICP01 ready to work
2025-12-24T02:10:51.147563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987277:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.151115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913079:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.151090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675283:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.154745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241197685987260:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30768
2025-12-24T02:10:51.155090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987260:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.159016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203429:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.159120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203430:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.159162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203431:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.159176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203432:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.159618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203429:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.159645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203430:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.159664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203432:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.162833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241196545971831:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31663477ns) {Configure:104(31661638ns) {ProcessConfigured:135(31356297ns) {StartInitialHandshake:157(31355101ns) {DropHandshakes:470(833ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(135ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:477(1291ns) {SwitchToState:179(617ns)}}}}}
2025-12-24T02:10:51.162975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241196545971829:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31473576ns) {Configure:104(31472032ns) {ProcessConfigured:135(31256780ns) {StartInitialHandshake:157(31256585ns) {DropHandshakes:470(149ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:477(663ns) {SwitchToState:179(433ns)}}}}}
2025-12-24T02:10:51.163272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241197685987273:2112] [node 5] ICP01 ready to work
2025-12-24T02:10:51.164268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675233:2062] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.166950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203424:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.166969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199273203424:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.166823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241196039913076:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:51.166259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241198658858265:2067] [node 4] ICP01 ready to work
2025-12-24T02:10:51.172677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241198658858236:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56693344ns) {Configure:104(56691775ns) {ProcessConfigured:135(56404318ns) {StartInitialHandshake:157(56403348ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:477(1438ns) {SwitchToState:179(724ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.173651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241197685987279:2115] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26422031ns) {Configure:104(26420789ns) {ProcessConfigured:135(26117513ns) {StartInitialHandshake:157(26116973ns) {DropHandshakes:470(1044ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(159ns)} GenerateSessionVirtualId:988(379ns) PrepareNewSessionHandshake:477(591ns) {SwitchToState:179(275ns)}}}}}
2025-12-24T02:10:51.172800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241199273203427:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36927314ns) {Configure:104(36926696ns) {ProcessConfigured:135(36652379ns) {StartInitialHandshake:157(36652210ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:477(597ns) {SwitchToState:179(158ns)}}}}}
2025-12-24T02:10:51.174744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241196986675216:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.174786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241196986675216:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23624890ns) {IncomingHandshake:261(23624314ns) {DropIncomingHandshake:442(135ns) PrepareNewSessionHandshake:477(720ns) {SwitchToState:179(171ns)} GenerateSessionVirtualId:988(229ns) IssueIncomingHandshakeReply:193(23615476ns) {SwitchToState:179(430ns)}}}
2025-12-24T02:10:51.174845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241196039913007:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.174877Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241196039913007:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23705556ns) {IncomingHandshake:261(23705111ns) {DropIncomingHandshake:442(66ns) PrepareNewSessionHandshake:477(650ns) {SwitchToState:179(264ns)} GenerateSessionVirtualId:988(248ns) IssueIncomingHandshakeReply:193(23697270ns) {SwitchToState:179(577ns)}}}
2025-12-24T02:10:51.175160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971907:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.179002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241198658858277:2085] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.179072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241198658858235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.179263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203431:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.180209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241196545971907:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.180270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241196545971830:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.182962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241197685987273:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19579363ns) {Configure:104(19578781ns) {ProcessConfigured:135(19250183ns) {StartInitialHandshake:157(19233371ns) {DropHandshakes:470(924ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(411ns) PrepareNewSessionHandshake:477(1136ns) {SwitchToState:179(842ns)}} IncomingHandshake:261(7311ns) {DropIncomingHandshake:442(122ns) PrepareNewSessionHandshake:477(139ns) {SwitchToState:179(107ns)} GenerateSessionVirtualId:988(90ns) IssueIncomingHandshakeReply:193(566ns)}}}}
2025-12-24T02:10:51.183114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241199273203352:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(54949204ns) {Configure:104(54948744ns) {ProcessConfigured:135(54618457ns) {StartInitialHandshake:157(54618046ns) {DropHandshakes:470(746ns) {DropIncomingHandshake:442(135ns) DropOutgoingHandshake:457(129ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:477(509ns) {SwitchToState:179(184ns)}}}}}
2025-12-24T02:10:51.183150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241199273203352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.190104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.186829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987262:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.186844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241197685987262:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.187508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241197685987262:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.187681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241197685987201:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.187695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.187701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241197685987201:2048] [node 2] ICP22 created new session: [1:7587241197685987288:2048]
2025-12-24T02:10:51.187708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241197685987288:2048] [node 2] ICS09 handshake done sender: [1:7587241197685987262:2107] self: [1:7587241197685987261:0] peer: [2:7587241199273203438:0] socket: 105 qp: -1
2025-12-24T02:10:51.187714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241197685987288:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:51.187734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.186831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241198658858265:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.186885Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241198658858265:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20517109ns) {Configure:104(20516461ns) {ProcessConfigured:135(20127407ns) {StartInitialHandshake:157(20086421ns) {DropHandshakes:470(1001ns) {DropIncomingHandshake:442(163ns) DropOutgoingHandshake:457(236ns)} GenerateSessionVirtualId:988(773ns) PrepareNewSessionHandshake:477(1254ns) {SwitchToState:179(498ns)}} IncomingHandshake:261(31507ns) {DropIncomingHandshake:442(44ns) PrepareNewSessionHandshake:477(159ns) {SwitchToState:179(107ns)} GenerateSessionVirtualId:988(191ns) IssueIncomingHandshakeReply:193(22930ns) {SwitchToState:179(41ns)}}}}}
2025-12-24T02:10:51.191290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987267:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.191311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241197685987267:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.190108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241196545971881:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.190122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241196545971881:2095] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.191198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199273203424:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.194684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203415:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.194708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199273203415:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.194729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199273203353:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.194752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.194761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199273203353:2049] [node 3] ICP22 created new session: [2:7587241199273203441:2049]
2025-12-24T02:10:51.194768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199273203441:2049] [node 3] ICS09 handshake done sender: [2:7587241199273203424:2112] self: [2:7587241199273203423:0] peer: [3:7587241196039913081:0] socket: 117 qp: -1
2025-12-24T02:10:51.194777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199273203441:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.194786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.195585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241198658858276:2084] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.195636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241198658858262:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.195614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241196545971878:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.195623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241196545971878:2094] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.196165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241196545971878:2094] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.196236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241196545971874:2091] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.196245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.196250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241196545971874:2091] [node 5] ICP22 created new session: [4:7587241196545971910:2091]
2025-12-24T02:10:51.196256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241196545971910:2091] [node 5] ICS09 handshake done sender: [4:7587241196545971878:2094] self: [4:7587241196545971877:0] peer: [5:7587241196986675298:0] socket: 92 qp: -1
2025-12-24T02:10:51.196261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241196545971910:2091] [node 5] ICS10 traffic start
2025-12-24T02:10:51.196276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.194748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241196986675278:2077] [node 4] ICP01 ready to work
2025-12-24T02:10:51.195197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241196986675278:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.196551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241196039913077:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(63480582ns) {Configure:104(63479977ns) {ProcessConfigured:135(63213109ns) {StartInitialHandshake:157(63212772ns) {DropHandshakes:470(415ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(79ns) PrepareNewSessionHandshake:477(1140ns) {SwitchToState:179(730ns)}}}}}
2025-12-24T02:10:51.206742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675240:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.206762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241196986675240:2066] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.206736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241196039913082:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.207172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203422:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.207187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199273203422:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.208162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987274:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.208172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241197685987274:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.207861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913080:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.207891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241196039913006:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.208066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.206945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241196545971881:2095] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:10:51.210811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913089:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
::ffff:127.1.0.0
2025-12-24T02:10:51.210910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241196039913079:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.210838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199273203431:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.210943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241197685987274:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.211479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199273203422:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.211600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241197685987267:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.213113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241197685987202:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.213135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.213143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241197685987202:2049] [node 3] ICP22 created new session: [1:7587241197685987293:2049]
2025-12-24T02:10:51.213150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241197685987293:2049] [node 3] ICS09 handshake done sender: [1:7587241197685987274:2113] self: [1:7587241197685987271:0] peer: [3:7587241196039913087:0] socket: 123 qp: -1
2025-12-24T02:10:51.213156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241197685987293:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.213189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.213495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199273203415:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.213586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241199273203352:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.213599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.213605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199273203352:2048] [node 1] ICP22 created new session: [2:7587241199273203446:2048]
2025-12-24T02:10:51.213610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199273203446:2048] [node 1] ICS09 handshake done sender: [2:7587241199273203431:2115] self: [2:7587241199273203438:0] peer: [1:7587241197685987261:0] socket: 128 qp: -1
2025-12-24T02:10:51.213616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199273203446:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.213632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.213773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199273203413:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.213782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.213787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199273203413:2107] [node 4] ICP22 created new session: [2:7587241199273203448:2107]
2025-12-24T02:10:51.213792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199273203448:2107] [node 4] ICS09 handshake done sender: [2:7587241199273203415:2108] self: [2:7587241199273203414:0] peer: [4:7587241196545971908:0] socket: 111 qp: -1
2025-12-24T02:10:51.213796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199273203448:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:51.213801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.211134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241196039913007:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.211155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.211161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241196039913007:2049] [node 2] ICP22 created new session: [3:7587241196039913091:2049]
2025-12-24T02:10:51.211168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241196039913091:2049] [node 2] ICS09 handshake done sender: [3:7587241196039913079:2113] self: [3:7587241196039913081:0] peer: [2:7587241199273203423:0] socket: 119 qp: -1
2025-12-24T02:10:51.211175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241196039913091:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.211185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.214192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199273203418:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.214206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.214211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199273203418:2109] [node 5] ICP22 created new session: [2:7587241199273203450:2109]
2025-12-24T02:10:51.214215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199273203450:2109] [node 5] ICS09 handshake done sender: [2:7587241199273203422:2111] self: [2:7587241199273203419:0] peer: [5:7587241196986675295:0] socket: 116 qp: -1
2025-12-24T02:10:51.214220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199273203450:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:51.214226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.215216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.215247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675299:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.215269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675301:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.214785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241196545971879:2093] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.214802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.214809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241196545971879:2093] [node 6] ICP22 created new session: [4:7587241196545971914:2093]
2025-12-24T02:10:51.214814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241196545971914:2093] [node 6] ICS09 handshake done sender: [4:7587241196545971881:2095] self: [4:7587241196545971880:0] peer: [6:7587241198658858317:0] socket: 94 qp: -1
2025-12-24T02:10:51.214819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241196545971914:2093] [node 6] ICS10 traffic start
2025-12-24T02:10:51.214833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.216608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913090:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
2025-12-24T02:10:51.216568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241196986675283:2106] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.218902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241197685987265:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.218933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.218941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241197685987265:2108] [node 6] ICP22 created new session: [1:7587241197685987295:2108]
2025-12-24T02:10:51.218948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241197685987295:2108] [node 6] ICS09 handshake done sender: [1:7587241197685987267:2109] self: [1:7587241197685987266:0] peer: [6:7587241198658858315:0] socket: 109 qp: -1
2025-12-24T02:10:51.218954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241197685987295:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:51.218973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.219812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858319:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.220258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.220298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241198658858248:2060] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.220372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858321:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.220223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241196986675233:2062] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.219566Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241196039913076:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52657124ns) {Configure:104(52656381ns) {ProcessConfigured:135(52360620ns) {StartInitialHandshake:157(52359517ns) {DropHandshakes:470(545ns) {DropIncomingHandshake:442(102ns) DropOutgoingHandshake:457(100ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:477(1519ns) {SwitchToState:179(67ns)}}}}}
2025-12-24T02:10:51.219630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241196039913080:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.220373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913089:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.220381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241196039913006:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.220391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.220397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241196039913006:2048] [node 1] ICP22 created new session: [3:7587241196039913094:2048]
2025-12-24T02:10:51.220401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241196039913094:2048] [node 1] ICS09 handshake done sender: [3:7587241196039913080:2114] self: [3:7587241196039913087:0] peer: [1:7587241197685987271:0] socket: 126 qp: -1
2025-12-24T02:10:51.220407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241196039913094:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.220425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.223571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241198658858321:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.223637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241198658858277:2085] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.223251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241196545971874:2091] [node 5] ICP35 event processing took too much time PendingConnection:279(26987219ns) {HandleHandshakeStatus:329(26986648ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(128ns) SwitchToState:179(59ns) ProcessPendingSessionEvents:495(16354323ns) {ForwardSessionEventToSession:612(16346027ns) ForwardSessionEventToSession:612(1402ns) ForwardSessionEventToSession:612(677ns) ForwardSessionEventToSession:612(453ns) ForwardSessionEventToSession:612(414ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(512ns) ForwardSessionEventToSession:612(411ns) ForwardSessionEventToSession:612(400ns) ForwardSessionEventToSession:612(963ns)}}}
2025-12-24T02:10:51.223735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241198658858235:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.223748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.223753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241198658858235:2048] [node 1] ICP22 created new session: [6:7587241198658858323:2048]
2025-12-24T02:10:51.223757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241198658858323:2048] [node 1] ICS09 handshake done sender: [6:7587241198658858277:2085] self: [6:7587241198658858315:0] peer: [1:7587241197685987266:0] socket: 110 qp: -1
2025-12-24T02:10:51.223761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241198658858323:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.223776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.223909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241198658858236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.222949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675300:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.222971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241196986675216:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.222985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.222991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241196986675216:2049] [node 2] ICP22 created new session: [5:7587241196986675304:2049]
2025-12-24T02:10:51.222996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241196986675304:2049] [node 2] ICS09 handshake done sender: [5:7587241196986675283:2106] self: [5:7587241196986675295:0] peer: [2:7587241199273203419:0] socket: 118 qp: -1
2025-12-24T02:10:51.223001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241196986675304:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.223008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.223516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241196986675217:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.223762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241196986675240:2066] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.223905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241196986675238:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.223940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.223948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241196986675238:2064] [node 6] ICP22 created new session: [5:7587241196986675307:2064]
2025-12-24T02:10:51.223954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241196986675307:2064] [node 6] ICS09 handshake done sender: [5:7587241196986675240:2066] self: [5:7587241196986675239:0] peer: [6:7587241198658858318:0] socket: 107 qp: -1
2025-12-24T02:10:51.223960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241196986675307:2064] [node 6] ICS10 traffic start
2025-12-24T02:10:51.223967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.225755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858322:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.225950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241198658858276:2084] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.226014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241198658858262:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.226025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.226031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241198658858262:2064] [node 5] ICP22 created new session: [6:7587241198658858327:2064]
2025-12-24T02:10:51.226036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241198658858327:2064] [node 5] ICS09 handshake done sender: [6:7587241198658858276:2084] self: [6:7587241198658858318:0] peer: [5:7587241196986675239:0] socket: 108 qp: -1
2025-12-24T02:10:51.226040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241198658858327:2064] [node 5] ICS10 traffic start
2025-12-24T02:10:51.226045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.227129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199273203436:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.227148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199273203436:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.227305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241198658858265:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.227322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.227329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241198658858265:2067] [node 4] ICP22 created new session: [6:7587241198658858330:2067]
2025-12-24T02:10:51.227336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241198658858330:2067] [node 4] ICS09 handshake done sender: [6:7587241198658858248:2060] self: [6:7587241198658858317:0] peer: [4:7587241196545971880:0] socket: 95 qp: -1
2025-12-24T02:10:51.227343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241198658858330:2067] [node 4] ICS10 traffic start
2025-12-24T02:10:51.227352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.226781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241196986675278:2077] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.226800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.226806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241196986675278:2077] [node 4] ICP22 created new session: [5:7587241196986675311:2077]
2025-12-24T02:10:51.226812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241196986675311:2077] [node 4] ICS09 handshake done sender: [5:7587241196986675233:2062] self: [5:7587241196986675298:0] peer: [4:7587241196545971877:0] socket: 103 qp: -1
2025-12-24T02:10:51.226819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241196986675311:2077] [node 4] ICS10 traffic start
2025-12-24T02:10:51.226836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.228130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241197685987297:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30768
2025-12-24T02:10:51.229319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987297:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.227815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241199273203353:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33063872ns) {HandleHandshakeStatus:329(33062918ns) {DropIncomingHandshake:442(4768ns) DropOutgoingHandshake:457(346ns) SwitchToState:179(254ns) ProcessPendingSessionEvents:495(20632297ns) {ForwardSessionEventToSession:612(20622276ns) ForwardSessionEventToSession:612(1671ns) ForwardSessionEventToSession:612(1211ns) ForwardSessionEventToSession:612(493ns) ForwardSessionEventToSession:612(627ns) ForwardSessionEventToSession:612(1099ns) ForwardSessionEventToSession:612(856ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(559ns)}}}
2025-12-24T02:10:51.227469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241196039913082:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20639393ns) {Configure:104(20638978ns) {ProcessConfigured:135(20328086ns) {StartInitialHandshake:157(20316358ns) {DropHandshakes:470(981ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(816ns) PrepareNewSessionHandshake:477(868ns) {SwitchToState:179(157ns)}} IncomingHandshake:261(5417ns) {DropIncomingHandshake:442(124ns) PrepareNewSessionHandshake:477(142ns) {SwitchToState:179(91ns)} GenerateSessionVirtualId:988(169ns) IssueIncomingHandshakeReply:193(314ns)}}}}
2025-12-24T02:10:51.227708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241196039913100:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57626
2025-12-24T02:10:51.228219Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241196039913007:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17061275ns) {HandleHandshakeStatus:329(17060395ns) {DropIncomingHandshake:442(198ns) DropOutgoingHandshake:457(2177ns) SwitchToState:179(220ns) ProcessPendingSessionEvents:495(11798591ns) {ForwardSessionEventToSession:612(11788966ns) ForwardSessionEventToSession:612(1294ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(447ns) ForwardSessionEventToSession:612(1003ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(486ns) ForwardSessionEventToSession:612(585ns)}}}
2025-12-24T02:10:51.228958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913084:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.228963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241196039913084:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.229018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913100:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.230763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675302:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.231017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241196986675302:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.231066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241196986675215:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.232331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858332:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.232494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199273203436:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.232682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241198658858321:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.232778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241198658858236:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.232796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.232802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241198658858236:2049] [node 2] ICP22 created new session: [6:7587241198658858333:2049]
2025-12-24T02:10:51.232819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241198658858333:2049] [node 2] ICS09 handshake done sender: [6:7587241198658858321:2120] self: [6:7587241198658858325:0] peer: [2:7587241199273203428:0] socket: 155 qp: -1
2025-12-24T02:10:51.232826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241198658858333:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.232833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.233561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199273203427:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.233573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.233578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199273203427:2110] [node 6] ICP22 created new session: [2:7587241199273203454:2110]
2025-12-24T02:10:51.233584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199273203454:2110] [node 6] ICS09 handshake done sender: [2:7587241199273203436:2117] self: [2:7587241199273203428:0] peer: [6:7587241198658858325:0] socket: 146 qp: -1
2025-12-24T02:10:51.233589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199273203454:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:51.233595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.233028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241196545971830:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26278) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241196545971907:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.235005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199273203453:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42598
2025-12-24T02:10:51.235744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241199273203427:2110] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.239490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675314:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.238735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987284:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.238763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241197685987284:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.239649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241197685987284:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.239925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241196986675302:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.240061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241196986675215:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.240083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.240091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241196986675215:2048] [node 1] ICP22 created new session: [5:7587241196986675315:2048]
2025-12-24T02:10:51.240098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241196986675315:2048] [node 1] ICS09 handshake done sender: [5:7587241196986675302:2121] self: [5:7587241196986675313:0] peer: [1:7587241197685987282:0] socket: 152 qp: -1
2025-12-24T02:10:51.240106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241196986675315:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.240128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.247073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858335:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.246949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971917:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.250808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241197685987273:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.250855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.250866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241197685987273:2112] [node 5] ICP22 created new session: [1:7587241197685987300:2112]
2025-12-24T02:10:51.250873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241197685987300:2112] [node 5] ICS09 handshake done sender: [1:7587241197685987284:2117] self: [1:7587241197685987282:0] peer: [5:7587241196986675313:0] socket: 147 qp: -1
2025-12-24T02:10:51.250881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241197685987300:2112] [node 5] ICS10 traffic start
2025-12-24T02:10:51.250896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.251122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241196545971917:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.251181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241196545971829:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.254849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241197685987283:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.254864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241197685987283:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.257431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241196545971829:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12408) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241196545971917:2116] held: no
2025-12-24T02:10:51.255862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241197685987283:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.255956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241197685987279:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.255973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.255981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241197685987279:2115] [node 4] ICP22 created new session: [1:7587241197685987303:2115]
2025-12-24T02:10:51.255987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241197685987303:2115] [node 4] ICS09 handshake done sender: [1:7587241197685987283:2116] self: [1:7587241197685987280:0] peer: [4:7587241196545971918:0] socket: 131 qp: -1
2025-12-24T02:10:51.255994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241197685987303:2115] [node 4] ICS10 traffic start
2025-12-24T02:10:51.256003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.259086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241198658858335:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.259138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241198658858237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.260120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241196986675215:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20030571ns) {HandleHandshakeStatus:329(20029942ns) {DropIncomingHandshake:442(393ns) DropOutgoingHandshake:457(2285ns) SwitchToState:179(384ns) ProcessPendingSessionEvents:495(9314727ns) {ForwardSessionEventToSession:612(9303284ns) ForwardSessionEventToSession:612(2038ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(449ns) ForwardSessionEventToSession:612(413ns) ForwardSessionEventToSession:612(1126ns) ForwardSessionEventToSession:612(453ns) ForwardSessionEventToSession:612(742ns) ForwardSessionEventToSession:612(430ns) ForwardSessionEventToSession:612(1020ns)}}}
2025-12-24T02:10:51.260276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241196986675317:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62303
2025-12-24T02:10:51.260473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241196986675300:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.260344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971922:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.260691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241196986675217:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.260709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.260717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241196986675217:2050] [node 3] ICP22 created new session: [5:7587241196986675318:2050]
2025-12-24T02:10:51.260724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241196986675318:2050] [node 3] ICS09 handshake done sender: [5:7587241196986675300:2119] self: [5:7587241196986675306:0] peer: [3:7587241196039913078:0] socket: 150 qp: -1
2025-12-24T02:10:51.260730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241196986675318:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.260741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.261250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971919:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.259310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913098:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.259322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241196039913098:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.259424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241196039913084:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.260908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241196039913098:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.261026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241196039913082:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.261039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241196039913077:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.261041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.261048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241196039913082:2112] [node 6] ICP22 created new session: [3:7587241196039913105:2112]
2025-12-24T02:10:51.261051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.261054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241196039913105:2112] [node 6] ICS09 handshake done sender: [3:7587241196039913098:2120] self: [3:7587241196039913086:0] peer: [6:7587241198658858336:0] socket: 121 qp: -1
2025-12-24T02:10:51.261058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241196039913077:2111] [node 5] ICP22 created new session: [3:7587241196039913106:2111]
2025-12-24T02:10:51.261060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241196039913105:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:51.261063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241196039913106:2111] [node 5] ICS09 handshake done sender: [3:7587241196039913084:2115] self: [3:7587241196039913078:0] peer: [5:7587241196986675306:0] socket: 137 qp: -1
2025-12-24T02:10:51.261065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.261068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241196039913106:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:51.261074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.262815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241198658858337:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55545
2025-12-24T02:10:51.263055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241198658858335:2123] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.262797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241196545971907:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.262846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971920:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.262906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971921:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.263086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971923:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.263156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241198658858237:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.263175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.263179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241198658858237:2050] [node 3] ICP22 created new session: [6:7587241198658858338:2050]
2025-12-24T02:10:51.263184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241198658858338:2050] [node 3] ICS09 handshake done sender: [6:7587241198658858335:2123] self: [6:7587241198658858336:0] peer: [3:7587241196039913086:0] socket: 124 qp: -1
2025-12-24T02:10:51.263190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241198658858338:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.263199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.264816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241196545971874:2091] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:51.265247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241196545971920:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.265265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241196545971830:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.265274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.265279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241196545971830:2049] [node 2] ICP22 created new session: [4:7587241196545971924:2049]
2025-12-24T02:10:51.265288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241196545971924:2049] [node 2] ICS09 handshake done sender: [4:7587241196545971907:2115] self: [4:7587241196545971908:0] peer: [2:7587241199273203414:0] socket: 112 qp: -1
2025-12-24T02:10:51.265294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241196545971924:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.265323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.265325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241196545971831:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.265615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241196545971917:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.265769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241196039913093:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.265780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241196039913093:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.266795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241196039913093:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.266882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241196039913076:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.266899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.266906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241196039913076:2110] [node 4] ICP22 created new session: [3:7587241196039913110:2110]
2025-12-24T02:10:51.266911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241196039913110:2110] [node 4] ICS09 handshake done sender: [3:7587241196039913093:2119] self: [3:7587241196039913083:0] peer: [4:7587241196545971925:0] socket: 90 qp: -1
2025-12-24T02:10:51.266916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241196039913110:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:51.266932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.271775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241196545971831:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241196545971920:2118] held: no
2025-12-24T02:10:51.271854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241196545971927:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32110
2025-12-24T02:10:51.272079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241196545971920:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.272120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241196545971831:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.272129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.272134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241196545971831:2050] [node 3] ICP22 created new session: [4:7587241196545971928:2050]
2025-12-24T02:10:51.272140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241196545971928:2050] [node 3] ICS09 handshake done sender: [4:7587241196545971920:2118] self: [4:7587241196545971925:0] peer: [3:7587241196039913083:0] socket: 99 qp: -1
2025-12-24T02:10:51.272145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241196545971928:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.272151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.274864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241196545971829:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.274878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.274886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241196545971829:2048] [node 1] ICP22 created new session: [4:7587241196545971930:2048]
2025-12-24T02:10:51.274892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241196545971930:2048] [node 1] ICS09 handshake done sender: [4:7587241196545971917:2116] self: [4:7587241196545971918:0] peer: [1:7587241197685987280:0] socket: 133 qp: -1
2025-12-24T02:10:51.274902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241196545971930:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.274917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.279167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:51.279253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241199273203446:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-12-24T02:10:51.279261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241199273203352:2048] [node 1] ICP30 unregister session Session# [2:7587241199273203446:2048] VirtualId# [2:7587241199273203438:0]
2025-12-24T02:10:51.279268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241199273203446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.279296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.279316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.280988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:51.281068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241196039913094:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-12-24T02:10:51.281074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241196039913006:2048] [node 1] ICP30 unregister session Session# [3:7587241196039913094:2048] VirtualId# [3:7587241196039913087:0]
2025-12-24T02:10:51.281081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241196039913094:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.281114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.281134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.283693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:51.283777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241198658858323:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:10:51.283784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241198658858235:2048] [node 1] ICP30 unregister session Session# [6:7587241198658858323:2048] VirtualId# [6:7587241198658858315:0]
2025-12-24T02:10:51.283790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241198658858323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.283826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.283848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.286762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:51.286844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241196986675315:2048] [node 1] ICS07 socket disconnect 152 reason# EndOfStream
2025-12-24T02:10:51.286849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241196986675315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.286896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.286917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241196986675315:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:51.287529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241196986675215:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12408) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.287539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241196986675315:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.287543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241196986675215:2048] [node 1] ICP30 unregister session Session# [5:7587241196986675315:2048] VirtualId# [5:7587241196986675313:0]
2025-12-24T02:10:51.287553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.287559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241196986675215:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12408) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.287564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12408) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.287121Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:51.287179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241196545971930:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-12-24T02:10:51.287186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241196545971829:2048] [node 1] ICP30 unregister session Session# [4:7587241196545971930:2048] VirtualId# [4:7587241196545971918:0]
2025-12-24T02:10:51.287190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241196545971930:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.287198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.287222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.296002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:51.296101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241196039913091:2049] [node 2] ICS07 socket disconnect 119 reason# EndOfStream
2025-12-24T02:10:51.296114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241196039913091:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.296174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.296200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241196039913091:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:51.298760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241196986675215:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:51.300003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:51.300093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241198658858333:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-12-24T02:10:51.300101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241198658858236:2049] [node 2] ICP30 unregister session Session# [6:7587241198658858333:2049] VirtualId# [6:7587241198658858325:0]
2025-12-24T02:10:51.300106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241198658858333:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.300148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.300170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.300372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:51.300424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241196986675304:2049] [node 2] ICS01 socket: 118 reason# EndOfStream
2025-12-24T02:10:51.300428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241196986675216:2049] [node 2] ICP30 unregister session Session# [5:7587241196986675304:2049] VirtualId# [5:7587241196986675295:0]
2025-12-24T02:10:51.300432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241196986675304:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.300453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.300469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.301060Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241196039913007:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26278) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:51.301074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241196039913091:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:51.301389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241196039913007:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26278) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.301404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241196039913091:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.301409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241196039913007:2049] [node 2] ICP30 unregister session Session# [3:7587241196039913091:2049] VirtualId# [3:7587241196039913081:0]
2025-12-24T02:10:51.301424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.301429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241196039913007:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.301432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26278) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.300198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:51.300275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241196545971924:2049] [node 2] ICS01 socket: 112 reason# EndOfStream
2025-12-24T02:10:51.300281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241196545971830:2049] [node 2] ICP30 unregister session Session# [4:7587241196545971924:2049] VirtualId# [4:7587241196545971908:0]
2025-12-24T02:10:51.300286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241196545971924:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.300319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.300338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.318874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:51.318951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241198658858338:2050] [node 3] ICS01 socket: 124 reason# EndOfStream
2025-12-24T02:10:51.318957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241198658858237:2050] [node 3] ICP30 unregister session Session# [6:7587241198658858338:2050] VirtualId# [6:7587241198658858336:0]
2025-12-24T02:10:51.318969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241198658858338:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.319017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.319047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.320708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:51.320780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241196986675318:2050] [node 3] ICS07 socket disconnect 150 reason# EndOfStream
2025-12-24T02:10:51.320785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241196986675318:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.320827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.320845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241196986675318:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:51.321235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241196986675217:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6881) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.321245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241196986675318:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.321249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241196986675217:2050] [node 3] ICP30 unregister session Session# [5:7587241196986675318:2050] VirtualId# [5:7587241196986675306:0]
2025-12-24T02:10:51.321259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.321264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241196986675217:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.321267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6881) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.332357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241196986675217:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:51.346776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:51.346879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241198658858330:2067] [node 4] ICS07 socket disconnect 95 reason# EndOfStream
2025-12-24T02:10:51.346885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241198658858330:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.346953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:51.346974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241198658858330:2067] [node 4] ICS15 start handshake
2025-12-24T02:10:51.351407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241198658858265:2067] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28285) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.351427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241198658858330:2067] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.351434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241198658858265:2067] [node 4] ICP30 unregister session Session# [6:7587241198658858330:2067] VirtualId# [6:7587241198658858317:0]
2025-12-24T02:10:51.351453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:51.351459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241198658858265:2067] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.351463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28285) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.362598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241198658858265:2067] [node 4] ICP33 wake up from error state
2025-12-24T02:10:51.363990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:51.364084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241198658858327:2064] [node 5] ICS01 socket: 108 reason# EndOfStream
2025-12-24T02:10:51.364093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241198658858262:2064] [node 5] ICP30 unregister session Session# [6:7587241198658858327:2064] VirtualId# [6:7587241198658858318:0]
2025-12-24T02:10:51.364099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241198658858327:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.364146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:51.364170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:51.377275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195072722809:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.377290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195072722810:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.377402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241196747647566:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:51.377414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241196747647567:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.381177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241197734951343:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.381192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241197734951344:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.381196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241197734951345:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.378774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199118134455:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.378790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199118134456:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.384290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241196543356545:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.384308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241196543356546:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.384313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241196543356547:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.379542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241195583831693:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.379548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241195583831694:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.379552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241195583831695:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.393506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241196543356559:2060] [node 5] ICP01 ready to work
2025-12-24T02:10:51.393766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199118134512:2103] [node 4] ICP01 ready to work
2025-12-24T02:10:51.395014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241196747647625:2100] [node 4] ICP01 ready to work
2025-12-24T02:10:51.395114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241196747647627:2102] [node 5] ICP01 ready to work
2025-12-24T02:10:51.395180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241196747647628:2103] [node 6] ICP01 ready to work
2025-12-24T02:10:51.396115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195072722869:2106] [node 5] ICP01 ready to work
2025-12-24T02:10:51.397289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195072722879:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:51.396004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241196543356576:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.399050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195072722885:2113] [node 6] ICP01 ready to work
2025-12-24T02:10:51.399087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199118134519:2104] [node 5] ICP01 ready to work
2025-12-24T02:10:51.399164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199118134520:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:51.399468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831754:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.412365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241197734951408:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.414522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241195583831760:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:51.414605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241195583831761:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.414901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.412486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241197734951409:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:51.412542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951406:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.412611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951407:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.417043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831754:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.418018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951406:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.416171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241196747647625:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20654377ns) {Configure:104(20654224ns) {ProcessConfigured:135(20454993ns) {StartInitialHandshake:157(20454811ns) {DropHandshakes:470(147ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(874ns) {SwitchToState:179(661ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.432493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241199118134520:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20335516ns) {Configure:104(20335031ns) {ProcessConfigured:135(15478342ns) {StartInitialHandshake:157(15477047ns) {DropHandshakes:470(425ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(597ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(331ns)}}}}}
2025-12-24T02:10:51.432570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134523:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.432595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134524:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.432610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134525:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.433421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.433444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.446851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134525:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.449407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831774:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.449712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241196543356623:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.449747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241196543356559:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.450092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134535:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.450165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134536:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.450198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134518:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.450207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199118134518:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.451860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241196747647643:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31548
2025-12-24T02:10:51.451886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241196747647644:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31548
2025-12-24T02:10:51.451928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241196747647645:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31548
2025-12-24T02:10:51.452108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647645:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.452974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951411:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.452989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241197734951411:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.453332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199118134518:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.453656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241197734951411:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.454317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199118134512:2103] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.454340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.454348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199118134512:2103] [node 4] ICP22 created new session: [2:7587241199118134543:2103]
2025-12-24T02:10:51.454363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199118134543:2103] [node 4] ICS09 handshake done sender: [2:7587241199118134518:2108] self: [2:7587241199118134513:0] peer: [4:7587241195583831775:0] socket: 126 qp: -1
2025-12-24T02:10:51.454369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199118134543:2103] [node 4] ICS10 traffic start
2025-12-24T02:10:51.454379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.450085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831774:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.450104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241195583831694:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.450763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831776:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.452142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831777:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.454168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241195583831774:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.454266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241195583831694:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.454278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.454282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241195583831694:2049] [node 2] ICP22 created new session: [4:7587241195583831778:2049]
2025-12-24T02:10:51.454286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241195583831778:2049] [node 2] ICS09 handshake done sender: [4:7587241195583831774:2117] self: [4:7587241195583831775:0] peer: [2:7587241199118134513:0] socket: 130 qp: -1
2025-12-24T02:10:51.454292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241195583831778:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.454298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.454947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241197734951408:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.454964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.454970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241197734951408:2107] [node 6] ICP22 created new session: [5:7587241197734951425:2107]
2025-12-24T02:10:51.454975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241197734951425:2107] [node 6] ICS09 handshake done sender: [5:7587241197734951411:2112] self: [5:7587241197734951410:0] peer: [6:7587241196543356624:0] socket: 111 qp: -1
2025-12-24T02:10:51.454979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241197734951425:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:51.454985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.455381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356625:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.455576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241196543356623:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.455653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241196543356559:2060] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.455664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.455669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241196543356559:2060] [node 5] ICP22 created new session: [6:7587241196543356626:2060]
2025-12-24T02:10:51.455673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241196543356626:2060] [node 5] ICS09 handshake done sender: [6:7587241196543356623:2115] self: [6:7587241196543356624:0] peer: [5:7587241197734951410:0] socket: 114 qp: -1
2025-12-24T02:10:51.455678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241196543356626:2060] [node 5] ICS10 traffic start
2025-12-24T02:10:51.455684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.456475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241196747647646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31548
2025-12-24T02:10:51.456565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647644:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.456785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134523:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.456781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647646:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.457258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951427:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.457977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241199118134455:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.458603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722888:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.458669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722889:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.458544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951407:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.458571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241197734951345:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.460066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.460093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951431:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.461240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951427:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.461272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241197734951344:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.461370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134531:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.461386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199118134531:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.461902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.462045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951431:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.462073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241197734951409:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.462173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831767:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.462184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241195583831767:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.462221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199118134531:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.462804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241195583831767:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.458756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722871:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.458765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195072722871:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.459165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722890:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.459218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.464283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199118134519:2104] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.464309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.464316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199118134519:2104] [node 5] ICP22 created new session: [2:7587241199118134548:2104]
2025-12-24T02:10:51.464323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199118134548:2104] [node 5] ICS09 handshake done sender: [2:7587241199118134531:2116] self: [2:7587241199118134530:0] peer: [5:7587241197734951432:0] socket: 97 qp: -1
2025-12-24T02:10:51.464331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199118134548:2104] [node 5] ICS10 traffic start
2025-12-24T02:10:51.464339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.464624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241195583831760:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.464640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.464650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241195583831760:2109] [node 5] ICP22 created new session: [4:7587241195583831782:2109]
2025-12-24T02:10:51.464656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241195583831782:2109] [node 5] ICS09 handshake done sender: [4:7587241195583831767:2115] self: [4:7587241195583831766:0] peer: [5:7587241197734951434:0] socket: 116 qp: -1
2025-12-24T02:10:51.464663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241195583831782:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:51.464670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.464212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647618:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.464228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241196747647618:2098] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.464626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647643:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.464944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722891:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.465007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195072722871:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.465030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241197734951427:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.465127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241196747647618:2098] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.465097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241197734951344:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.465114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.465121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241197734951344:2049] [node 2] ICP22 created new session: [5:7587241197734951436:2049]
2025-12-24T02:10:51.465126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241197734951436:2049] [node 2] ICS09 handshake done sender: [5:7587241197734951427:2117] self: [5:7587241197734951432:0] peer: [2:7587241199118134530:0] socket: 98 qp: -1
2025-12-24T02:10:51.465132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241197734951436:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.465137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.465218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722888:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.465298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831777:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.465337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241195583831693:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.465288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722889:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.465365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722890:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.465379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195072722869:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.465387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.465395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195072722869:2106] [node 5] ICP22 created new session: [3:7587241195072722894:2106]
2025-12-24T02:10:51.465400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195072722894:2106] [node 5] ICS09 handshake done sender: [3:7587241195072722871:2107] self: [3:7587241195072722870:0] peer: [5:7587241197734951429:0] socket: 100 qp: -1
2025-12-24T02:10:51.465405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195072722894:2106] [node 5] ICS10 traffic start
2025-12-24T02:10:51.465412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.465544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241195072722869:2106] [node 5] ICP18 handshake (actor [3:7587241195072722891:2118]) is too old
2025-12-24T02:10:51.465570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241195072722810:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.465579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241195072722809:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.465599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722893:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.466109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356629:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.466247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647638:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.466256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241196747647638:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.466843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241196747647566:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.466857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.466862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241196747647566:2048] [node 2] ICP22 created new session: [1:7587241196747647654:2048]
2025-12-24T02:10:51.466868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241196747647654:2048] [node 2] ICS09 handshake done sender: [1:7587241196747647618:2098] self: [1:7587241196747647617:0] peer: [2:7587241199118134546:0] socket: 96 qp: -1
2025-12-24T02:10:51.466874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241196747647654:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:51.466881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.467562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134524:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.467716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199118134550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42769
2025-12-24T02:10:51.467861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199118134523:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.472060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241199118134455:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.472091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.472101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199118134455:2048] [node 1] ICP22 created new session: [2:7587241199118134553:2048]
2025-12-24T02:10:51.472108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199118134553:2048] [node 1] ICS09 handshake done sender: [2:7587241199118134523:2111] self: [2:7587241199118134546:0] peer: [1:7587241196747647617:0] socket: 105 qp: -1
2025-12-24T02:10:51.472117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199118134553:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.472129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.473760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831786:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.473786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831787:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.474074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356630:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.474825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647631:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.474838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241196747647631:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.475228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241196747647638:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.475395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722893:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.476091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241195583831777:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.476196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241195583831693:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.476214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.476221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241195583831693:2048] [node 1] ICP22 created new session: [4:7587241195583831788:2048]
2025-12-24T02:10:51.476228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241195583831788:2048] [node 1] ICS09 handshake done sender: [4:7587241195583831777:2119] self: [4:7587241195583831785:0] peer: [1:7587241196747647632:0] socket: 136 qp: -1
2025-12-24T02:10:51.476234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241195583831788:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.476244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.476711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241196747647625:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.476733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.476739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241196747647625:2100] [node 4] ICP22 created new session: [1:7587241196747647659:2100]
2025-12-24T02:10:51.476745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241196747647659:2100] [node 4] ICS09 handshake done sender: [1:7587241196747647638:2112] self: [1:7587241196747647632:0] peer: [4:7587241195583831785:0] socket: 135 qp: -1
2025-12-24T02:10:51.476750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241196747647659:2100] [node 4] ICS10 traffic start
2025-12-24T02:10:51.476758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.478643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722901:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.478765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241196747647631:2106] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.478862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241197734951345:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241197734951407:2111] held: no
2025-12-24T02:10:51.478921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951435:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.478990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241196543356629:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.478956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195072722890:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.479018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241195072722809:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.479042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.479050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195072722809:2048] [node 1] ICP22 created new session: [3:7587241195072722902:2048]
2025-12-24T02:10:51.479055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195072722902:2048] [node 1] ICS09 handshake done sender: [3:7587241195072722890:2117] self: [3:7587241195072722898:0] peer: [1:7587241196747647630:0] socket: 139 qp: -1
2025-12-24T02:10:51.479062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195072722902:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.479069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.479129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241196543356576:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.479154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356631:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.479524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831769:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.479534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241195583831769:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.479306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241199118134455:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:51.481072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241197734951431:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.481146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241197734951407:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.481224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951438:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.481269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241197734951409:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.481278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.481283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241197734951409:2108] [node 4] ICP22 created new session: [5:7587241197734951440:2108]
2025-12-24T02:10:51.481288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241197734951440:2108] [node 4] ICS09 handshake done sender: [5:7587241197734951431:2119] self: [5:7587241197734951434:0] peer: [4:7587241195583831766:0] socket: 145 qp: -1
2025-12-24T02:10:51.481293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241197734951440:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:51.481300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.481456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241197734951345:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.481462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.481467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241197734951345:2050] [node 3] ICP22 created new session: [5:7587241197734951442:2050]
2025-12-24T02:10:51.481471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241197734951442:2050] [node 3] ICS09 handshake done sender: [5:7587241197734951407:2111] self: [5:7587241197734951429:0] peer: [3:7587241195072722870:0] socket: 109 qp: -1
2025-12-24T02:10:51.481475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241197734951442:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.481478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.482892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356633:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.483578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241196543356629:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.483632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241196543356576:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.483642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.483654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241196543356576:2063] [node 4] ICP22 created new session: [6:7587241196543356634:2063]
2025-12-24T02:10:51.483659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241196543356634:2063] [node 4] ICS09 handshake done sender: [6:7587241196543356629:2117] self: [6:7587241196543356632:0] peer: [4:7587241195583831768:0] socket: 154 qp: -1
2025-12-24T02:10:51.483664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241196543356634:2063] [node 4] ICS10 traffic start
2025-12-24T02:10:51.483671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.483179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831791:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
2025-12-24T02:10:51.483417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241195583831769:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.486442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134527:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.486467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199118134527:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.491010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199118134527:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.491525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241196543356630:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.490813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241195583831761:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.490844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.490853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241195583831761:2111] [node 6] ICP22 created new session: [4:7587241195583831792:2111]
2025-12-24T02:10:51.490860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241195583831792:2111] [node 6] ICS09 handshake done sender: [4:7587241195583831769:2116] self: [4:7587241195583831768:0] peer: [6:7587241196543356632:0] socket: 151 qp: -1
2025-12-24T02:10:51.490868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241195583831792:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:51.490880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.491655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241196543356546:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.491996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241196747647567:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.492014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.492020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241196747647567:2049] [node 3] ICP22 created new session: [1:7587241196747647662:2049]
2025-12-24T02:10:51.492025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241196747647662:2049] [node 3] ICS09 handshake done sender: [1:7587241196747647631:2106] self: [1:7587241196747647630:0] peer: [3:7587241195072722898:0] socket: 134 qp: -1
2025-12-24T02:10:51.492030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241196747647662:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.492037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.492392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241197734951438:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.492422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241197734951343:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.495004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199118134533:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.495016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199118134533:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.496190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199118134456:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.496210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.496216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199118134456:2049] [node 3] ICP22 created new session: [2:7587241199118134556:2049]
2025-12-24T02:10:51.496223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199118134556:2049] [node 3] ICS09 handshake done sender: [2:7587241199118134527:2114] self: [2:7587241199118134526:0] peer: [3:7587241195072722897:0] socket: 127 qp: -1
2025-12-24T02:10:51.496229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199118134556:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.496238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.496013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241195583831786:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.496048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241195583831695:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.494976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195072722904:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6955
2025-12-24T02:10:51.499865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722881:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.499881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195072722881:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.500896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195072722889:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.501375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647640:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.501388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241196747647640:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.502594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241196747647664:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31548
2025-12-24T02:10:51.502744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241195072722810:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.502765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.502883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195072722810:2049] [node 2] ICP22 created new session: [3:7587241195072722907:2049]
2025-12-24T02:10:51.502898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195072722907:2049] [node 2] ICS09 handshake done sender: [3:7587241195072722889:2116] self: [3:7587241195072722897:0] peer: [2:7587241199118134526:0] socket: 138 qp: -1
2025-12-24T02:10:51.502906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195072722907:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.502917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.504142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356638:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.504295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241197734951445:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49759
2025-12-24T02:10:51.504342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241196747647640:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.504429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241196543356631:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.504470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241196543356545:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.504708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241197734951438:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.504796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241197734951343:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.504813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.504819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241197734951343:2048] [node 1] ICP22 created new session: [5:7587241197734951446:2048]
2025-12-24T02:10:51.504826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241197734951446:2048] [node 1] ICS09 handshake done sender: [5:7587241197734951438:2122] self: [5:7587241197734951444:0] peer: [1:7587241196747647639:0] socket: 128 qp: -1
2025-12-24T02:10:51.504832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241197734951446:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.504840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.505755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241195583831795:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.506789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356640:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.506913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199118134533:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.507100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241196543356630:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.507172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241196543356546:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.507187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.507193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241196543356546:2049] [node 2] ICP22 created new session: [6:7587241196543356641:2049]
2025-12-24T02:10:51.507200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241196543356641:2049] [node 2] ICS09 handshake done sender: [6:7587241196543356630:2118] self: [6:7587241196543356636:0] peer: [2:7587241199118134532:0] socket: 133 qp: -1
2025-12-24T02:10:51.507206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241196543356641:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.507214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.508035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199118134520:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.508056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.508062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199118134520:2107] [node 6] ICP22 created new session: [2:7587241199118134559:2107]
2025-12-24T02:10:51.508067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199118134559:2107] [node 6] ICS09 handshake done sender: [2:7587241199118134533:2117] self: [2:7587241199118134532:0] peer: [6:7587241196543356636:0] socket: 155 qp: -1
2025-12-24T02:10:51.508073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199118134559:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:51.508081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.510822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647642:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.510836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241196747647642:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.511505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241196747647627:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.511521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.511530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241196747647627:2102] [node 5] ICP22 created new session: [1:7587241196747647666:2102]
2025-12-24T02:10:51.511537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241196747647666:2102] [node 5] ICS09 handshake done sender: [1:7587241196747647640:2113] self: [1:7587241196747647639:0] peer: [5:7587241197734951444:0] socket: 159 qp: -1
2025-12-24T02:10:51.511544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241196747647666:2102] [node 5] ICS10 traffic start
2025-12-24T02:10:51.511553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.512832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195072722881:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.513069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241195583831786:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.513152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241195583831695:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.513168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.513174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241195583831695:2050] [node 3] ICP22 created new session: [4:7587241195583831796:2050]
2025-12-24T02:10:51.513180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241195583831796:2050] [node 3] ICS09 handshake done sender: [4:7587241195583831786:2120] self: [4:7587241195583831794:0] peer: [3:7587241195072722880:0] socket: 157 qp: -1
2025-12-24T02:10:51.513186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241195583831796:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.513194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.515332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241196747647664:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.515794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356643:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.515911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241196747647642:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.516031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241196747647628:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.516044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.516051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241196747647628:2103] [node 6] ICP22 created new session: [1:7587241196747647670:2103]
2025-12-24T02:10:51.516056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241196747647670:2103] [node 6] ICS09 handshake done sender: [1:7587241196747647642:2114] self: [1:7587241196747647641:0] peer: [6:7587241196543356639:0] socket: 122 qp: -1
2025-12-24T02:10:51.516063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241196747647670:2103] [node 6] ICS10 traffic start
2025-12-24T02:10:51.516071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.518819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195072722879:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.518847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.518854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195072722879:2109] [node 4] ICP22 created new session: [3:7587241195072722909:2109]
2025-12-24T02:10:51.518860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195072722909:2109] [node 4] ICS09 handshake done sender: [3:7587241195072722881:2111] self: [3:7587241195072722880:0] peer: [4:7587241195583831794:0] socket: 142 qp: -1
2025-12-24T02:10:51.518867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195072722909:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:51.518875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.525435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241196543356638:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.525468Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241196543356545:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15483) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241196543356631:2119] held: no
2025-12-24T02:10:51.525513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241196543356631:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.525618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241196543356545:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.525634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.525645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241196543356545:2048] [node 1] ICP22 created new session: [6:7587241196543356644:2048]
2025-12-24T02:10:51.525651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241196543356644:2048] [node 1] ICS09 handshake done sender: [6:7587241196543356631:2119] self: [6:7587241196543356639:0] peer: [1:7587241196747647641:0] socket: 160 qp: -1
2025-12-24T02:10:51.525658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241196543356644:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.525665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.525775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241196543356547:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.525986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195072722887:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.525998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195072722887:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.526930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195072722887:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.526985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241196543356647:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58874
2025-12-24T02:10:51.527160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241196543356638:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.527022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195072722885:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.527054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.527059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195072722885:2113] [node 6] ICP22 created new session: [3:7587241195072722912:2113]
2025-12-24T02:10:51.527064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195072722912:2113] [node 6] ICS09 handshake done sender: [3:7587241195072722887:2114] self: [3:7587241195072722886:0] peer: [6:7587241196543356646:0] socket: 120 qp: -1
2025-12-24T02:10:51.527069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195072722912:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:51.527077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.533835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241196543356547:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.533864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.533875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241196543356547:2050] [node 3] ICP22 created new session: [6:7587241196543356648:2050]
2025-12-24T02:10:51.533882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241196543356648:2050] [node 3] ICS09 handshake done sender: [6:7587241196543356638:2121] self: [6:7587241196543356646:0] peer: [3:7587241195072722886:0] socket: 104 qp: -1
2025-12-24T02:10:51.533893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241196543356648:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.533904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.533999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241196543356547:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:51.544926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:51.545025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241199118134553:2048] [node 1] ICS01 socket: 105 reason# EndOfStream
2025-12-24T02:10:51.545034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241199118134455:2048] [node 1] ICP30 unregister session Session# [2:7587241199118134553:2048] VirtualId# [2:7587241199118134546:0]
2025-12-24T02:10:51.545041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241199118134553:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.545081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.545103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.558779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:51.558877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241195072722902:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-12-24T02:10:51.558888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241195072722809:2048] [node 1] ICP30 unregister session Session# [3:7587241195072722902:2048] VirtualId# [3:7587241195072722898:0]
2025-12-24T02:10:51.558896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241195072722902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.558948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.558971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.578871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:51.578965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241195583831788:2048] [node 1] ICS07 socket disconnect 136 reason# EndOfStream
2025-12-24T02:10:51.578971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241195583831788:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.579017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.579053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241195583831788:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:51.579375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241195583831693:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15483) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.579389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241195583831788:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.579394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241195583831693:2048] [node 1] ICP30 unregister session Session# [4:7587241195583831788:2048] VirtualId# [4:7587241195583831785:0]
2025-12-24T02:10:51.579408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.579414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241195583831693:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15483) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.579417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15483) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.579334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:51.579434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241196543356644:2048] [node 1] ICS01 socket: 160 reason# EndOfStream
2025-12-24T02:10:51.579442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241196543356545:2048] [node 1] ICP30 unregister session Session# [6:7587241196543356644:2048] VirtualId# [6:7587241196543356639:0]
2025-12-24T02:10:51.579449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241196543356644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.579511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.579543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.579587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:51.579654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241197734951446:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-12-24T02:10:51.579660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241197734951343:2048] [node 1] ICP30 unregister session Session# [5:7587241197734951446:2048] VirtualId# [5:7587241197734951444:0]
2025-12-24T02:10:51.579664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241197734951446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.579692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.579711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.582275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:51.582365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241195072722907:2049] [node 2] ICS01 socket: 138 reason# EndOfStream
2025-12-24T02:10:51.582376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241195072722810:2049] [node 2] ICP30 unregister session Session# [3:7587241195072722907:2049] VirtualId# [3:7587241195072722897:0]
2025-12-24T02:10:51.582382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241195072722907:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.582431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.582457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.586806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:51.586893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241195583831778:2049] [node 2] ICS01 socket: 130 reason# EndOfStream
2025-12-24T02:10:51.586901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241195583831694:2049] [node 2] ICP30 unregister session Session# [4:7587241195583831778:2049] VirtualId# [4:7587241195583831775:0]
2025-12-24T02:10:51.586907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241195583831778:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.586958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.586981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.587016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:51.587091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241197734951436:2049] [node 2] ICS07 socket disconnect 98 reason# EndOfStream
2025-12-24T02:10:51.587095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241197734951436:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.587124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.587144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241197734951436:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:51.587333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:51.587406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241196543356641:2049] [node 2] ICS01 socket: 133 reason# EndOfStream
2025-12-24T02:10:51.587411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241196543356546:2049] [node 2] ICP30 unregister session Session# [6:7587241196543356641:2049] VirtualId# [6:7587241196543356636:0]
2025-12-24T02:10:51.587416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241196543356641:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.587443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.587492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.587948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241197734951344:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.587959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241197734951436:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.587963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241197734951344:2049] [node 2] ICP30 unregister session Session# [5:7587241197734951436:2049] VirtualId# [5:7587241197734951432:0]
2025-12-24T02:10:51.587974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.587978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241197734951344:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.587981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.594751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241195583831693:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:51.597391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241197734951344:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:51.598854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:51.598965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241196543356648:2050] [node 3] ICS01 socket: 104 reason# EndOfStream
2025-12-24T02:10:51.598973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241196543356547:2050] [node 3] ICP30 unregister session Session# [6:7587241196543356648:2050] VirtualId# [6:7587241196543356646:0]
2025-12-24T02:10:51.598979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241196543356648:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.599056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.599081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.602778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:51.602879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241197734951442:2050] [node 3] ICS01 socket: 109 reason# EndOfStream
2025-12-24T02:10:51.602890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241197734951345:2050] [node 3] ICP30 unregister session Session# [5:7587241197734951442:2050] VirtualId# [5:7587241197734951429:0]
2025-12-24T02:10:51.602897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241197734951442:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.602973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.603003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.611863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:51.611949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241196543356634:2063] [node 4] ICS01 socket: 154 reason# EndOfStream
2025-12-24T02:10:51.611958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241196543356576:2063] [node 4] ICP30 unregister session Session# [6:7587241196543356634:2063] VirtualId# [6:7587241196543356632:0]
2025-12-24T02:10:51.611965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241196543356634:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.612007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:51.612029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:51.624422Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:51.624515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241196543356626:2060] [node 5] ICS01 socket: 114 reason# EndOfStream
2025-12-24T02:10:51.624524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241196543356559:2060] [node 5] ICP30 unregister session Session# [6:7587241196543356626:2060] VirtualId# [6:7587241196543356624:0]
2025-12-24T02:10:51.624530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241196543356626:2060] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.624576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:51.624599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:51.648872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198486983659:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:51.648886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198486983660:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.650862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198740158378:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.650960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198740158379:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.653545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195511576527:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.653553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195511576528:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.655173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241198238064897:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.655181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241198238064898:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.655185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241198238064899:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.656951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241198332964464:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.656961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241198332964465:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.656968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241198332964466:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.671330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195511576579:2096] [node 4] ICP01 ready to work
2025-12-24T02:10:51.671544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195511576580:2098] [node 5] ICP01 ready to work
2025-12-24T02:10:51.674430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198740158441:2106] [node 4] ICP01 ready to work
2025-12-24T02:10:51.674574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198740158442:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:51.674666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198740158443:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:51.673602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198486983723:2105] [node 6] ICP01 ready to work
2025-12-24T02:10:51.674290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198486983721:2103] [node 4] ICP01 ready to work
2025-12-24T02:10:51.674390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198486983725:2104] [node 5] ICP01 ready to work
2025-12-24T02:10:51.658797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241197856790610:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.658808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241197856790611:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.658814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241197856790612:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.671311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241197856790672:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:51.690833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241195511576580:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19217709ns) {Configure:104(19216788ns) {ProcessConfigured:135(14730300ns) {StartInitialHandshake:157(14729047ns) {DropHandshakes:470(396ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(125ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(2072ns) {SwitchToState:179(992ns)}}}}}
2025-12-24T02:10:51.691063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241195511576595:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.702812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241198486983723:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29128412ns) {Configure:104(29127572ns) {ProcessConfigured:135(28853261ns) {StartInitialHandshake:157(28852504ns) {DropHandshakes:470(417ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(91ns) PrepareNewSessionHandshake:477(1541ns) {SwitchToState:179(764ns)}}}}}
2025-12-24T02:10:51.703167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241198486983734:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56584
2025-12-24T02:10:51.703396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241198486983725:2104] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22331071ns) {Configure:104(22329815ns) {ProcessConfigured:135(22016970ns) {StartInitialHandshake:157(22016636ns) {DropHandshakes:470(791ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(653ns) {SwitchToState:179(449ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.703819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241198332964531:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:51.703920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241198332964532:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:51.704939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964528:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.704997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964529:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.705015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964530:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.712378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241198740158379:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37652923ns) {Configure:104(37652302ns) {ProcessConfigured:135(37380683ns) {StartInitialHandshake:157(37380201ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(1122ns) {SwitchToState:179(409ns)}}}}}
2025-12-24T02:10:51.712560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158457:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
2025-12-24T02:10:51.712575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158458:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
2025-12-24T02:10:51.712586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158459:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.714892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158460:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.714881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241195511576579:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43475819ns) {Configure:104(43474762ns) {ProcessConfigured:135(38884421ns) {StartInitialHandshake:157(38883462ns) {DropHandshakes:470(541ns) {DropIncomingHandshake:442(134ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1106ns) {SwitchToState:179(510ns)}}}}}
2025-12-24T02:10:51.716317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
2025-12-24T02:10:51.716332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576606:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
2025-12-24T02:10:51.717434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241198486983739:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56584
2025-12-24T02:10:51.726119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241197856790685:2113] [node 4] ICP01 ready to work
2025-12-24T02:10:51.730732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241198238064898:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44817060ns) {Configure:104(44816094ns) {ProcessConfigured:135(43338851ns) {StartInitialHandshake:157(43337929ns) {DropHandshakes:470(698ns) {DropIncomingHandshake:442(130ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(365ns) PrepareNewSessionHandshake:477(1589ns) {SwitchToState:179(1143ns)}}}}}
2025-12-24T02:10:51.730942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241198238064964:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:51.731067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241198238064965:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.739144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158457:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.739280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158461:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
2025-12-24T02:10:51.740696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241198486983740:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56584
2025-12-24T02:10:51.740726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241198486983741:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56584
2025-12-24T02:10:51.735963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064969:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.736128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064968:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.741239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576605:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.739955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790687:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.741285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241195511576527:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.741408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241198332964464:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42613055ns) {Configure:104(42611951ns) {ProcessConfigured:135(42161500ns) {StartInitialHandshake:157(42160819ns) {DropHandshakes:470(1244ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(883ns) PrepareNewSessionHandshake:477(1399ns) {SwitchToState:179(985ns)}}}}}
2025-12-24T02:10:51.742503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964530:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.742533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964529:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.742555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964528:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.742596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241198332964466:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.742609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241198332964465:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.744178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983739:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.747263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983722:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.747281Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241198486983722:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.747338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158450:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.747353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241198740158450:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.747367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576594:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.747381Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195511576594:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.747018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241197856790685:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20819428ns) {Configure:104(20818372ns) {ProcessConfigured:135(20522403ns) {StartInitialHandshake:157(20521795ns) {DropHandshakes:470(816ns) {DropIncomingHandshake:442(168ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(997ns) {SwitchToState:179(371ns)}}}}}
2025-12-24T02:10:51.750844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983734:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.750891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983740:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.751554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064968:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.751586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064969:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.751611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241198238064898:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-12-24T02:10:51.751622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241198238064897:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-12-24T02:10:51.752059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.750977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158458:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.751013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158460:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.751052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241198740158378:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.754822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576606:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.758731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158448:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.758758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241198740158448:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.758855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983720:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.758870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241198486983720:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.759026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064977:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.759237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983728:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.759250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241198486983728:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.762998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241198486983722:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.764331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241198740158450:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.764490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158459:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.764505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241198740158442:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.764514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.764522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198740158442:2107] [node 5] ICP22 created new session: [2:7587241198740158466:2107]
2025-12-24T02:10:51.764528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241198740158466:2107] [node 5] ICS09 handshake done sender: [2:7587241198740158450:2112] self: [2:7587241198740158449:0] peer: [5:7587241198332964546:0] socket: 105 qp: -1
2025-12-24T02:10:51.764537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241198740158466:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:51.764567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.764758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241198740158442:2107] [node 5] ICP18 handshake (actor [2:7587241198740158459:2117]) is too old
2025-12-24T02:10:51.766588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195511576594:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.763270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241198486983728:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.763401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198486983660:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.763422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.763434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198486983660:2049] [node 3] ICP22 created new session: [1:7587241198486983747:2049]
2025-12-24T02:10:51.763507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241198486983747:2049] [node 3] ICS09 handshake done sender: [1:7587241198486983722:2107] self: [1:7587241198486983719:0] peer: [3:7587241195511576608:0] socket: 115 qp: -1
2025-12-24T02:10:51.763515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241198486983747:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.763539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.766739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241198332964465:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241198332964529:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.767733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064981:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.766751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195511576580:2098] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.766773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.766780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195511576580:2098] [node 5] ICP22 created new session: [3:7587241195511576611:2098]
2025-12-24T02:10:51.766787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195511576611:2098] [node 5] ICS09 handshake done sender: [3:7587241195511576594:2110] self: [3:7587241195511576589:0] peer: [5:7587241198332964545:0] socket: 108 qp: -1
2025-12-24T02:10:51.766794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195511576611:2098] [node 5] ICS10 traffic start
2025-12-24T02:10:51.766813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.767968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241198238064969:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.768026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241198238064897:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.768040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.768046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241198238064897:2048] [node 1] ICP22 created new session: [4:7587241198238064982:2048]
2025-12-24T02:10:51.768051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241198238064982:2048] [node 1] ICS09 handshake done sender: [4:7587241198238064969:2114] self: [4:7587241198238064979:0] peer: [1:7587241198486983726:0] socket: 135 qp: -1
2025-12-24T02:10:51.768056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241198238064982:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.768073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.767382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198486983721:2103] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.767400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.767407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198486983721:2103] [node 4] ICP22 created new session: [1:7587241198486983749:2103]
2025-12-24T02:10:51.767412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241198486983749:2103] [node 4] ICS09 handshake done sender: [1:7587241198486983728:2109] self: [1:7587241198486983726:0] peer: [4:7587241198238064979:0] socket: 113 qp: -1
2025-12-24T02:10:51.767420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241198486983749:2103] [node 4] ICS10 traffic start
2025-12-24T02:10:51.767428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.769786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241198486983720:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.770023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198486983659:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.770036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.770043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198486983659:2048] [node 2] ICP22 created new session: [1:7587241198486983754:2048]
2025-12-24T02:10:51.770048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241198486983754:2048] [node 2] ICS09 handshake done sender: [1:7587241198486983720:2106] self: [1:7587241198486983716:0] peer: [2:7587241198740158463:0] socket: 114 qp: -1
2025-12-24T02:10:51.770054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241198486983754:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:51.770059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.770343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241195511576527:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241195511576605:2115] held: no
2025-12-24T02:10:51.771017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576614:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.774991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790689:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.775069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790690:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.775082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.775104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790692:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.786927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241198332964466:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30251) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241198332964530:2112] held: no
2025-12-24T02:10:51.787006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964550:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.787116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.791068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195511576605:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.796792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241198486983660:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(33351036ns) {HandleHandshakeStatus:329(33350145ns) {DropIncomingHandshake:442(6011ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(146ns) ProcessPendingSessionEvents:495(25465607ns) {ForwardSessionEventToSession:612(25454305ns) ForwardSessionEventToSession:612(1471ns) ForwardSessionEventToSession:612(933ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(467ns) ForwardSessionEventToSession:612(1166ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(486ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(796ns)}}}
2025-12-24T02:10:51.795013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964549:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.800843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241197856790692:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.800890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241197856790610:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.800972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241197856790689:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.800996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241197856790611:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.801151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241195511576527:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.801172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.801183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195511576527:2048] [node 1] ICP22 created new session: [3:7587241195511576616:2048]
2025-12-24T02:10:51.801194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195511576616:2048] [node 1] ICS09 handshake done sender: [3:7587241195511576605:2115] self: [3:7587241195511576608:0] peer: [1:7587241198486983719:0] socket: 131 qp: -1
2025-12-24T02:10:51.801206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195511576616:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.801220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.801701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241197856790691:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.801749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241197856790612:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.800486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241198740158378:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241198740158458:2116] held: no
2025-12-24T02:10:51.802467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241198740158448:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.798835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964551:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.798900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964552:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.800303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241198332964530:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.806719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158452:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.806731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241198740158452:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.807359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241198740158441:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.807371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.807380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198740158441:2106] [node 4] ICP22 created new session: [2:7587241198740158472:2106]
2025-12-24T02:10:51.807386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241198740158472:2106] [node 4] ICS09 handshake done sender: [2:7587241198740158448:2111] self: [2:7587241198740158446:0] peer: [4:7587241198238064978:0] socket: 106 qp: -1
2025-12-24T02:10:51.807392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241198740158472:2106] [node 4] ICS10 traffic start
2025-12-24T02:10:51.807400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.808756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241198740158471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13867
2025-12-24T02:10:51.810011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241198740158452:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.810216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241198740158443:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.810236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.810242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198740158443:2108] [node 6] ICP22 created new session: [2:7587241198740158475:2108]
2025-12-24T02:10:51.810248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241198740158475:2108] [node 6] ICS09 handshake done sender: [2:7587241198740158452:2113] self: [2:7587241198740158451:0] peer: [6:7587241197856790694:0] socket: 128 qp: -1
2025-12-24T02:10:51.810254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241198740158475:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:51.810263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.810367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241198740158458:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.810682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241198740158378:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.810711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.810717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198740158378:2048] [node 1] ICP22 created new session: [2:7587241198740158477:2048]
2025-12-24T02:10:51.810723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241198740158477:2048] [node 1] ICS09 handshake done sender: [2:7587241198740158458:2116] self: [2:7587241198740158463:0] peer: [1:7587241198486983716:0] socket: 127 qp: -1
2025-12-24T02:10:51.810728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241198740158477:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.810734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.812400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983733:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.812417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241198486983733:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.814814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576601:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.814834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195511576601:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.815476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158461:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.815506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241198740158441:2106] [node 4] ICP18 handshake (actor [2:7587241198740158461:2119]) is too old
2025-12-24T02:10:51.818193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241198486983733:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.818323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198486983723:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.818341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.818348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198486983723:2105] [node 6] ICP22 created new session: [1:7587241198486983757:2105]
2025-12-24T02:10:51.818354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241198486983757:2105] [node 6] ICS09 handshake done sender: [1:7587241198486983733:2113] self: [1:7587241198486983724:0] peer: [6:7587241197856790693:0] socket: 151 qp: -1
2025-12-24T02:10:51.818360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241198486983757:2105] [node 6] ICS10 traffic start
2025-12-24T02:10:51.818367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.818831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241198238064898:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241198238064968:2113] held: no
2025-12-24T02:10:51.818931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064985:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.818948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241198238064899:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.819703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241197856790611:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11062) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241197856790689:2116] held: no
2025-12-24T02:10:51.819818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790698:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.820244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195511576601:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.822779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241198332964529:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.822833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964551:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.822879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241198332964466:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.822900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.822904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241198332964466:2050] [node 3] ICP22 created new session: [5:7587241198332964553:2050]
2025-12-24T02:10:51.822910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241198332964553:2050] [node 3] ICS09 handshake done sender: [5:7587241198332964530:2112] self: [5:7587241198332964545:0] peer: [3:7587241195511576589:0] socket: 117 qp: -1
2025-12-24T02:10:51.822915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241198332964553:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.822920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.823161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241198332964464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.823576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964552:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.824616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241198332964531:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.824564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064988:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.824843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576604:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.824855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241195511576604:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.826720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241198332964465:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.826733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.826739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241198332964465:2049] [node 2] ICP22 created new session: [5:7587241198332964558:2049]
2025-12-24T02:10:51.826744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241198332964558:2049] [node 2] ICS09 handshake done sender: [5:7587241198332964529:2111] self: [5:7587241198332964546:0] peer: [2:7587241198740158449:0] socket: 116 qp: -1
2025-12-24T02:10:51.826750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241198332964558:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.826755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.827167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241198238064968:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:51.830743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064973:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.830754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241198238064973:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.831354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195511576595:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.831371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.831379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195511576595:2101] [node 6] ICP22 created new session: [3:7587241195511576622:2101]
2025-12-24T02:10:51.831386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195511576622:2101] [node 6] ICS09 handshake done sender: [3:7587241195511576601:2113] self: [3:7587241195511576600:0] peer: [6:7587241197856790695:0] socket: 132 qp: -1
2025-12-24T02:10:51.831392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195511576622:2101] [node 6] ICS10 traffic start
2025-12-24T02:10:51.831399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.831527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
2025-12-24T02:10:51.831572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576619:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
2025-12-24T02:10:51.834848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195511576604:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.834799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241198486983723:2105] [node 6] ICP35 event processing took too much time PendingConnection:279(16437402ns) {HandleHandshakeStatus:329(16436400ns) {DropIncomingHandshake:442(3176ns) DropOutgoingHandshake:457(145ns) SwitchToState:179(282ns) ProcessPendingSessionEvents:495(16356469ns) {ForwardSessionEventToSession:612(16341497ns) ForwardSessionEventToSession:612(2208ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(1574ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(501ns) ForwardSessionEventToSession:612(767ns) ForwardSessionEventToSession:612(588ns) ForwardSessionEventToSession:612(933ns) ForwardSessionEventToSession:612(1029ns)}}}
2025-12-24T02:10:51.834921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983738:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.834925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241198486983738:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.835020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576619:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.835048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241195511576579:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.835058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.835066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195511576579:2096] [node 4] ICP22 created new session: [3:7587241195511576624:2096]
2025-12-24T02:10:51.835072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195511576624:2096] [node 4] ICS09 handshake done sender: [3:7587241195511576604:2114] self: [3:7587241195511576588:0] peer: [4:7587241198238064987:0] socket: 133 qp: -1
2025-12-24T02:10:51.835078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195511576624:2096] [node 4] ICS10 traffic start
2025-12-24T02:10:51.835091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.835318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241197856790687:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.835352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790696:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.835363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241197856790690:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.835402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241197856790685:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.835414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241197856790672:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.835741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241198238064898:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.835759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.835765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241198238064898:2049] [node 2] ICP22 created new session: [4:7587241198238064989:2049]
2025-12-24T02:10:51.835771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241198238064989:2049] [node 2] ICS09 handshake done sender: [4:7587241198238064968:2113] self: [4:7587241198238064978:0] peer: [2:7587241198740158446:0] socket: 111 qp: -1
2025-12-24T02:10:51.835778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241198238064989:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.835785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.835898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241198238064967:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.835902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241198238064967:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.835404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241195511576528:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:51.836282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241195511576620:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.836740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241198486983759:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56584
2025-12-24T02:10:51.836568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241198238064973:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.836891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241195511576579:2096] [node 4] ICP18 handshake (actor [3:7587241195511576620:2119]) is too old
2025-12-24T02:10:51.837091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790697:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.837425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238064991:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.837611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241198486983738:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.837466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241198332964540:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.837482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241198332964540:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:51.837691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198486983725:2104] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.837700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.837706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198486983725:2104] [node 5] ICP22 created new session: [1:7587241198486983761:2104]
2025-12-24T02:10:51.837712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241198486983761:2104] [node 5] ICS09 handshake done sender: [1:7587241198486983738:2115] self: [1:7587241198486983730:0] peer: [5:7587241198332964556:0] socket: 152 qp: -1
2025-12-24T02:10:51.837717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241198486983761:2104] [node 5] ICS10 traffic start
2025-12-24T02:10:51.837724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.846273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241198740158456:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.846303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241198740158456:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.847012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241195511576627:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11126
2025-12-24T02:10:51.847081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241197856790689:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:10:51.850833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241198740158456:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.847858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241198332964540:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.847982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241198332964532:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.847999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.848008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241198332964532:2109] [node 6] ICP22 created new session: [5:7587241198332964561:2109]
2025-12-24T02:10:51.848015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241198332964561:2109] [node 6] ICS09 handshake done sender: [5:7587241198332964540:2116] self: [5:7587241198332964539:0] peer: [6:7587241197856790700:0] socket: 118 qp: -1
2025-12-24T02:10:51.848022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241198332964561:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:51.848031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.851168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241195511576619:2118] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:10:51.851170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241198740158379:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.851190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.851195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198740158379:2049] [node 3] ICP22 created new session: [2:7587241198740158480:2049]
2025-12-24T02:10:51.851201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241198740158480:2049] [node 3] ICS09 handshake done sender: [2:7587241198740158456:2114] self: [2:7587241198740158444:0] peer: [3:7587241195511576626:0] socket: 159 qp: -1
2025-12-24T02:10:51.851207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241198740158480:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:51.851216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.851013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241198486983759:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:51.853039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241198238064965:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.853065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.853074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241198238064965:2108] [node 5] ICP22 created new session: [4:7587241198238064994:2108]
2025-12-24T02:10:51.853081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241198238064994:2108] [node 5] ICS09 handshake done sender: [4:7587241198238064973:2116] self: [4:7587241198238064972:0] peer: [5:7587241198332964557:0] socket: 137 qp: -1
2025-12-24T02:10:51.853088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241198238064994:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:51.853096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:51.853798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241198486983725:2104] [node 5] ICP35 event processing took too much time PendingConnection:279(16067153ns) {HandleHandshakeStatus:329(16066585ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(175ns) ProcessPendingSessionEvents:495(16000995ns) {ForwardSessionEventToSession:612(15990184ns) ForwardSessionEventToSession:612(2084ns) ForwardSessionEventToSession:612(824ns) ForwardSessionEventToSession:612(652ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(1574ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(691ns)}}}
2025-12-24T02:10:51.853896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241198486983725:2104] [node 5] ICP18 handshake (actor [1:7587241198486983759:2119]) is too old
2025-12-24T02:10:51.854199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241198238064985:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.854540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241198238064899:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30251) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241198238064985:2119] held: no
2025-12-24T02:10:51.854564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241198238064899:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.854578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.854584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241198238064899:2050] [node 3] ICP22 created new session: [4:7587241198238064996:2050]
2025-12-24T02:10:51.854590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241198238064996:2050] [node 3] ICS09 handshake done sender: [4:7587241198238064985:2119] self: [4:7587241198238064987:0] peer: [3:7587241195511576588:0] socket: 141 qp: -1
2025-12-24T02:10:51.854596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241198238064996:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.854603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:51.854846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241198238064967:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:51.856034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241198238064964:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:51.856049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:51.856056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241198238064964:2107] [node 6] ICP22 created new session: [4:7587241198238064998:2107]
2025-12-24T02:10:51.856062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241198238064998:2107] [node 6] ICS09 handshake done sender: [4:7587241198238064967:2112] self: [4:7587241198238064966:0] peer: [6:7587241197856790699:0] socket: 139 qp: -1
2025-12-24T02:10:51.856069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241198238064998:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:51.856076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:51.856687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964563:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.857973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241197856790611:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.857991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.857999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241197856790611:2049] [node 2] ICP22 created new session: [6:7587241197856790703:2049]
2025-12-24T02:10:51.858006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241197856790703:2049] [node 2] ICS09 handshake done sender: [6:7587241197856790689:2116] self: [6:7587241197856790694:0] peer: [2:7587241198740158451:0] socket: 153 qp: -1
2025-12-24T02:10:51.858013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241197856790703:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.858020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.856831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241198332964564:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:538
2025-12-24T02:10:51.857847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241198332964552:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.858146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241197856790691:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:51.858225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241198332964551:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.859004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241197856790692:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:51.859093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241197856790672:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241197856790690:2117] held: no
2025-12-24T02:10:51.859153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241197856790610:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241197856790692:2119] held: no
2025-12-24T02:10:51.859165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241197856790610:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.859172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.859177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241197856790610:2048] [node 1] ICP22 created new session: [6:7587241197856790705:2048]
2025-12-24T02:10:51.859182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241197856790705:2048] [node 1] ICS09 handshake done sender: [6:7587241197856790692:2119] self: [6:7587241197856790693:0] peer: [1:7587241198486983724:0] socket: 156 qp: -1
2025-12-24T02:10:51.859187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241197856790705:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.859193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.858915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241198332964531:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.858934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.858940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241198332964531:2108] [node 4] ICP22 created new session: [5:7587241198332964565:2108]
2025-12-24T02:10:51.858952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241198332964565:2108] [node 4] ICS09 handshake done sender: [5:7587241198332964552:2121] self: [5:7587241198332964557:0] peer: [4:7587241198238064972:0] socket: 158 qp: -1
2025-12-24T02:10:51.858960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241198332964565:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:51.858975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.859152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241198332964464:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.859161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:51.859165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241198332964464:2048] [node 1] ICP22 created new session: [5:7587241198332964567:2048]
2025-12-24T02:10:51.859170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241198332964567:2048] [node 1] ICS09 handshake done sender: [5:7587241198332964551:2120] self: [5:7587241198332964556:0] peer: [1:7587241198486983730:0] socket: 157 qp: -1
2025-12-24T02:10:51.859174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241198332964567:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:51.859179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:51.862774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241195511576528:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.862801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:51.862810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241195511576528:2049] [node 2] ICP22 created new session: [3:7587241195511576628:2049]
2025-12-24T02:10:51.862817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241195511576628:2049] [node 2] ICS09 handshake done sender: [3:7587241195511576619:2118] self: [3:7587241195511576626:0] peer: [2:7587241198740158444:0] socket: 160 qp: -1
2025-12-24T02:10:51.862825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241195511576628:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:51.862833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:51.867135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241197856790612:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.867157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:51.867162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241197856790612:2050] [node 3] ICP22 created new session: [6:7587241197856790707:2050]
2025-12-24T02:10:51.867170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241197856790707:2050] [node 3] ICS09 handshake done sender: [6:7587241197856790691:2118] self: [6:7587241197856790695:0] peer: [3:7587241195511576600:0] socket: 155 qp: -1
2025-12-24T02:10:51.867195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241197856790707:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:51.867201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.875209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241198332964464:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:51.875248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241198332964531:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:51.871207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790701:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.872340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241197856790702:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15132
2025-12-24T02:10:51.875188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241197856790690:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:51.875304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241197856790687:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:51.875362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241197856790685:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.875372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:51.875379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241197856790685:2113] [node 4] ICP22 created new session: [6:7587241197856790711:2113]
2025-12-24T02:10:51.875384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241197856790711:2113] [node 4] ICS09 handshake done sender: [6:7587241197856790687:2114] self: [6:7587241197856790699:0] peer: [4:7587241198238064966:0] socket: 142 qp: -1
2025-12-24T02:10:51.875389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241197856790711:2113] [node 4] ICS10 traffic start
2025-12-24T02:10:51.875397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:51.875689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241197856790672:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:51.875696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:51.875701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241197856790672:2108] [node 5] ICP22 created new session: [6:7587241197856790713:2108]
2025-12-24T02:10:51.875707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241197856790713:2108] [node 5] ICS09 handshake done sender: [6:7587241197856790690:2117] self: [6:7587241197856790700:0] peer: [5:7587241198332964539:0] socket: 112 qp: -1
2025-12-24T02:10:51.875711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241197856790713:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:51.875715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:51.879199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241198238065000:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24435
2025-12-24T02:10:51.883071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:51.883169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241198740158477:2048] [node 1] ICS01 socket: 127 reason# EndOfStream
2025-12-24T02:10:51.883177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241198740158378:2048] [node 1] ICP30 unregister session Session# [2:7587241198740158477:2048] VirtualId# [2:7587241198740158463:0]
2025-12-24T02:10:51.883184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241198740158477:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.883219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.883239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.887047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:51.887137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241195511576616:2048] [node 1] ICS01 socket: 131 reason# EndOfStream
2025-12-24T02:10:51.887146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241195511576527:2048] [node 1] ICP30 unregister session Session# [3:7587241195511576616:2048] VirtualId# [3:7587241195511576608:0]
2025-12-24T02:10:51.887154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241195511576616:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.887198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.887218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.889937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:51.890021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241198238064982:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-12-24T02:10:51.890029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241198238064897:2048] [node 1] ICP30 unregister session Session# [4:7587241198238064982:2048] VirtualId# [4:7587241198238064979:0]
2025-12-24T02:10:51.890035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241198238064982:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.890086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.890111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.890838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:51.890906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241197856790705:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-12-24T02:10:51.890912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241197856790610:2048] [node 1] ICP30 unregister session Session# [6:7587241197856790705:2048] VirtualId# [6:7587241197856790693:0]
2025-12-24T02:10:51.890916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241197856790705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.890959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.890979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.890822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:51.890905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241198332964567:2048] [node 1] ICS07 socket disconnect 157 reason# EndOfStream
2025-12-24T02:10:51.890911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241198332964567:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.890965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:51.890987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241198332964567:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:51.891395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241198332964464:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.891408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241198332964567:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.891413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241198332964464:2048] [node 1] ICP30 unregister session Session# [5:7587241198332964567:2048] VirtualId# [5:7587241198332964556:0]
2025-12-24T02:10:51.891427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:51.891432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241198332964464:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.891435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2269) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.897211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:51.897296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241195511576628:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-12-24T02:10:51.897302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241195511576528:2049] [node 2] ICP30 unregister session Session# [3:7587241195511576628:2049] VirtualId# [3:7587241195511576626:0]
2025-12-24T02:10:51.897308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241195511576628:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.897340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.897366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.899307Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:51.899381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241198332964558:2049] [node 2] ICS01 socket: 116 reason# EndOfStream
2025-12-24T02:10:51.899388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241198332964465:2049] [node 2] ICP30 unregister session Session# [5:7587241198332964558:2049] VirtualId# [5:7587241198332964546:0]
2025-12-24T02:10:51.899392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241198332964558:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.899432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.899461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.902802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241198332964464:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:51.902768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:51.902845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241197856790703:2049] [node 2] ICS01 socket: 153 reason# EndOfStream
2025-12-24T02:10:51.902850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241197856790611:2049] [node 2] ICP30 unregister session Session# [6:7587241197856790703:2049] VirtualId# [6:7587241197856790694:0]
2025-12-24T02:10:51.902855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241197856790703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.902911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.902941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.902756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:51.902858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241198238064989:2049] [node 2] ICS01 socket: 111 reason# EndOfStream
2025-12-24T02:10:51.902866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241198238064898:2049] [node 2] ICP30 unregister session Session# [4:7587241198238064989:2049] VirtualId# [4:7587241198238064978:0]
2025-12-24T02:10:51.902873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241198238064989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.902943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:51.902969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:51.909595Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:51.909659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241198332964553:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-12-24T02:10:51.909663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241198332964466:2050] [node 3] ICP30 unregister session Session# [5:7587241198332964553:2050] VirtualId# [5:7587241198332964545:0]
2025-12-24T02:10:51.909667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241198332964553:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.909691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.909708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.910759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:51.910827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241197856790707:2050] [node 3] ICS07 socket disconnect 155 reason# EndOfStream
2025-12-24T02:10:51.910831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241197856790707:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.910872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:51.910892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241197856790707:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:51.911234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241197856790612:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30251) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:51.911248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241197856790707:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:51.911253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241197856790612:2050] [node 3] ICP30 unregister session Session# [6:7587241197856790707:2050] VirtualId# [6:7587241197856790695:0]
2025-12-24T02:10:51.911265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:51.911271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241197856790612:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.911274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30251) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:51.922765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:51.922850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241197856790711:2113] [node 4] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:10:51.922857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241197856790685:2113] [node 4] ICP30 unregister session Session# [6:7587241197856790711:2113] VirtualId# [6:7587241197856790699:0]
2025-12-24T02:10:51.922862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241197856790711:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:51.922911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:51.922937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:51.922964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241197856790612:2050] [node 3] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:51.974794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241199252983927:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.974804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241199252983928:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.974808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241199252983929:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.969203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198973730187:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.969218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241198973730188:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.953787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198957376621:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:51.953795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241198957376622:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:51.974186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241197653415362:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.974197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241197653415363:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.974202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241197653415364:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:51.973714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241195799304939:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:51.973721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241195799304940:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:51.973726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241195799304941:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.002729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241197180409271:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.002734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241197180409270:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.026657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201475376619:2097] [node 6] ICP01 ready to work
2025-12-24T02:10:52.026799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201475376623:2099] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.028574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951284:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.027713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241203268697543:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.026486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241200094272296:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:52.029927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272300:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.031675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201475376624:2095] [node 4] ICP01 ready to work
2025-12-24T02:10:52.032266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203547951284:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.032451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203547951288:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:52.032752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203547951288:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.032058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697543:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.032472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241198973730187:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.034727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201475376631:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15927
2025-12-24T02:10:52.036177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343977:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.036194Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203252343977:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.036280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241200094272298:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.036293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241200094272298:2108] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.039461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203252343982:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10799
2025-12-24T02:10:52.039347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241200094272300:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.039584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241195799304941:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.039548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201475376631:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.039910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241197180409270:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.041770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382719:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.042604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241203268697551:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:52.042726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241203268697552:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:52.043094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951291:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.042898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203252343986:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:52.042998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203252343987:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:52.043110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203252343988:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:52.043601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343980:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.043618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203252343980:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.043517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201475376627:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.043533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201475376627:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.043225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382719:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.043691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203547951291:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.044045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241199252983929:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.044629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241203268697546:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:52.045127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201475376630:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.045140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201475376630:2105] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.045401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241203268697555:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.045912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697555:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.046152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382721:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.045824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951293:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.049904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241197653415364:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.051954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241203268697564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51244
2025-12-24T02:10:52.050147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203252343977:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.050324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203252343980:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.052868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203547951293:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.052934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241199252983928:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.052807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241203268697543:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.053207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697556:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.053221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241203268697556:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.053675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241198973730187:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.053696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.053703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198973730187:2048] [node 1] ICP22 created new session: [2:7587241203268697565:2048]
2025-12-24T02:10:52.053710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241203268697565:2048] [node 1] ICS09 handshake done sender: [2:7587241203268697543:2107] self: [2:7587241203268697544:0] peer: [1:7587241203252343976:0] socket: 112 qp: -1
2025-12-24T02:10:52.053718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241203268697565:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.053726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.054184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241203268697556:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.054515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241203268697551:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.054539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.054546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241203268697551:2109] [node 5] ICP22 created new session: [2:7587241203268697568:2109]
2025-12-24T02:10:52.054558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241203268697568:2109] [node 5] ICS09 handshake done sender: [2:7587241203268697556:2113] self: [2:7587241203268697553:0] peer: [5:7587241203547951294:0] socket: 129 qp: -1
2025-12-24T02:10:52.054564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241203268697568:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:52.054571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.050213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201475376640:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.050224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201475376640:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.051227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241200094272298:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.051355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241200094272296:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.051373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.051382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241200094272296:2107] [node 5] ICP22 created new session: [6:7587241200094272307:2107]
2025-12-24T02:10:52.051389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241200094272307:2107] [node 5] ICS09 handshake done sender: [6:7587241200094272298:2108] self: [6:7587241200094272297:0] peer: [5:7587241203547951289:0] socket: 108 qp: -1
2025-12-24T02:10:52.051397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241200094272307:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:52.051413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.058353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951295:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.058407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951297:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.055862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272309:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.055366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343982:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.062046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198957376622:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.062070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.062080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198957376622:2049] [node 3] ICP22 created new session: [1:7587241203252343999:2049]
2025-12-24T02:10:52.062087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203252343999:2049] [node 3] ICS09 handshake done sender: [1:7587241203252343980:2107] self: [1:7587241203252343979:0] peer: [3:7587241201475376642:0] socket: 114 qp: -1
2025-12-24T02:10:52.062102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203252343999:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.062113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.061716Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241199252983928:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241203547951293:2112] held: no
2025-12-24T02:10:52.061787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951296:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.061971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203547951284:2107] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.062826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201475376627:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.062300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382721:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.062819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241197653415363:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.062915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203547951288:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.062936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.062944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203547951288:2109] [node 6] ICP22 created new session: [5:7587241203547951298:2109]
2025-12-24T02:10:52.062953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203547951298:2109] [node 6] ICS09 handshake done sender: [5:7587241203547951284:2107] self: [5:7587241203547951289:0] peer: [6:7587241200094272297:0] socket: 117 qp: -1
2025-12-24T02:10:52.062970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203547951298:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:52.062993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.063246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203547951293:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.063488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241199252983928:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.063503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.063507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241199252983928:2049] [node 2] ICP22 created new session: [5:7587241203547951301:2049]
2025-12-24T02:10:52.063513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203547951301:2049] [node 2] ICS09 handshake done sender: [5:7587241203547951293:2112] self: [5:7587241203547951294:0] peer: [2:7587241203268697553:0] socket: 131 qp: -1
2025-12-24T02:10:52.063518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203547951301:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.063524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.063665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951300:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.063628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697560:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.063644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241203268697560:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.066767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241203268697560:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.062826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272310:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.066932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382724:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.062904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241200094272300:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.062984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241195799304941:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.063018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.063024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241195799304941:2050] [node 3] ICP22 created new session: [6:7587241200094272311:2050]
2025-12-24T02:10:52.063030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241200094272311:2050] [node 3] ICS09 handshake done sender: [6:7587241200094272300:2109] self: [6:7587241200094272305:0] peer: [3:7587241201475376626:0] socket: 118 qp: -1
2025-12-24T02:10:52.063048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241200094272311:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.063055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.064040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241200094272309:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.064373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241195799304940:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.067027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201948382721:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.066906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697563:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.066914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241203268697563:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.067188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241203268697546:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.067207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.067215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241203268697546:2108] [node 4] ICP22 created new session: [2:7587241203268697572:2108]
2025-12-24T02:10:52.067221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241203268697572:2108] [node 4] ICS09 handshake done sender: [2:7587241203268697560:2114] self: [2:7587241203268697559:0] peer: [4:7587241201948382722:0] socket: 132 qp: -1
2025-12-24T02:10:52.067229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241203268697572:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:52.067239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.069165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241198957376621:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.069194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.069205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241198957376621:2048] [node 2] ICP22 created new session: [1:7587241203252344001:2048]
2025-12-24T02:10:52.069212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203252344001:2048] [node 2] ICS09 handshake done sender: [1:7587241203252343977:2106] self: [1:7587241203252343976:0] peer: [2:7587241203268697544:0] socket: 109 qp: -1
2025-12-24T02:10:52.069220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203252344001:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:52.069229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.069455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382723:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.075780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241197180409271:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32256983ns) {Configure:104(32256282ns) {ProcessConfigured:135(31400959ns) {StartInitialHandshake:157(31400375ns) {DropHandshakes:470(360ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(863ns) {SwitchToState:179(97ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.076573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203547951307:2118] [node 4] ICP01 ready to work
2025-12-24T02:10:52.075873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201475376630:2105] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.077258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201475376640:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.077294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201475376623:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.077304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.077311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201475376623:2099] [node 5] ICP22 created new session: [3:7587241201475376648:2099]
2025-12-24T02:10:52.077316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201475376648:2099] [node 5] ICS09 handshake done sender: [3:7587241201475376630:2105] self: [3:7587241201475376629:0] peer: [5:7587241203547951292:0] socket: 122 qp: -1
2025-12-24T02:10:52.077322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201475376648:2099] [node 5] ICS10 traffic start
2025-12-24T02:10:52.077330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.077481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201475376619:2097] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.077499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.077506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201475376619:2097] [node 6] ICP22 created new session: [3:7587241201475376650:2097]
2025-12-24T02:10:52.077511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201475376650:2097] [node 6] ICS09 handshake done sender: [3:7587241201475376627:2104] self: [3:7587241201475376626:0] peer: [6:7587241200094272305:0] socket: 111 qp: -1
2025-12-24T02:10:52.077516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201475376650:2097] [node 6] ICS10 traffic start
2025-12-24T02:10:52.077522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.077527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201475376624:2095] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.077538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.077546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201475376624:2095] [node 4] ICP22 created new session: [3:7587241201475376652:2095]
2025-12-24T02:10:52.077552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201475376652:2095] [node 4] ICS09 handshake done sender: [3:7587241201475376640:2113] self: [3:7587241201475376634:0] peer: [4:7587241201948382720:0] socket: 126 qp: -1
2025-12-24T02:10:52.077560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201475376652:2095] [node 4] ICS10 traffic start
2025-12-24T02:10:52.077592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.078406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203547951291:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.076948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203252344004:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10799
2025-12-24T02:10:52.077392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201948382727:2112] [node 5] ICP01 ready to work
2025-12-24T02:10:52.077497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201948382728:2113] [node 6] ICP01 ready to work
2025-12-24T02:10:52.079656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241200094272315:2114] [node 4] ICP01 ready to work
2025-12-24T02:10:52.082833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241203268697563:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.087698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382729:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.087726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201948382719:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.087672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241199252983929:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.087697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.087707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241199252983929:2050] [node 3] ICP22 created new session: [5:7587241203547951310:2050]
2025-12-24T02:10:52.087716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203547951310:2050] [node 3] ICS09 handshake done sender: [5:7587241203547951291:2111] self: [5:7587241203547951292:0] peer: [3:7587241201475376629:0] socket: 123 qp: -1
2025-12-24T02:10:52.087727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203547951310:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.087738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.090235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241197653415363:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.090267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.090281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241197653415363:2049] [node 2] ICP22 created new session: [4:7587241201948382735:2049]
2025-12-24T02:10:52.090288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201948382735:2049] [node 2] ICS09 handshake done sender: [4:7587241201948382721:2109] self: [4:7587241201948382722:0] peer: [2:7587241203268697559:0] socket: 133 qp: -1
2025-12-24T02:10:52.090299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201948382735:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.090324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.087835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241203268697552:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.087855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.087863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241203268697552:2111] [node 6] ICP22 created new session: [2:7587241203268697575:2111]
2025-12-24T02:10:52.087870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241203268697575:2111] [node 6] ICS09 handshake done sender: [2:7587241203268697563:2115] self: [2:7587241203268697554:0] peer: [6:7587241200094272313:0] socket: 141 qp: -1
2025-12-24T02:10:52.087879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241203268697575:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:52.087887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.094815Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241198957376622:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(32720045ns) {HandleHandshakeStatus:329(32719146ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(227ns) SwitchToState:179(263ns) ProcessPendingSessionEvents:495(25462729ns) {ForwardSessionEventToSession:612(25447192ns) ForwardSessionEventToSession:612(3026ns) ForwardSessionEventToSession:612(913ns) ForwardSessionEventToSession:612(481ns) ForwardSessionEventToSession:612(452ns) ForwardSessionEventToSession:612(1563ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(844ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(855ns)}}}
2025-12-24T02:10:52.099144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203547951295:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.099194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241199252983927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.099778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343992:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.099793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203252343992:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.100431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203252343992:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.100518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203252343987:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.100531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.100539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203252343987:2110] [node 5] ICP22 created new session: [1:7587241203252344008:2110]
2025-12-24T02:10:52.100546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203252344008:2110] [node 5] ICS09 handshake done sender: [1:7587241203252343992:2113] self: [1:7587241203252343991:0] peer: [5:7587241203547951313:0] socket: 135 qp: -1
2025-12-24T02:10:52.100554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203252344008:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:52.100575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.102913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951314:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.103005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241200094272315:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23186609ns) {Configure:104(23185108ns) {ProcessConfigured:135(22876262ns) {StartInitialHandshake:157(22875626ns) {DropHandshakes:470(575ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(131ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(932ns) {SwitchToState:179(513ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.103239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201475376656:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15927
2025-12-24T02:10:52.103322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201475376655:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15927
2025-12-24T02:10:52.103451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201475376631:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.103564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241197180409270:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.103565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201475376655:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.103582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.103589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241197180409270:2048] [node 1] ICP22 created new session: [3:7587241201475376657:2048]
2025-12-24T02:10:52.103594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241197180409271:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.103597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201475376657:2048] [node 1] ICS09 handshake done sender: [3:7587241201475376631:2106] self: [3:7587241201475376642:0] peer: [1:7587241203252343979:0] socket: 115 qp: -1
2025-12-24T02:10:52.103604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201475376657:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.103623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.103143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203547951295:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.103376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241199252983927:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.103388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.103396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241199252983927:2048] [node 1] ICP22 created new session: [5:7587241203547951315:2048]
2025-12-24T02:10:52.103401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203547951315:2048] [node 1] ICS09 handshake done sender: [5:7587241203547951295:2113] self: [5:7587241203547951313:0] peer: [1:7587241203252343991:0] socket: 136 qp: -1
2025-12-24T02:10:52.103407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203547951315:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.103413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.106774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697549:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.106789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241203268697549:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.102978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241198957376621:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(47542779ns) {HandleHandshakeStatus:329(47542048ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(675ns) SwitchToState:179(79ns) ProcessPendingSessionEvents:495(23214211ns) {ForwardSessionEventToSession:612(23200314ns) ForwardSessionEventToSession:612(1463ns) ForwardSessionEventToSession:612(1296ns) ForwardSessionEventToSession:612(670ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(1216ns) ForwardSessionEventToSession:612(1089ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(925ns) ForwardSessionEventToSession:612(884ns)}}}
2025-12-24T02:10:52.106916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252344004:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.106971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241203252343987:2110] [node 5] ICP18 handshake (actor [1:7587241203252344004:2116]) is too old
2025-12-24T02:10:52.107399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241203268697549:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.107527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241198973730188:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.107540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.107545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241198973730188:2049] [node 3] ICP22 created new session: [2:7587241203268697579:2049]
2025-12-24T02:10:52.107550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241203268697579:2049] [node 3] ICS09 handshake done sender: [2:7587241203268697549:2110] self: [2:7587241203268697547:0] peer: [3:7587241201475376658:0] socket: 128 qp: -1
2025-12-24T02:10:52.107555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241203268697579:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.107561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.107078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241201948382728:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28767379ns) {Configure:104(28766961ns) {ProcessConfigured:135(28590118ns) {StartInitialHandshake:157(28589610ns) {DropHandshakes:470(133ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(1096ns) {SwitchToState:179(472ns)}}}}}
2025-12-24T02:10:52.107181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241197653415364:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.107192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.107197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241197653415364:2050] [node 3] ICP22 created new session: [4:7587241201948382738:2050]
2025-12-24T02:10:52.107203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201948382738:2050] [node 3] ICS09 handshake done sender: [4:7587241201948382719:2108] self: [4:7587241201948382720:0] peer: [3:7587241201475376634:0] socket: 127 qp: -1
2025-12-24T02:10:52.107209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201948382738:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.107229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.107650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382740:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.107663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382741:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.110510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382741:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.110557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382729:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.110639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241197653415362:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.112511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241199252983929:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24809063ns) {HandleHandshakeStatus:329(24808348ns) {DropIncomingHandshake:442(564ns) DropOutgoingHandshake:457(147ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(24676224ns) {ForwardSessionEventToSession:612(24662713ns) ForwardSessionEventToSession:612(2461ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(648ns) ForwardSessionEventToSession:612(1868ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(487ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(459ns)}}}
2025-12-24T02:10:52.112439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272319:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.112555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272320:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.112615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241203268697577:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51244
2025-12-24T02:10:52.112686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241200094272309:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.112753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241197653415363:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(43247228ns) {HandleHandshakeStatus:329(43246678ns) {DropIncomingHandshake:442(1175ns) DropOutgoingHandshake:457(144ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(392ns)}}
2025-12-24T02:10:52.114880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241203268697577:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.114934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241198973730188:2049] [node 3] ICP18 handshake (actor [2:7587241203268697577:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.114994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201475376660:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15927
2025-12-24T02:10:52.118791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201475376655:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.122064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203252344010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10799
2025-12-24T02:10:52.122166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343993:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.122176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203252343993:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.122777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241195799304940:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.122800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.122809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241195799304940:2049] [node 2] ICP22 created new session: [6:7587241200094272322:2049]
2025-12-24T02:10:52.122816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241200094272322:2049] [node 2] ICS09 handshake done sender: [6:7587241200094272309:2112] self: [6:7587241200094272313:0] peer: [2:7587241203268697554:0] socket: 142 qp: -1
2025-12-24T02:10:52.122823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241200094272322:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.122833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.123056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241200094272319:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.123184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241195799304939:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.123262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.124561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241199252983927:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21148514ns) {HandleHandshakeStatus:329(21147980ns) {DropIncomingHandshake:442(325ns) DropOutgoingHandshake:457(1751ns) SwitchToState:179(453ns) ProcessPendingSessionEvents:495(9768326ns) {ForwardSessionEventToSession:612(9758233ns) ForwardSessionEventToSession:612(1244ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(1105ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(475ns) ForwardSessionEventToSession:612(494ns) ForwardSessionEventToSession:612(636ns)}}}
2025-12-24T02:10:52.126272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382740:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.125684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241197180409271:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241201475376655:2115] held: no
2025-12-24T02:10:52.125710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241197180409271:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.125719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.125725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241197180409271:2049] [node 2] ICP22 created new session: [3:7587241201475376661:2049]
2025-12-24T02:10:52.125732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201475376661:2049] [node 2] ICS09 handshake done sender: [3:7587241201475376655:2115] self: [3:7587241201475376658:0] peer: [2:7587241203268697547:0] socket: 120 qp: -1
2025-12-24T02:10:52.125739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201475376661:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.125747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.126820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252343998:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.126832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203252343998:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.127804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203252344010:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.127913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203252343998:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.128074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203252343988:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.128089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.128095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203252343988:2111] [node 6] ICP22 created new session: [1:7587241203252344013:2111]
2025-12-24T02:10:52.128099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203252344013:2111] [node 6] ICS09 handshake done sender: [1:7587241203252343998:2115] self: [1:7587241203252343996:0] peer: [6:7587241200094272324:0] socket: 155 qp: -1
2025-12-24T02:10:52.128104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203252344013:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:52.128109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.129892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382733:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.129915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241201948382733:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.130639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201948382748:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37237
2025-12-24T02:10:52.128530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272325:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.128318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951317:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.128595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203547951317:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.128622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203547951307:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.130767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201948382733:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.130797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203547951319:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43575
2025-12-24T02:10:52.130876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241201948382727:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.130896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.130903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201948382727:2112] [node 5] ICP22 created new session: [4:7587241201948382750:2112]
2025-12-24T02:10:52.130915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201948382750:2112] [node 5] ICS09 handshake done sender: [4:7587241201948382733:2116] self: [4:7587241201948382732:0] peer: [5:7587241203547951318:0] socket: 171 qp: -1
2025-12-24T02:10:52.130924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201948382750:2112] [node 5] ICS10 traffic start
2025-12-24T02:10:52.130931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.130925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203547951317:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.131106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203547951307:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.131123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.131130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203547951307:2118] [node 4] ICP22 created new session: [5:7587241203547951320:2118]
2025-12-24T02:10:52.131136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203547951320:2118] [node 4] ICS09 handshake done sender: [5:7587241203547951317:2121] self: [5:7587241203547951318:0] peer: [4:7587241201948382732:0] socket: 177 qp: -1
2025-12-24T02:10:52.131143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203547951320:2118] [node 4] ICS10 traffic start
2025-12-24T02:10:52.131151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.134834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241200094272319:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.135027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203252343993:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.135266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203252343986:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.135283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.135291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203252343986:2109] [node 4] ICP22 created new session: [1:7587241203252344016:2109]
2025-12-24T02:10:52.135298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203252344016:2109] [node 4] ICS09 handshake done sender: [1:7587241203252343993:2114] self: [1:7587241203252343989:0] peer: [4:7587241201948382746:0] socket: 156 qp: -1
2025-12-24T02:10:52.135304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203252344016:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:52.135312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.135248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241195799304939:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.135272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.135280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241195799304939:2048] [node 1] ICP22 created new session: [6:7587241200094272326:2048]
2025-12-24T02:10:52.135288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241200094272326:2048] [node 1] ICS09 handshake done sender: [6:7587241200094272319:2116] self: [6:7587241200094272324:0] peer: [1:7587241203252343996:0] socket: 119 qp: -1
2025-12-24T02:10:52.135295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241200094272326:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.135304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.135402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241195799304939:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.139016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241200094272321:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.139165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241200094272315:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.139690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241197653415362:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241201948382729:2114] held: no
2025-12-24T02:10:52.139930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201948382729:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.139956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201948382737:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.139972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241201948382737:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.140011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241197653415362:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.140025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.140032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241197653415362:2048] [node 1] ICP22 created new session: [4:7587241201948382752:2048]
2025-12-24T02:10:52.140038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201948382752:2048] [node 1] ICS09 handshake done sender: [4:7587241201948382729:2114] self: [4:7587241201948382746:0] peer: [1:7587241203252343989:0] socket: 157 qp: -1
2025-12-24T02:10:52.140046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201948382752:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.140054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.141760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241200094272329:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43285
2025-12-24T02:10:52.142856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201948382737:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.142988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241201948382728:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.143004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.143012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201948382728:2113] [node 6] ICP22 created new session: [4:7587241201948382755:2113]
2025-12-24T02:10:52.143018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201948382755:2113] [node 6] ICS09 handshake done sender: [4:7587241201948382737:2117] self: [4:7587241201948382734:0] peer: [6:7587241200094272328:0] socket: 172 qp: -1
2025-12-24T02:10:52.143025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201948382755:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:52.143049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.144077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241200094272321:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.146801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241200094272315:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30097) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241200094272321:2118] held: no
2025-12-24T02:10:52.146864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241200094272315:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.146873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.146879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241200094272315:2114] [node 4] ICP22 created new session: [6:7587241200094272330:2114]
2025-12-24T02:10:52.146886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241200094272330:2114] [node 4] ICS09 handshake done sender: [6:7587241200094272321:2118] self: [6:7587241200094272328:0] peer: [4:7587241201948382734:0] socket: 173 qp: -1
2025-12-24T02:10:52.146895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241200094272330:2114] [node 4] ICS10 traffic start
2025-12-24T02:10:52.146908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.152720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:52.152815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241203268697565:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-12-24T02:10:52.152823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241198973730187:2048] [node 1] ICP30 unregister session Session# [2:7587241203268697565:2048] VirtualId# [2:7587241203268697544:0]
2025-12-24T02:10:52.152829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241203268697565:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.152871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.152894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.158757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:52.158857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241201475376657:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-12-24T02:10:52.158866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241197180409270:2048] [node 1] ICP30 unregister session Session# [3:7587241201475376657:2048] VirtualId# [3:7587241201475376642:0]
2025-12-24T02:10:52.158873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241201475376657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.158941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.158973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.159030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:52.159112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241201948382752:2048] [node 1] ICS01 socket: 157 reason# EndOfStream
2025-12-24T02:10:52.159117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241197653415362:2048] [node 1] ICP30 unregister session Session# [4:7587241201948382752:2048] VirtualId# [4:7587241201948382746:0]
2025-12-24T02:10:52.159122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241201948382752:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.159156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.159180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.159278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:52.159352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241200094272326:2048] [node 1] ICS07 socket disconnect 119 reason# EndOfStream
2025-12-24T02:10:52.159357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241200094272326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.159384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.159404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241200094272326:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:52.159901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241195799304939:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.159912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241200094272326:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.159917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241195799304939:2048] [node 1] ICP30 unregister session Session# [6:7587241200094272326:2048] VirtualId# [6:7587241200094272324:0]
2025-12-24T02:10:52.159930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.159935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241195799304939:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.159938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.160149Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:52.160239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241203547951315:2048] [node 1] ICS07 socket disconnect 136 reason# EndOfStream
2025-12-24T02:10:52.160248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203547951315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.160310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.160333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241203547951315:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:52.163046Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:52.163134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241201475376661:2049] [node 2] ICS01 socket: 120 reason# EndOfStream
2025-12-24T02:10:52.163142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241197180409271:2049] [node 2] ICP30 unregister session Session# [3:7587241201475376661:2049] VirtualId# [3:7587241201475376658:0]
2025-12-24T02:10:52.163148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241201475376661:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.163182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.163204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.164385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241199252983927:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.164407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203547951315:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.164413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241199252983927:2048] [node 1] ICP30 unregister session Session# [5:7587241203547951315:2048] VirtualId# [5:7587241203547951313:0]
2025-12-24T02:10:52.164432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.164441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241199252983927:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.164443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12074) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.166789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:52.166882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241201948382735:2049] [node 2] ICS07 socket disconnect 133 reason# EndOfStream
2025-12-24T02:10:52.166889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241201948382735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.166950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.166972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241201948382735:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:52.167360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241197653415363:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.167373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241201948382735:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.167377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241197653415363:2049] [node 2] ICP30 unregister session Session# [4:7587241201948382735:2049] VirtualId# [4:7587241201948382722:0]
2025-12-24T02:10:52.167393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.167399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241197653415363:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.167401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.168387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:52.168278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:52.168362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241200094272322:2049] [node 2] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:10:52.168369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241195799304940:2049] [node 2] ICP30 unregister session Session# [6:7587241200094272322:2049] VirtualId# [6:7587241200094272313:0]
2025-12-24T02:10:52.168375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241200094272322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.168430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.168462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.168463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241203547951301:2049] [node 2] ICS07 socket disconnect 131 reason# EndOfStream
2025-12-24T02:10:52.168473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203547951301:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.168528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.168556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241203547951301:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:52.168981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241199252983928:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.168995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203547951301:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.169002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241199252983928:2049] [node 2] ICP30 unregister session Session# [5:7587241203547951301:2049] VirtualId# [5:7587241203547951294:0]
2025-12-24T02:10:52.169018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.169023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241199252983928:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.169025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11464) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.170354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:52.170438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241203547951310:2050] [node 3] ICS07 socket disconnect 123 reason# EndOfStream
2025-12-24T02:10:52.170443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203547951310:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.170499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.170524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241203547951310:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:52.172032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241199252983929:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.172041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203547951310:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.172047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241199252983929:2050] [node 3] ICP30 unregister session Session# [5:7587241203547951310:2050] VirtualId# [5:7587241203547951292:0]
2025-12-24T02:10:52.172068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.172074Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241199252983929:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.172414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.173973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241199252983927:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:52.174741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241195799304939:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:52.176913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:52.177015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241200094272311:2050] [node 3] ICS07 socket disconnect 118 reason# EndOfStream
2025-12-24T02:10:52.177022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241200094272311:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.177087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.177113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241200094272311:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:52.177482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241195799304941:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.177494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241200094272311:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.177500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241195799304941:2050] [node 3] ICP30 unregister session Session# [6:7587241200094272311:2050] VirtualId# [6:7587241200094272305:0]
2025-12-24T02:10:52.177515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.177520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241195799304941:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.177523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14142) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.182783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:52.182887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241200094272330:2114] [node 4] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:10:52.182895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241200094272315:2114] [node 4] ICP30 unregister session Session# [6:7587241200094272330:2114] VirtualId# [6:7587241200094272328:0]
2025-12-24T02:10:52.182904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241200094272330:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.182968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:52.182993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:52.183129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:52.183193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241200094272307:2107] [node 5] ICS07 socket disconnect 108 reason# EndOfStream
2025-12-24T02:10:52.183198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241200094272307:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.183236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:52.183258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241200094272307:2107] [node 5] ICS15 start handshake
2025-12-24T02:10:52.183643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241200094272296:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.183656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241200094272307:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.183661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241200094272296:2107] [node 5] ICP30 unregister session Session# [6:7587241200094272307:2107] VirtualId# [6:7587241200094272297:0]
2025-12-24T02:10:52.183673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:10:52.183679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241200094272296:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.183682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:52.201731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241201395208986:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.201817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241201395208987:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.199522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200220127322:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:52.199529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200220127323:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.204252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201433112701:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.204259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201433112702:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.206274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201329813517:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.206282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201329813518:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.206286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201329813519:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.208102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241200194245293:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.208110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241200194245294:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.208114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241200194245295:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.210539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202102394989:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.210552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202102394991:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.210564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202102394990:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.221311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202102395049:2106] [node 5] ICP01 ready to work
2025-12-24T02:10:52.223528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200220127388:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:52.223615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200220127389:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.224004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200220127383:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15099
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.226215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201433112759:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-12-24T02:10:52.225182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200220127398:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.227650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201433112761:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.227795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200220127402:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15099
2025-12-24T02:10:52.227256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241201395209049:2106] [node 4] ICP01 ready to work
2025-12-24T02:10:52.227693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241201395209057:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:52.228018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241201395209061:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.228496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209062:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
2025-12-24T02:10:52.227906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112759:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.228415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241201433112701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.228718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201433112771:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:52.228822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201433112773:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:52.228909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241201433112774:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.227129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813578:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.227986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241200194245354:2105] [node 4] ICP01 ready to work
2025-12-24T02:10:52.231183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127387:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.231198Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200220127387:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.231352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127402:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.235366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209065:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.242192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200220127387:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.242268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200220127405:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15099
2025-12-24T02:10:52.242423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200220127323:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.242441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.242449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200220127323:2049] [node 3] ICP22 created new session: [1:7587241200220127406:2049]
2025-12-24T02:10:52.242456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200220127406:2049] [node 3] ICS09 handshake done sender: [1:7587241200220127387:2110] self: [1:7587241200220127385:0] peer: [3:7587241201433112767:0] socket: 119 qp: -1
2025-12-24T02:10:52.242464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200220127406:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.242472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.248442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200220127409:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15099
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.249041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209067:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.247074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395061:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.248633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202102395061:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.248705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202102394989:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.249207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127391:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.249217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200220127391:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.249694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127409:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.249763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127400:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.249766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200220127400:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.250282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200220127391:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.250379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200220127388:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.250391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.250406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200220127388:2108] [node 4] ICP22 created new session: [1:7587241200220127414:2108]
2025-12-24T02:10:52.250414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200220127414:2108] [node 4] ICS09 handshake done sender: [1:7587241200220127391:2111] self: [1:7587241200220127390:0] peer: [4:7587241201329813590:0] socket: 121 qp: -1
2025-12-24T02:10:52.250422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200220127414:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:52.250430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.250611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200220127400:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.247524Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241201329813517:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18186225ns) {Configure:104(18184768ns) {ProcessConfigured:135(15326728ns) {StartInitialHandshake:157(15326103ns) {DropHandshakes:470(647ns) {DropIncomingHandshake:442(157ns) DropOutgoingHandshake:457(135ns)} GenerateSessionVirtualId:988(437ns) PrepareNewSessionHandshake:477(1048ns) {SwitchToState:179(545ns)}}}}}
2025-12-24T02:10:52.247717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201329813585:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:52.247768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813578:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.248250Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241201329813518:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18887680ns) {Configure:104(18886854ns) {ProcessConfigured:135(15863496ns) {StartInitialHandshake:157(15863235ns) {DropHandshakes:470(307ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(509ns) {SwitchToState:179(71ns)}}}}}
2025-12-24T02:10:52.248532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241201329813517:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.250918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209068:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
2025-12-24T02:10:52.254137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209068:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.254189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241201395208986:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.253796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200220127398:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.253819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.253828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200220127398:2113] [node 6] ICP22 created new session: [1:7587241200220127417:2113]
2025-12-24T02:10:52.253834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200220127417:2113] [node 6] ICS09 handshake done sender: [1:7587241200220127400:2114] self: [1:7587241200220127399:0] peer: [6:7587241202102395063:0] socket: 143 qp: -1
2025-12-24T02:10:52.253841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200220127417:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:52.253848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.254431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127401:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.254436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200220127401:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.255021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200220127401:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.255091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200220127322:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.255103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.255109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200220127322:2048] [node 2] ICP22 created new session: [1:7587241200220127420:2048]
2025-12-24T02:10:52.255115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200220127420:2048] [node 2] ICS09 handshake done sender: [1:7587241200220127401:2115] self: [1:7587241200220127382:0] peer: [2:7587241201395209069:0] socket: 154 qp: -1
2025-12-24T02:10:52.255120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200220127420:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:52.255127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.258802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241201433112773:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29842664ns) {Configure:104(29841834ns) {ProcessConfigured:135(29580061ns) {StartInitialHandshake:157(29579552ns) {DropHandshakes:470(421ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1128ns) {SwitchToState:179(526ns)}}}}}
2025-12-24T02:10:52.258933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201433112783:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-12-24T02:10:52.259159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201433112759:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.259238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241201433112701:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.259254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.259261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201433112701:2048] [node 1] ICP22 created new session: [3:7587241201433112784:2048]
2025-12-24T02:10:52.259267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201433112784:2048] [node 1] ICS09 handshake done sender: [3:7587241201433112759:2105] self: [3:7587241201433112767:0] peer: [1:7587241200220127385:0] socket: 123 qp: -1
2025-12-24T02:10:52.259274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201433112784:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.259291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.262990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200220127410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15099
2025-12-24T02:10:52.263503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241201395208986:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241201395209068:2118] held: no
2025-12-24T02:10:52.262981Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241202102394991:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33516131ns) {Configure:104(33514450ns) {ProcessConfigured:135(33258055ns) {StartInitialHandshake:157(33257392ns) {DropHandshakes:470(478ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(1002ns) {SwitchToState:179(480ns)}}}}}
2025-12-24T02:10:52.266722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241200194245364:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.267930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245370:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.268015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245369:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.268395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209062:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.268544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112761:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.268581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241201433112702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.275133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.275191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813596:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.275226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813597:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.278859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209071:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
2025-12-24T02:10:52.279026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241201395209068:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.279132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127410:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.279171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241200220127388:2108] [node 4] ICP18 handshake (actor [1:7587241200220127410:2119]) is too old
2025-12-24T02:10:52.279201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209067:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.279316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241201395208986:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.279330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.279068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813595:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.279111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813596:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.279129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201329813578:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.279337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241201395208986:2048] [node 1] ICP22 created new session: [2:7587241201395209073:2048]
2025-12-24T02:10:52.279344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241201395209073:2048] [node 1] ICS09 handshake done sender: [2:7587241201395209068:2118] self: [2:7587241201395209069:0] peer: [1:7587241200220127382:0] socket: 155 qp: -1
2025-12-24T02:10:52.279351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241201395209073:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.279366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.279429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241200220127322:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(24297674ns) {HandleHandshakeStatus:329(24296684ns) {DropIncomingHandshake:442(2929ns) DropOutgoingHandshake:457(151ns) SwitchToState:179(93ns) ProcessPendingSessionEvents:495(24229004ns) {ForwardSessionEventToSession:612(24213424ns) ForwardSessionEventToSession:612(1866ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(923ns) ForwardSessionEventToSession:612(662ns) ForwardSessionEventToSession:612(2578ns) ForwardSessionEventToSession:612(717ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(537ns)}}}
2025-12-24T02:10:52.279585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241201329813517:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.279599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.279604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201329813517:2048] [node 1] ICP22 created new session: [4:7587241201329813598:2048]
2025-12-24T02:10:52.279608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201329813598:2048] [node 1] ICS09 handshake done sender: [4:7587241201329813578:2108] self: [4:7587241201329813590:0] peer: [1:7587241200220127390:0] socket: 124 qp: -1
2025-12-24T02:10:52.279613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201329813598:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.279618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.279741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241201329813518:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.279900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813594:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.280294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241201329813583:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:52.282242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209051:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.282260Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241201395209051:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.279844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241202102394990:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48013820ns) {Configure:104(48012778ns) {ProcessConfigured:135(47726028ns) {StartInitialHandshake:157(47725509ns) {DropHandshakes:470(412ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(734ns) {SwitchToState:179(464ns)}}}}}
2025-12-24T02:10:52.279869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202102394989:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241202102395061:2112] held: no
2025-12-24T02:10:52.282359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245370:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.282386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245369:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.282416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241200194245293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.287370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241201329813519:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39554955ns) {Configure:104(39554528ns) {ProcessConfigured:135(39253186ns) {StartInitialHandshake:157(39252996ns) {DropHandshakes:470(806ns) {DropIncomingHandshake:442(117ns) DropOutgoingHandshake:457(145ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(674ns) {SwitchToState:179(192ns)}}}}}
2025-12-24T02:10:52.287399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241201329813519:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.287640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200220127393:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.287652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200220127393:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.288708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200220127393:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.288886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200220127389:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.288897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.288906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200220127389:2109] [node 5] ICP22 created new session: [1:7587241200220127423:2109]
2025-12-24T02:10:52.288913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200220127423:2109] [node 5] ICS09 handshake done sender: [1:7587241200220127393:2112] self: [1:7587241200220127392:0] peer: [5:7587241200194245371:0] socket: 120 qp: -1
2025-12-24T02:10:52.288920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200220127423:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:52.288938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.286736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202102395059:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:52.288901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112778:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.288912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201433112778:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.289719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201433112778:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.289781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201433112771:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.289789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.289796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201433112771:2110] [node 4] ICP22 created new session: [3:7587241201433112789:2110]
2025-12-24T02:10:52.289801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201433112789:2110] [node 4] ICS09 handshake done sender: [3:7587241201433112778:2114] self: [3:7587241201433112776:0] peer: [4:7587241201329813603:0] socket: 136 qp: -1
2025-12-24T02:10:52.289806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201433112789:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:52.289812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.290798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241200194245364:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23979351ns) {Configure:104(23978526ns) {ProcessConfigured:135(23641065ns) {StartInitialHandshake:157(23640161ns) {DropHandshakes:470(738ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(466ns) PrepareNewSessionHandshake:477(1037ns) {SwitchToState:179(465ns)}}}}}
2025-12-24T02:10:52.291055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241200194245295:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54456532ns) {Configure:104(54456032ns) {ProcessConfigured:135(54148296ns) {StartInitialHandshake:157(54147866ns) {DropHandshakes:470(520ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:477(572ns) {SwitchToState:179(79ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.291605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209055:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.291622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241201395209055:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.293964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813604:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.294112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241201395209075:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58659
2025-12-24T02:10:52.294426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201329813595:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.291281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395069:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.291356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395070:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.294535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241201329813519:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.294550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.294556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201329813519:2050] [node 3] ICP22 created new session: [4:7587241201329813605:2050]
2025-12-24T02:10:52.294560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201329813605:2050] [node 3] ICS09 handshake done sender: [4:7587241201329813595:2115] self: [4:7587241201329813603:0] peer: [3:7587241201433112776:0] socket: 160 qp: -1
2025-12-24T02:10:52.294567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201329813605:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.294574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.298809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241201329813583:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18432998ns) {Configure:104(18432337ns) {ProcessConfigured:135(18151091ns) {StartInitialHandshake:157(18150429ns) {DropHandshakes:470(561ns) {DropIncomingHandshake:442(100ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(2052ns) {SwitchToState:179(1318ns)}}}}}
2025-12-24T02:10:52.302873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241201395209051:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.303714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202102395069:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.303738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202102395061:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.306884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202102394989:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.306901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.306909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202102394989:2048] [node 1] ICP22 created new session: [6:7587241202102395072:2048]
2025-12-24T02:10:52.306916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202102395072:2048] [node 1] ICS09 handshake done sender: [6:7587241202102395061:2112] self: [6:7587241202102395063:0] peer: [1:7587241200220127399:0] socket: 144 qp: -1
2025-12-24T02:10:52.306924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202102395072:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.306933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.306840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241200220127389:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(17916784ns) {HandleHandshakeStatus:329(17915882ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(179ns) SwitchToState:179(116ns) ProcessPendingSessionEvents:495(17815251ns) {ForwardSessionEventToSession:612(17795126ns) ForwardSessionEventToSession:612(3332ns) ForwardSessionEventToSession:612(1196ns) ForwardSessionEventToSession:612(776ns) ForwardSessionEventToSession:612(895ns) ForwardSessionEventToSession:612(1300ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(1045ns) ForwardSessionEventToSession:612(798ns) ForwardSessionEventToSession:612(1469ns)}}}
2025-12-24T02:10:52.309786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202102395059:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.306769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241201395208987:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.306806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.306816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241201395208987:2049] [node 3] ICP22 created new session: [2:7587241201395209078:2049]
2025-12-24T02:10:52.306823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241201395209078:2049] [node 3] ICS09 handshake done sender: [2:7587241201395209051:2109] self: [2:7587241201395209050:0] peer: [3:7587241201433112786:0] socket: 126 qp: -1
2025-12-24T02:10:52.306833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241201395209078:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.306865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.312311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209075:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.312786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245375:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.311595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395071:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.313296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202102395071:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.313326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395076:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.313347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202102394991:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.313042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201433112793:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-12-24T02:10:52.313080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241201433112792:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13610
2025-12-24T02:10:52.313585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112780:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.313590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201433112780:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.314132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201433112761:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.314283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241201433112702:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.314298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.314306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201433112702:2049] [node 2] ICP22 created new session: [3:7587241201433112795:2049]
2025-12-24T02:10:52.314312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201433112795:2049] [node 2] ICS09 handshake done sender: [3:7587241201433112761:2107] self: [3:7587241201433112786:0] peer: [2:7587241201395209050:0] socket: 127 qp: -1
2025-12-24T02:10:52.314320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201433112795:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.314326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.314818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201433112780:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.314927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241201395209055:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.311766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241200220127323:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.311794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7587241200220127323:2049] [node 3] ICP35 event processing took too much time StateWork:294(31501484ns) {HandleHandshakeStatus:395(31499888ns) {LogHandshakeFail:472(1017ns) {UpdateErrorStateLog:545(274ns)}}}
2025-12-24T02:10:52.318932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241200194245370:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.318894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241201395209049:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.318923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.318932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241201395209049:2106] [node 4] ICP22 created new session: [2:7587241201395209083:2106]
2025-12-24T02:10:52.318940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241201395209083:2106] [node 4] ICS09 handshake done sender: [2:7587241201395209055:2110] self: [2:7587241201395209054:0] peer: [4:7587241201329813600:0] socket: 139 qp: -1
2025-12-24T02:10:52.318948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241201395209083:2106] [node 4] ICS10 traffic start
2025-12-24T02:10:52.318957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.319011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241200194245293:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.319026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.319032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241200194245293:2048] [node 1] ICP22 created new session: [5:7587241200194245379:2048]
2025-12-24T02:10:52.319051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241200194245379:2048] [node 1] ICS09 handshake done sender: [5:7587241200194245370:2114] self: [5:7587241200194245371:0] peer: [1:7587241200220127392:0] socket: 128 qp: -1
2025-12-24T02:10:52.319058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241200194245379:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.319088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.321069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241201329813518:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241201329813596:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.323111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245378:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.326931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813589:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.326951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241201329813589:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.326901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241200220127389:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.330726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245376:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.330849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245378:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.330886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241200194245354:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.339054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245376:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.339101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241200194245295:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.340293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209065:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.339618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813609:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
2025-12-24T02:10:52.340109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201329813596:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.340174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241201329813518:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.340187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.340195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201329813518:2049] [node 2] ICP22 created new session: [4:7587241201329813610:2049]
2025-12-24T02:10:52.340203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201329813610:2049] [node 2] ICS09 handshake done sender: [4:7587241201329813596:2116] self: [4:7587241201329813600:0] peer: [2:7587241201395209054:0] socket: 162 qp: -1
2025-12-24T02:10:52.340212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201329813610:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.340220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.340656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813594:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.339405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395078:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.339664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395079:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.342470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112782:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.342483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241201433112782:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.342885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241201433112793:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.343470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201433112774:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.343492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.343504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201433112774:2112] [node 6] ICP22 created new session: [3:7587241201433112798:2112]
2025-12-24T02:10:52.343510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201433112798:2112] [node 6] ICS09 handshake done sender: [3:7587241201433112780:2115] self: [3:7587241201433112779:0] peer: [6:7587241202102395077:0] socket: 161 qp: -1
2025-12-24T02:10:52.343519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201433112798:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:52.343545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.343795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241201433112782:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.346812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202102395071:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.344378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245377:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.346728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813607:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.346750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241201329813607:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.348258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201329813607:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.348486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241201329813589:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.348609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202102395079:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.348609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241201329813585:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.348623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.348631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201329813585:2110] [node 6] ICP22 created new session: [4:7587241201329813615:2110]
2025-12-24T02:10:52.348638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201329813615:2110] [node 6] ICS09 handshake done sender: [4:7587241201329813589:2113] self: [4:7587241201329813588:0] peer: [6:7587241202102395074:0] socket: 149 qp: -1
2025-12-24T02:10:52.348645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201329813615:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:52.348654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.348710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241201433112773:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.348729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.348740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241201433112773:2111] [node 5] ICP22 created new session: [3:7587241201433112800:2111]
2025-12-24T02:10:52.348746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241201433112800:2111] [node 5] ICS09 handshake done sender: [3:7587241201433112782:2116] self: [3:7587241201433112775:0] peer: [5:7587241200194245383:0] socket: 166 qp: -1
2025-12-24T02:10:52.348752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241201433112800:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:52.348767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.349963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241201329813583:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.349986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.349992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241201329813583:2109] [node 5] ICP22 created new session: [4:7587241201329813617:2109]
2025-12-24T02:10:52.349999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241201329813617:2109] [node 5] ICS09 handshake done sender: [4:7587241201329813607:2120] self: [4:7587241201329813601:0] peer: [5:7587241200194245381:0] socket: 138 qp: -1
2025-12-24T02:10:52.350005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241201329813617:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:52.350013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.351579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209064:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.351591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241201395209064:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.353315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241201395209064:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.353501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241201395209061:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.353517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.353524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241201395209061:2112] [node 6] ICP22 created new session: [2:7587241201395209087:2112]
2025-12-24T02:10:52.353529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241201395209087:2112] [node 6] ICS09 handshake done sender: [2:7587241201395209064:2115] self: [2:7587241201395209063:0] peer: [6:7587241202102395084:0] socket: 174 qp: -1
2025-12-24T02:10:52.353534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241201395209087:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:52.353542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.350353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202102395049:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.350462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202102394991:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.350471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.350477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202102394991:2050] [node 3] ICP22 created new session: [6:7587241202102395081:2050]
2025-12-24T02:10:52.350483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202102395081:2050] [node 3] ICS09 handshake done sender: [6:7587241202102395071:2118] self: [6:7587241202102395077:0] peer: [3:7587241201433112779:0] socket: 170 qp: -1
2025-12-24T02:10:52.350489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202102395081:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.350508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.351358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202102395076:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.351469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202102394990:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.354461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202102394990:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241202102395076:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.355508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245377:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.355533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241200194245354:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241200194245378:2120] held: no
2025-12-24T02:10:52.355617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241200194245294:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.355653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245384:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.355863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245385:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.356078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241200194245376:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.356160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241200194245378:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.356217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241200194245295:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.356231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.356237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241200194245295:2050] [node 3] ICP22 created new session: [5:7587241200194245387:2050]
2025-12-24T02:10:52.356242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241200194245387:2050] [node 3] ICS09 handshake done sender: [5:7587241200194245376:2118] self: [5:7587241200194245383:0] peer: [3:7587241201433112775:0] socket: 175 qp: -1
2025-12-24T02:10:52.356249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241200194245387:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.356268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.356437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241200194245354:2105] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.356445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.356451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241200194245354:2105] [node 4] ICP22 created new session: [5:7587241200194245389:2105]
2025-12-24T02:10:52.356459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241200194245389:2105] [node 4] ICS09 handshake done sender: [5:7587241200194245378:2120] self: [5:7587241200194245381:0] peer: [4:7587241201329813601:0] socket: 177 qp: -1
2025-12-24T02:10:52.356464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241200194245389:2105] [node 4] ICS10 traffic start
2025-12-24T02:10:52.356469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.358976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241201329813619:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6158
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.360311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241200194245372:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.360327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241200194245372:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.361177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241201433112774:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(46220643ns) {HandleHandshakeStatus:329(46220043ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(803ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(12449160ns) {ForwardSessionEventToSession:612(12437392ns) ForwardSessionEventToSession:612(2001ns) ForwardSessionEventToSession:612(801ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(969ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(513ns) ForwardSessionEventToSession:612(618ns) ForwardSessionEventToSession:612(803ns)}}}
2025-12-24T02:10:52.361204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241201433112774:2112] [node 6] ICP18 handshake (actor [3:7587241201433112793:2119]) is too old
2025-12-24T02:10:52.359094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241201395209060:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.359106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241201395209060:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.362877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241200194245392:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20722
2025-12-24T02:10:52.362877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241200194245372:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.363346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241201329813619:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.363391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241201329813585:2110] [node 6] ICP18 handshake (actor [4:7587241201329813619:2122]) is too old
2025-12-24T02:10:52.363506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241200194245364:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.363525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.363534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241200194245364:2110] [node 6] ICP22 created new session: [5:7587241200194245393:2110]
2025-12-24T02:10:52.363540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241200194245393:2110] [node 6] ICS09 handshake done sender: [5:7587241200194245372:2115] self: [5:7587241200194245368:0] peer: [6:7587241202102395080:0] socket: 114 qp: -1
2025-12-24T02:10:52.363546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241200194245393:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:52.363552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.362916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241201395209060:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.363072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241201395209057:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.363089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.363097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241201395209057:2111] [node 5] ICP22 created new session: [2:7587241201395209090:2111]
2025-12-24T02:10:52.363103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241201395209090:2111] [node 5] ICS09 handshake done sender: [2:7587241201395209060:2113] self: [2:7587241201395209059:0] peer: [5:7587241200194245386:0] socket: 125 qp: -1
2025-12-24T02:10:52.363110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241201395209090:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:52.363132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.364714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241200194245377:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.360119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395085:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.360227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395086:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.363215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202102395087:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53874
2025-12-24T02:10:52.365991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202102395059:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241202102395069:2116] held: no
2025-12-24T02:10:52.366029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202102395069:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.366078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202102395059:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.366086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.366092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202102395059:2110] [node 4] ICP22 created new session: [6:7587241202102395088:2110]
2025-12-24T02:10:52.366097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202102395088:2110] [node 4] ICS09 handshake done sender: [6:7587241202102395069:2116] self: [6:7587241202102395074:0] peer: [4:7587241201329813588:0] socket: 146 qp: -1
2025-12-24T02:10:52.366104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202102395088:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:52.366111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.366223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202102395049:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62032) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241202102395079:2121] held: no
2025-12-24T02:10:52.366241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202102395076:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.366410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202102394990:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.366419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.366424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202102394990:2049] [node 2] ICP22 created new session: [6:7587241202102395090:2049]
2025-12-24T02:10:52.366429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202102395090:2049] [node 2] ICS09 handshake done sender: [6:7587241202102395076:2119] self: [6:7587241202102395084:0] peer: [2:7587241201395209063:0] socket: 179 qp: -1
2025-12-24T02:10:52.366436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202102395090:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.366441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.366585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202102395079:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.366640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202102395049:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.366647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.366652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202102395049:2106] [node 5] ICP22 created new session: [6:7587241202102395092:2106]
2025-12-24T02:10:52.366656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202102395092:2106] [node 5] ICS09 handshake done sender: [6:7587241202102395079:2121] self: [6:7587241202102395080:0] peer: [5:7587241200194245368:0] socket: 131 qp: -1
2025-12-24T02:10:52.366661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202102395092:2106] [node 5] ICS10 traffic start
2025-12-24T02:10:52.366666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.370819Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241200194245294:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241200194245377:2119] held: no
2025-12-24T02:10:52.373857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241200194245294:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.373878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.373888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241200194245294:2049] [node 2] ICP22 created new session: [5:7587241200194245395:2049]
2025-12-24T02:10:52.373893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241200194245395:2049] [node 2] ICS09 handshake done sender: [5:7587241200194245377:2119] self: [5:7587241200194245386:0] peer: [2:7587241201395209059:0] socket: 176 qp: -1
2025-12-24T02:10:52.373903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241200194245395:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.373913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.384238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:52.384332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241201395209073:2048] [node 1] ICS01 socket: 155 reason# EndOfStream
2025-12-24T02:10:52.384340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241201395208986:2048] [node 1] ICP30 unregister session Session# [2:7587241201395209073:2048] VirtualId# [2:7587241201395209069:0]
2025-12-24T02:10:52.384347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241201395209073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.384393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.384413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.394797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:52.394899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241201433112784:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-12-24T02:10:52.394910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241201433112701:2048] [node 1] ICP30 unregister session Session# [3:7587241201433112784:2048] VirtualId# [3:7587241201433112767:0]
2025-12-24T02:10:52.394917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241201433112784:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.394979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.395009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.398784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:52.398885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241200194245379:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-12-24T02:10:52.398893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241200194245293:2048] [node 1] ICP30 unregister session Session# [5:7587241200194245379:2048] VirtualId# [5:7587241200194245371:0]
2025-12-24T02:10:52.398901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241200194245379:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.398970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.398997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.398843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:52.398911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241201329813598:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-12-24T02:10:52.398918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241201329813517:2048] [node 1] ICP30 unregister session Session# [4:7587241201329813598:2048] VirtualId# [4:7587241201329813590:0]
2025-12-24T02:10:52.398923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241201329813598:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.398969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.398990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.399021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:52.399089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202102395072:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-12-24T02:10:52.399093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202102394989:2048] [node 1] ICP30 unregister session Session# [6:7587241202102395072:2048] VirtualId# [6:7587241202102395063:0]
2025-12-24T02:10:52.399097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202102395072:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.399122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.399142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.406772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:52.406866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241200194245395:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-12-24T02:10:52.406874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241200194245294:2049] [node 2] ICP30 unregister session Session# [5:7587241200194245395:2049] VirtualId# [5:7587241200194245386:0]
2025-12-24T02:10:52.406880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241200194245395:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.406937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.406963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.406821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:52.406893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241201433112795:2049] [node 2] ICS01 socket: 127 reason# EndOfStream
2025-12-24T02:10:52.406900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241201433112702:2049] [node 2] ICP30 unregister session Session# [3:7587241201433112795:2049] VirtualId# [3:7587241201433112786:0]
2025-12-24T02:10:52.406905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241201433112795:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.406945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.406965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.406994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:52.407061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241201329813610:2049] [node 2] ICS01 socket: 162 reason# EndOfStream
2025-12-24T02:10:52.407066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241201329813518:2049] [node 2] ICP30 unregister session Session# [4:7587241201329813610:2049] VirtualId# [4:7587241201329813600:0]
2025-12-24T02:10:52.407070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241201329813610:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.407093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.407113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.407138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:52.407207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202102395090:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-12-24T02:10:52.407211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202102394990:2049] [node 2] ICP30 unregister session Session# [6:7587241202102395090:2049] VirtualId# [6:7587241202102395084:0]
2025-12-24T02:10:52.407215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202102395090:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.407249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.407268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.426796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:52.426878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241200194245387:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-12-24T02:10:52.426886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241200194245295:2050] [node 3] ICP30 unregister session Session# [5:7587241200194245387:2050] VirtualId# [5:7587241200194245383:0]
2025-12-24T02:10:52.426892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241200194245387:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.426940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.426966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.426844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:52.426932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202102395081:2050] [node 3] ICS01 socket: 170 reason# EndOfStream
2025-12-24T02:10:52.426939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202102394991:2050] [node 3] ICP30 unregister session Session# [6:7587241202102395081:2050] VirtualId# [6:7587241202102395077:0]
2025-12-24T02:10:52.426945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202102395081:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.427000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.427025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.442914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:52.443000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202102395088:2110] [node 4] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:10:52.443009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202102395059:2110] [node 4] ICP30 unregister session Session# [6:7587241202102395088:2110] VirtualId# [6:7587241202102395074:0]
2025-12-24T02:10:52.443015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202102395088:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.443101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:52.443130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:52.481756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241202637597064:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.481764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241202637597065:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.481768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241202637597066:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.477716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241202097350163:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.477725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241202097350164:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.487911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202332552293:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.487920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202332552294:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.487926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202332552295:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.480033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199781358355:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.480040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199781358356:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.475701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200814252148:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:52.475707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200814252149:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.483946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203567059919:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.483954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203567059920:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.483958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203567059921:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.500317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202332552355:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:52.513388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200814252217:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:52.513770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200814252218:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.518899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203567059985:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:52.520483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203567059994:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:52.521060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567059983:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.518944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358422:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.522711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200814252219:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.527584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358422:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.528725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241199781358355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.529265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350230:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
2025-12-24T02:10:52.529315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350231:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
2025-12-24T02:10:52.529341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350232:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
2025-12-24T02:10:52.534779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241200814252218:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20953256ns) {Configure:104(20952852ns) {ProcessConfigured:135(20770216ns) {StartInitialHandshake:157(20769840ns) {DropHandshakes:470(205ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:477(975ns) {SwitchToState:179(427ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.535188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567059997:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.536210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252212:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.536224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200814252212:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.536249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241202637597129:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:52.536348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567059997:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.536372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241202097350229:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:52.536461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350231:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.536490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350230:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.536619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241202097350228:2106] [node 4] ICP01 ready to work
2025-12-24T02:10:52.538780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241202637597131:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:52.539290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241202097350227:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.539611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597138:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.540144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200814252226:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3393
2025-12-24T02:10:52.540163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200814252227:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3393
2025-12-24T02:10:52.540175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200814252225:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3393
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.541982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241202637597065:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24236952ns) {Configure:104(24235887ns) {ProcessConfigured:135(23969397ns) {StartInitialHandshake:157(23969054ns) {DropHandshakes:470(474ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(694ns) {SwitchToState:179(366ns)}}}}}
2025-12-24T02:10:52.538729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199781358423:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:52.539124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199781358424:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:52.539998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358429:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.540010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358430:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.540020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358431:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.544141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358433:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.543435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241202332552293:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36998809ns) {Configure:104(36998099ns) {ProcessConfigured:135(36753748ns) {StartInitialHandshake:157(36753529ns) {DropHandshakes:470(424ns) {DropIncomingHandshake:442(93ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(390ns) {SwitchToState:179(197ns)}}}}}
2025-12-24T02:10:52.540393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252225:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.540482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241200814252219:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17715195ns) {Configure:104(17714551ns) {ProcessConfigured:135(17508985ns) {StartInitialHandshake:157(17508751ns) {DropHandshakes:470(462ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(131ns) PrepareNewSessionHandshake:477(574ns) {SwitchToState:179(62ns)}}}}}
2025-12-24T02:10:52.542200Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241200814252217:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28748809ns) {Configure:104(28748028ns) {ProcessConfigured:135(28413843ns) {StartInitialHandshake:157(28413667ns) {DropHandshakes:470(473ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(545ns) {SwitchToState:179(299ns)}}}}}
2025-12-24T02:10:52.542625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252223:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.542631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200814252223:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.540446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241202097350163:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38513143ns) {Configure:104(38512420ns) {ProcessConfigured:135(38223957ns) {StartInitialHandshake:157(38223489ns) {DropHandshakes:470(318ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(37ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1063ns) {SwitchToState:179(414ns)}}}}}
2025-12-24T02:10:52.540478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241202097350163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.540652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350239:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.547595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
2025-12-24T02:10:52.547708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252209:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.547716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200814252209:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.547871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200814252212:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.548842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200814252149:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.548865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.548872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200814252149:2049] [node 3] ICP22 created new session: [1:7587241200814252233:2049]
2025-12-24T02:10:52.548879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200814252233:2049] [node 3] ICS09 handshake done sender: [1:7587241200814252212:2108] self: [1:7587241200814252211:0] peer: [3:7587241199781358427:0] socket: 125 qp: -1
2025-12-24T02:10:52.548887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200814252233:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.548907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.549013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200814252209:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.549340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200814252148:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.549352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.549359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200814252148:2048] [node 2] ICP22 created new session: [1:7587241200814252235:2048]
2025-12-24T02:10:52.549364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200814252235:2048] [node 2] ICS09 handshake done sender: [1:7587241200814252209:2107] self: [1:7587241200814252208:0] peer: [2:7587241202097350241:0] socket: 120 qp: -1
2025-12-24T02:10:52.549370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200814252235:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:52.549386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.550069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567059983:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.550139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597138:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.548654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199781358422:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.550834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552369:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.551413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350232:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.551440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350239:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.550776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252227:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.554055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200814252223:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.554232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200814252218:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.554251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.554257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200814252218:2110] [node 5] ICP22 created new session: [1:7587241200814252241:2110]
2025-12-24T02:10:52.554263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200814252241:2110] [node 5] ICS09 handshake done sender: [1:7587241200814252223:2112] self: [1:7587241200814252221:0] peer: [5:7587241203567059998:0] socket: 146 qp: -1
2025-12-24T02:10:52.554269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200814252241:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:52.554276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.551018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358430:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.551060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241199781358355:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.552091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203567059919:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.551073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.551079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199781358355:2048] [node 1] ICP22 created new session: [3:7587241199781358434:2048]
2025-12-24T02:10:52.552495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241203567059994:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31958700ns) {Configure:104(31958189ns) {ProcessConfigured:135(31783093ns) {StartInitialHandshake:157(31782673ns) {DropHandshakes:470(317ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(36ns) PrepareNewSessionHandshake:477(479ns) {SwitchToState:179(18ns)}}}}}
2025-12-24T02:10:52.551085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199781358434:2048] [node 1] ICS09 handshake done sender: [3:7587241199781358422:2111] self: [3:7587241199781358427:0] peer: [1:7587241200814252211:0] socket: 128 qp: -1
2025-12-24T02:10:52.551092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199781358434:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.551100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.551110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358429:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.551180Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241199781358355:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.552018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199781358436:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:52.551856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241202332552294:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20827864ns) {Configure:104(20826885ns) {ProcessConfigured:135(20504557ns) {StartInitialHandshake:157(20504097ns) {DropHandshakes:470(546ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1518ns) {SwitchToState:179(585ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.555062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552370:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.558741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241203567059919:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16653) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241203567059997:2115] held: no
2025-12-24T02:10:52.558915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252229:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.558746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202332552369:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.558927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200814252229:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.558794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202332552293:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.559263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350234:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.559271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241202097350234:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.560237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241202097350234:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.560611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241202097350164:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.560631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.560637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241202097350164:2049] [node 3] ICP22 created new session: [2:7587241202097350252:2049]
2025-12-24T02:10:52.560644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241202097350252:2049] [node 3] ICS09 handshake done sender: [2:7587241202097350234:2113] self: [2:7587241202097350233:0] peer: [3:7587241199781358443:0] socket: 144 qp: -1
2025-12-24T02:10:52.560649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241202097350252:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.560666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.560925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252226:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.558830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241199781358424:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19640081ns) {Configure:104(19639694ns) {ProcessConfigured:135(15789671ns) {StartInitialHandshake:157(15788835ns) {DropHandshakes:470(587ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:477(1180ns) {SwitchToState:179(639ns)}}}}}
2025-12-24T02:10:52.559107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202332552370:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.559155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358431:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.559155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202332552295:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.559177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241199781358356:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.560639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241202332552367:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:52.560500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.560885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199781358431:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.562733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358438:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.562744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199781358438:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.562742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597142:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.562834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350248:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.566879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241202637597131:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28019770ns) {Configure:104(28019057ns) {ProcessConfigured:135(27698054ns) {StartInitialHandshake:157(27697416ns) {DropHandshakes:470(431ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:477(638ns) {SwitchToState:179(237ns)}}}}}
2025-12-24T02:10:52.566941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241199781358356:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.566960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.566968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199781358356:2049] [node 2] ICP22 created new session: [3:7587241199781358445:2049]
2025-12-24T02:10:52.566974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199781358445:2049] [node 2] ICS09 handshake done sender: [3:7587241199781358431:2115] self: [3:7587241199781358443:0] peer: [2:7587241202097350233:0] socket: 151 qp: -1
2025-12-24T02:10:52.566724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060001:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.566981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199781358445:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.566989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.572154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.572447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597142:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.572488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241202637597064:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.572713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252230:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.572727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200814252230:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.572792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597145:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.572833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241202637597066:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.573452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.573648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200814252230:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.573861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241202637597142:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.570811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241199781358423:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31992444ns) {Configure:104(31991669ns) {ProcessConfigured:135(31675085ns) {StartInitialHandshake:157(31674534ns) {DropHandshakes:470(641ns) {DropIncomingHandshake:442(98ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:477(576ns) {SwitchToState:179(169ns)}}}}}
2025-12-24T02:10:52.573983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241202637597064:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.574002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.574010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241202637597064:2048] [node 1] ICP22 created new session: [4:7587241202637597149:2048]
2025-12-24T02:10:52.574016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241202637597149:2048] [node 1] ICS09 handshake done sender: [4:7587241202637597142:2115] self: [4:7587241202637597146:0] peer: [1:7587241200814252220:0] socket: 164 qp: -1
2025-12-24T02:10:52.574023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241202637597149:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.574056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.575944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200814252217:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.575963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.575973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200814252217:2109] [node 4] ICP22 created new session: [1:7587241200814252245:2109]
2025-12-24T02:10:52.575979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200814252245:2109] [node 4] ICS09 handshake done sender: [1:7587241200814252230:2117] self: [1:7587241200814252220:0] peer: [4:7587241202637597146:0] socket: 163 qp: -1
2025-12-24T02:10:52.575985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200814252245:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:52.575993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.579717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200814252229:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.580337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200814252219:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.580350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.580355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200814252219:2111] [node 6] ICP22 created new session: [1:7587241200814252249:2111]
2025-12-24T02:10:52.580360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200814252249:2111] [node 6] ICS09 handshake done sender: [1:7587241200814252229:2116] self: [1:7587241200814252222:0] peer: [6:7587241202332552371:0] socket: 140 qp: -1
2025-12-24T02:10:52.580366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200814252249:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:52.580382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.580536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241202332552367:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19818790ns) {Configure:104(19818032ns) {ProcessConfigured:135(19570867ns) {StartInitialHandshake:157(19570369ns) {DropHandshakes:470(347ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(161ns) PrepareNewSessionHandshake:477(665ns) {SwitchToState:179(526ns)}}}}}
2025-12-24T02:10:52.578765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358447:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.578786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199781358447:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.582933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567060001:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.582984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203567059920:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.583204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350244:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.583216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241202097350244:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.584016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241202097350244:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.584104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241202097350227:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.584120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.584127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241202097350227:2107] [node 5] ICP22 created new session: [2:7587241202097350257:2107]
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.586195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552375:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.586264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552376:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.586597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552377:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.583474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199781358438:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.586750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202332552369:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.586790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202332552370:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.586901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202332552293:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.586917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.586924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202332552293:2048] [node 1] ICP22 created new session: [6:7587241202332552378:2048]
2025-12-24T02:10:52.586931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202332552378:2048] [node 1] ICS09 handshake done sender: [6:7587241202332552369:2114] self: [6:7587241202332552371:0] peer: [1:7587241200814252222:0] socket: 166 qp: -1
2025-12-24T02:10:52.586939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202332552378:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.586959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.592120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241202097350259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24668
2025-12-24T02:10:52.592425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200814252248:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3393
2025-12-24T02:10:52.592423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.592471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241202097350257:2107] [node 5] ICS09 handshake done sender: [2:7587241202097350244:2116] self: [2:7587241202097350243:0] peer: [5:7587241203567060003:0] socket: 156 qp: -1
2025-12-24T02:10:52.592485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241202097350257:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:52.592498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.592587Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241202097350164:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(31941693ns) {HandleHandshakeStatus:329(31940990ns) {DropIncomingHandshake:442(3506ns) DropOutgoingHandshake:457(135ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(13379441ns) {ForwardSessionEventToSession:612(13368357ns) ForwardSessionEventToSession:612(1798ns) ForwardSessionEventToSession:612(812ns) ForwardSessionEventToSession:612(481ns) ForwardSessionEventToSession:612(486ns) ForwardSessionEventToSession:612(905ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(758ns)}}}
2025-12-24T02:10:52.592774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241202097350230:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.590823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199781358436:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.590848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.590858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199781358436:2110] [node 6] ICP22 created new session: [3:7587241199781358451:2110]
2025-12-24T02:10:52.590865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199781358451:2110] [node 6] ICS09 handshake done sender: [3:7587241199781358438:2117] self: [3:7587241199781358437:0] peer: [6:7587241202332552372:0] socket: 130 qp: -1
2025-12-24T02:10:52.590873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199781358451:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:52.590882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.593840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200814252248:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.593882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241200814252148:2048] [node 2] ICP18 handshake (actor [1:7587241200814252248:2118]) is too old
2025-12-24T02:10:52.594743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241203567059920:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241203567060001:2117] held: no
2025-12-24T02:10:52.594814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203567059997:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.600792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241202637597066:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9795) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241202637597145:2117] held: no
2025-12-24T02:10:52.599715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199781358447:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.599822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199781358423:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.599839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.599850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199781358423:2108] [node 4] ICP22 created new session: [3:7587241199781358454:2108]
2025-12-24T02:10:52.599856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199781358454:2108] [node 4] ICS09 handshake done sender: [3:7587241199781358447:2120] self: [3:7587241199781358428:0] peer: [4:7587241202637597147:0] socket: 174 qp: -1
2025-12-24T02:10:52.599863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199781358454:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:52.599869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.602879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.602975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.603033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060007:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.603104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203567059919:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.603121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.603128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203567059919:2048] [node 1] ICP22 created new session: [5:7587241203567060008:2048]
2025-12-24T02:10:52.603135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203567060008:2048] [node 1] ICS09 handshake done sender: [5:7587241203567059997:2115] self: [5:7587241203567059998:0] peer: [1:7587241200814252221:0] socket: 147 qp: -1
2025-12-24T02:10:52.603142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203567060008:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.603161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.604087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202332552375:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.604874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202332552294:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.604916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552380:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.604964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552381:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.605225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350246:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.605237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241202097350246:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.605722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203567060001:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.603415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199781358456:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17190
2025-12-24T02:10:52.606836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241202097350246:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.606797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202332552380:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.606895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202332552367:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.606944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241202097350229:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.606962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.606968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241202097350229:2108] [node 6] ICP22 created new session: [2:7587241202097350262:2108]
2025-12-24T02:10:52.606973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241202097350262:2108] [node 6] ICS09 handshake done sender: [2:7587241202097350246:2117] self: [2:7587241202097350237:0] peer: [6:7587241202332552383:0] socket: 172 qp: -1
2025-12-24T02:10:52.606978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241202097350262:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:52.606986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.607016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202332552295:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.607030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.607034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202332552295:2050] [node 3] ICP22 created new session: [6:7587241202332552386:2050]
2025-12-24T02:10:52.607039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202332552386:2050] [node 3] ICS09 handshake done sender: [6:7587241202332552370:2115] self: [6:7587241202332552372:0] peer: [3:7587241199781358437:0] socket: 153 qp: -1
2025-12-24T02:10:52.607058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202332552386:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.607085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.607440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552384:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.611200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241202097350163:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.611222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.611231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241202097350163:2048] [node 1] ICP22 created new session: [2:7587241202097350264:2048]
2025-12-24T02:10:52.611237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241202097350264:2048] [node 1] ICS09 handshake done sender: [2:7587241202097350230:2110] self: [2:7587241202097350241:0] peer: [1:7587241200814252208:0] socket: 129 qp: -1
2025-12-24T02:10:52.611246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241202097350264:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.611254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.611479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567060005:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.611554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567060006:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.611564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203567059920:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.611576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.611584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203567059920:2049] [node 2] ICP22 created new session: [5:7587241203567060012:2049]
2025-12-24T02:10:52.611590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203567060012:2049] [node 2] ICS09 handshake done sender: [5:7587241203567060001:2117] self: [5:7587241203567060003:0] peer: [2:7587241202097350243:0] socket: 157 qp: -1
2025-12-24T02:10:52.611595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203567059985:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.611599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203567060012:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.611607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.611650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203567059921:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.613717Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241202097350227:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(29587514ns) {HandleHandshakeStatus:329(29586797ns) {DropIncomingHandshake:442(3183ns) DropOutgoingHandshake:457(181ns) SwitchToState:179(64ns) ProcessPendingSessionEvents:495(6608903ns) {ForwardSessionEventToSession:612(6598248ns) ForwardSessionEventToSession:612(1097ns) ForwardSessionEventToSession:612(796ns) ForwardSessionEventToSession:612(760ns) ForwardSessionEventToSession:612(480ns) ForwardSessionEventToSession:612(922ns) ForwardSessionEventToSession:612(520ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(1195ns)}}}
2025-12-24T02:10:52.613874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241202097350163:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.612147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241202332552293:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25215106ns) {HandleHandshakeStatus:329(25214865ns) {DropIncomingHandshake:442(243ns) DropOutgoingHandshake:457(1559ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(17213708ns) {ForwardSessionEventToSession:612(17204659ns) ForwardSessionEventToSession:612(1145ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(458ns) ForwardSessionEventToSession:612(397ns) ForwardSessionEventToSession:612(946ns) ForwardSessionEventToSession:612(460ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(1143ns)}}}
2025-12-24T02:10:52.612209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202332552375:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.611820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.611849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597153:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.611885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.614849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597141:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.614854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241202637597141:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.615268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597144:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.615272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241202637597144:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.615826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597153:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.616027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241202637597065:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.616270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241202637597144:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.616351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241202637597131:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.616361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.616368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241202637597131:2108] [node 6] ICP22 created new session: [4:7587241202637597159:2108]
2025-12-24T02:10:52.616374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241202637597159:2108] [node 6] ICS09 handshake done sender: [4:7587241202637597144:2116] self: [4:7587241202637597137:0] peer: [6:7587241202332552385:0] socket: 132 qp: -1
2025-12-24T02:10:52.616380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241202637597159:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:52.616387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.616775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241202637597141:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.617184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060015:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.617244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203567060016:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52720
2025-12-24T02:10:52.617551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203567060005:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.617667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203567060006:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.617758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203567059921:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.617776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.617783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203567059921:2050] [node 3] ICP22 created new session: [5:7587241203567060017:2050]
2025-12-24T02:10:52.617794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203567060017:2050] [node 3] ICS09 handshake done sender: [5:7587241203567060005:2119] self: [5:7587241203567060014:0] peer: [3:7587241199781358432:0] socket: 142 qp: -1
2025-12-24T02:10:52.617800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203567060017:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.617806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.617896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203567059985:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.617903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.617908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203567059985:2108] [node 4] ICP22 created new session: [5:7587241203567060019:2108]
2025-12-24T02:10:52.617912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203567060019:2108] [node 4] ICS09 handshake done sender: [5:7587241203567060006:2120] self: [5:7587241203567060011:0] peer: [4:7587241202637597136:0] socket: 139 qp: -1
2025-12-24T02:10:52.617916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203567060019:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:52.617920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.615558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358456:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.616469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199781358441:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.616475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199781358441:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.617385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199781358441:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.618732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199781358424:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.618757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.618764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199781358424:2109] [node 5] ICP22 created new session: [3:7587241199781358459:2109]
2025-12-24T02:10:52.618769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199781358459:2109] [node 5] ICS09 handshake done sender: [3:7587241199781358441:2118] self: [3:7587241199781358432:0] peer: [5:7587241203567060014:0] socket: 168 qp: -1
2025-12-24T02:10:52.616285Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241202332552295:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(29359981ns) {HandleHandshakeStatus:329(29359465ns) {DropIncomingHandshake:442(428ns) DropOutgoingHandshake:457(883ns) SwitchToState:179(37ns) ProcessPendingSessionEvents:495(2886566ns) {ForwardSessionEventToSession:612(2875378ns) ForwardSessionEventToSession:612(1673ns) ForwardSessionEventToSession:612(914ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(1474ns) ForwardSessionEventToSession:612(624ns) ForwardSessionEventToSession:612(837ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(1051ns)}}}
2025-12-24T02:10:52.616311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241202332552295:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.618965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202332552294:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.618980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.618993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202332552294:2049] [node 2] ICP22 created new session: [6:7587241202332552388:2049]
2025-12-24T02:10:52.618998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202332552388:2049] [node 2] ICS09 handshake done sender: [6:7587241202332552375:2117] self: [6:7587241202332552383:0] peer: [2:7587241202097350237:0] socket: 173 qp: -1
2025-12-24T02:10:52.619003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202332552388:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.619023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.619125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241202637597129:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.619141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.619146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241202637597129:2107] [node 5] ICP22 created new session: [4:7587241202637597161:2107]
2025-12-24T02:10:52.619152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241202637597161:2107] [node 5] ICS09 handshake done sender: [4:7587241202637597141:2114] self: [4:7587241202637597136:0] peer: [5:7587241203567060011:0] socket: 177 qp: -1
2025-12-24T02:10:52.619156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241202637597161:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:52.619162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.619302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241202637597155:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.619320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241202637597145:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.619378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241202637597066:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.619387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.619393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241202637597066:2050] [node 3] ICP22 created new session: [4:7587241202637597163:2050]
2025-12-24T02:10:52.619402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241202637597163:2050] [node 3] ICS09 handshake done sender: [4:7587241202637597145:2117] self: [4:7587241202637597147:0] peer: [3:7587241199781358428:0] socket: 175 qp: -1
2025-12-24T02:10:52.619408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241202637597163:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.619423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.619157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241202332552381:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.619331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241202332552355:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.618782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199781358459:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:52.618797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.622881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241202637597131:2108] [node 6] ICP18 handshake (actor [4:7587241202637597155:2121]) is too old
2025-12-24T02:10:52.625224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203567059999:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.625240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241203567059999:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.626724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241202097350247:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.626745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241202097350247:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.627386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202332552367:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11637) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241202332552380:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.634876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552391:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
2025-12-24T02:10:52.634918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241202332552392:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55675
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.635149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202332552380:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.634897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241202097350247:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.635219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241202097350228:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.635238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.635246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241202097350228:2106] [node 4] ICP22 created new session: [2:7587241202097350267:2106]
2025-12-24T02:10:52.635251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241202097350267:2106] [node 4] ICS09 handshake done sender: [2:7587241202097350247:2118] self: [2:7587241202097350238:0] peer: [4:7587241202637597158:0] socket: 143 qp: -1
2025-12-24T02:10:52.635257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241202097350267:2106] [node 4] ICS10 traffic start
2025-12-24T02:10:52.635272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.635361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203567059999:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.635283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202332552367:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.635297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.635303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202332552367:2110] [node 4] ICP22 created new session: [6:7587241202332552393:2110]
2025-12-24T02:10:52.635310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202332552393:2110] [node 4] ICS09 handshake done sender: [6:7587241202332552380:2120] self: [6:7587241202332552385:0] peer: [4:7587241202637597137:0] socket: 184 qp: -1
2025-12-24T02:10:52.635316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202332552393:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:52.635326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.635716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241202332552381:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.635798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241202332552355:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.635816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.635821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241202332552355:2108] [node 5] ICP22 created new session: [6:7587241202332552395:2108]
2025-12-24T02:10:52.635826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241202332552395:2108] [node 5] ICS09 handshake done sender: [6:7587241202332552381:2121] self: [6:7587241202332552390:0] peer: [5:7587241203567059995:0] socket: 185 qp: -1
2025-12-24T02:10:52.635831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241202332552395:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:52.635840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.635061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241202637597165:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29997
2025-12-24T02:10:52.638924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241203567059994:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.638952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.638960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203567059994:2109] [node 6] ICP22 created new session: [5:7587241203567060022:2109]
2025-12-24T02:10:52.638966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203567060022:2109] [node 6] ICS09 handshake done sender: [5:7587241203567059999:2116] self: [5:7587241203567059995:0] peer: [6:7587241202332552390:0] socket: 133 qp: -1
2025-12-24T02:10:52.638972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203567060022:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:52.638985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.640257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241202637597065:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23648) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241202637597153:2119] held: no
2025-12-24T02:10:52.642912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241202637597153:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.650794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241202637597065:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.650816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.650825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241202637597065:2049] [node 2] ICP22 created new session: [4:7587241202637597166:2049]
2025-12-24T02:10:52.650832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241202637597166:2049] [node 2] ICS09 handshake done sender: [4:7587241202637597153:2119] self: [4:7587241202637597158:0] peer: [2:7587241202097350238:0] socket: 150 qp: -1
2025-12-24T02:10:52.650841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241202637597166:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.650862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.663648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:52.663732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241199781358434:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-12-24T02:10:52.663743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241199781358355:2048] [node 1] ICP30 unregister session Session# [3:7587241199781358434:2048] VirtualId# [3:7587241199781358427:0]
2025-12-24T02:10:52.663748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241199781358434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.663774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.663802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.663431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:52.663518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241202097350264:2048] [node 1] ICS01 socket: 129 reason# EndOfStream
2025-12-24T02:10:52.663529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241202097350163:2048] [node 1] ICP30 unregister session Session# [2:7587241202097350264:2048] VirtualId# [2:7587241202097350241:0]
2025-12-24T02:10:52.663536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241202097350264:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.663585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.663615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.670244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:52.670355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241202332552378:2048] [node 1] ICS07 socket disconnect 166 reason# EndOfStream
2025-12-24T02:10:52.670360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202332552378:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.670425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.670449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241202332552378:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:52.670797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:52.670876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203567060008:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-12-24T02:10:52.670883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203567059919:2048] [node 1] ICP30 unregister session Session# [5:7587241203567060008:2048] VirtualId# [5:7587241203567059998:0]
2025-12-24T02:10:52.670890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203567060008:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.670941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.670966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.670996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202332552293:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16653) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.671009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202332552378:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.671014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202332552293:2048] [node 1] ICP30 unregister session Session# [6:7587241202332552378:2048] VirtualId# [6:7587241202332552371:0]
2025-12-24T02:10:52.671030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.671036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241202332552293:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.671040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16653) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.674807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:52.674887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241202637597149:2048] [node 1] ICS01 socket: 164 reason# EndOfStream
2025-12-24T02:10:52.674895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241202637597064:2048] [node 1] ICP30 unregister session Session# [4:7587241202637597149:2048] VirtualId# [4:7587241202637597146:0]
2025-12-24T02:10:52.674900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241202637597149:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.674949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.674970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.680160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:52.680248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241199781358445:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-12-24T02:10:52.680256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241199781358356:2049] [node 2] ICP30 unregister session Session# [3:7587241199781358445:2049] VirtualId# [3:7587241199781358443:0]
2025-12-24T02:10:52.680262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241199781358445:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.680297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.680318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.683422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241202332552293:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:52.685734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:52.685838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202332552388:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:10:52.685845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202332552294:2049] [node 2] ICP30 unregister session Session# [6:7587241202332552388:2049] VirtualId# [6:7587241202332552383:0]
2025-12-24T02:10:52.685852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202332552388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.685910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.685934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.686842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:52.686913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241202637597166:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-12-24T02:10:52.686919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241202637597065:2049] [node 2] ICP30 unregister session Session# [4:7587241202637597166:2049] VirtualId# [4:7587241202637597158:0]
2025-12-24T02:10:52.686924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241202637597166:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.686973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.686999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.687829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:52.687922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203567060012:2049] [node 2] ICS01 socket: 157 reason# EndOfStream
2025-12-24T02:10:52.687930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203567059920:2049] [node 2] ICP30 unregister session Session# [5:7587241203567060012:2049] VirtualId# [5:7587241203567060003:0]
2025-12-24T02:10:52.687935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203567060012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.687978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.688002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.699019Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:52.699102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203567060017:2050] [node 3] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:10:52.699108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203567059921:2050] [node 3] ICP30 unregister session Session# [5:7587241203567060017:2050] VirtualId# [5:7587241203567060014:0]
2025-12-24T02:10:52.699113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203567060017:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.699142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.699163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.698882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:52.698962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241202332552386:2050] [node 3] ICS07 socket disconnect 153 reason# EndOfStream
2025-12-24T02:10:52.698967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202332552386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.699017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.699037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241202332552386:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:52.699513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241202332552295:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9795) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:52.699525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202332552386:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:52.699530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202332552295:2050] [node 3] ICP30 unregister session Session# [6:7587241202332552386:2050] VirtualId# [6:7587241202332552372:0]
2025-12-24T02:10:52.699547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.699552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241202332552295:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.699555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9795) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:52.710769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241202332552295:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:52.714766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:52.714869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241202332552393:2110] [node 4] ICS01 socket: 184 reason# EndOfStream
2025-12-24T02:10:52.714879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241202332552367:2110] [node 4] ICP30 unregister session Session# [6:7587241202332552393:2110] VirtualId# [6:7587241202332552385:0]
2025-12-24T02:10:52.714887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241202332552393:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.714965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:52.714994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:52.753204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199632064669:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.753213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199632064670:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.754801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199971365689:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.754815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199971365690:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.762887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241201445777235:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.762922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241201445777236:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.762927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241201445777237:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.762939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200521970002:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:52.762957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200521970003:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:52.768841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203522943732:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.768859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203522943733:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.768864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203522943734:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.768499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241203422568232:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:52.768525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241203422568233:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:52.768529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241203422568234:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:52.776971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241201445777257:2068] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.777585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064685:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.777835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200521970024:2068] [node 5] ICP01 ready to work
2025-12-24T02:10:52.778583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200521970031:2072] [node 6] ICP01 ready to work
2025-12-24T02:10:52.777853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199632064688:2065] [node 6] ICP01 ready to work
2025-12-24T02:10:52.778875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199971365714:2071] [node 4] ICP01 ready to work
2025-12-24T02:10:52.783008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241203422568284:2081] [node 6] ICP01 ready to work
2025-12-24T02:10:52.783119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241203422568285:2086] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.795156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199971365743:2073] [node 5] ICP01 ready to work
2025-12-24T02:10:52.795253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241199971365744:2078] [node 6] ICP01 ready to work
2025-12-24T02:10:52.796006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365741:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
2025-12-24T02:10:52.796023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365742:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.799807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203522943793:2067] [node 6] ICP01 ready to work
2025-12-24T02:10:52.799916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241203522943794:2072] [node 4] ICP01 ready to work
2025-12-24T02:10:52.799998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943791:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.800013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943792:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.801099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943790:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.799303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199632064736:2072] [node 5] ICP01 ready to work
2025-12-24T02:10:52.799409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241199632064737:2084] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.833545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241201445777303:2079] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.830887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241200521970031:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(52225714ns) {Configure:104(52225115ns) {ProcessConfigured:135(51970172ns) {StartInitialHandshake:157(51969482ns) {DropHandshakes:470(392ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(717ns)}}}}}
2025-12-24T02:10:52.831218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241200521970074:2080] [node 4] ICP01 ready to work
2025-12-24T02:10:52.831322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200521970073:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63044
2025-12-24T02:10:52.832216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200521970072:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63044
2025-12-24T02:10:52.836301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241199632064736:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(33595170ns) {Configure:104(33594304ns) {ProcessConfigured:135(32670735ns) {StartInitialHandshake:157(32669997ns) {DropHandshakes:470(520ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(447ns) PrepareNewSessionHandshake:477(973ns) {SwitchToState:179(119ns)}}}}}
2025-12-24T02:10:52.835862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777302:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.837034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064748:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.837215Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241203522943794:2072] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35691407ns) {Configure:104(35691120ns) {ProcessConfigured:135(35473021ns) {StartInitialHandshake:157(35472879ns) {DropHandshakes:470(169ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(887ns) {SwitchToState:179(68ns)}}}}}
2025-12-24T02:10:52.837936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365742:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.838069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241201445777302:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.837668Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241203422568284:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(54500959ns) {Configure:104(54500313ns) {ProcessConfigured:135(54214118ns) {StartInitialHandshake:157(54213509ns) {DropHandshakes:470(490ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(1152ns) {SwitchToState:179(861ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.859025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568307:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.865229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241201445777237:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.865385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777315:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.863886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568307:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.862888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970073:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.863928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241203422568233:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.866350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200521970082:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63044
2025-12-24T02:10:52.867319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365741:2098] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.867417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241201445777315:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.866848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241203522943733:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29577093ns) {Configure:104(29576207ns) {ProcessConfigured:135(29299944ns) {StartInitialHandshake:157(29299220ns) {DropHandshakes:470(465ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(938ns) {SwitchToState:179(474ns)}}}}}
2025-12-24T02:10:52.866985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064685:2063] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.867083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064692:2068] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.867089Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199632064692:2068] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.867686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064750:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.871225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199632064692:2068] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.870770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365748:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.870791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199971365748:2101] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.874081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199632064688:2065] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.874101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.874109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199632064688:2065] [node 6] ICP22 created new session: [3:7587241199632064753:2065]
2025-12-24T02:10:52.874116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199632064753:2065] [node 6] ICS09 handshake done sender: [3:7587241199632064692:2068] self: [3:7587241199632064691:0] peer: [6:7587241201445777314:0] socket: 133 qp: -1
2025-12-24T02:10:52.874124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199632064753:2065] [node 6] ICS10 traffic start
2025-12-24T02:10:52.874134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.874456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943811:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.874739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199971365748:2101] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.875111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943791:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.876141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970072:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.877944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064750:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.878080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241199632064669:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.878393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241201445777303:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.878437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777316:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.878473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777317:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.878764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777318:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.878775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241201445777302:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.878855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064748:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.880019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241199632064688:2065] [node 6] ICP18 handshake (actor [3:7587241199632064748:2116]) is too old
2025-12-24T02:10:52.880227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241201445777237:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.880246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.880252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241201445777237:2050] [node 3] ICP22 created new session: [6:7587241201445777320:2050]
2025-12-24T02:10:52.880257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241201445777320:2050] [node 3] ICS09 handshake done sender: [6:7587241201445777302:2111] self: [6:7587241201445777314:0] peer: [3:7587241199632064691:0] socket: 146 qp: -1
2025-12-24T02:10:52.880264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241201445777320:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.880272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.881480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199971365714:2071] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.881505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.881512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199971365714:2071] [node 4] ICP22 created new session: [2:7587241199971365773:2071]
2025-12-24T02:10:52.881519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199971365773:2071] [node 4] ICS09 handshake done sender: [2:7587241199971365748:2101] self: [2:7587241199971365747:0] peer: [4:7587241203422568311:0] socket: 140 qp: -1
2025-12-24T02:10:52.881525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199971365773:2071] [node 4] ICS10 traffic start
2025-12-24T02:10:52.881534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.881928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970076:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.881942Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200521970076:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.882394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970082:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.882499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200521970086:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63044
2025-12-24T02:10:52.882940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943811:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.882970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943790:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.882984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203522943732:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.883042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203522943734:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.884625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365775:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
2025-12-24T02:10:52.884862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200521970076:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.885933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241201445777316:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.885991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241201445777235:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.886291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064743:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.886304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199632064743:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.884696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568304:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.884711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241203422568304:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.886262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970027:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.886273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200521970027:2070] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.886658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970071:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.886662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200521970071:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.887110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200521970003:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.887123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.887130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200521970003:2049] [node 3] ICP22 created new session: [1:7587241200521970089:2049]
2025-12-24T02:10:52.887136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200521970089:2049] [node 3] ICS09 handshake done sender: [1:7587241200521970076:2114] self: [1:7587241200521970075:0] peer: [3:7587241199632064756:0] socket: 160 qp: -1
2025-12-24T02:10:52.887145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200521970089:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.887153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.888450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943792:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.888490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943817:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.888509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203522943733:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.889158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.890423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970086:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.890475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200521970027:2070] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.887732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568312:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
2025-12-24T02:10:52.887924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568313:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
2025-12-24T02:10:52.887975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241203422568233:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10720) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241203422568307:2114] held: no
2025-12-24T02:10:52.888241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241203422568307:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.888712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568313:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.888719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241203422568233:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.888729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.888735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241203422568233:2049] [node 2] ICP22 created new session: [4:7587241203422568316:2049]
2025-12-24T02:10:52.888740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241203422568316:2049] [node 2] ICS09 handshake done sender: [4:7587241203422568307:2114] self: [4:7587241203422568311:0] peer: [2:7587241199971365747:0] socket: 155 qp: -1
2025-12-24T02:10:52.888746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241203422568316:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.888754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.889037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568315:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
2025-12-24T02:10:52.890298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241203422568304:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.890689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203522943791:2104] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.891345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241201445777318:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.891406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777326:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.892207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241201445777315:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.892257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241201445777303:2079] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.892271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.892276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241201445777303:2079] [node 4] ICP22 created new session: [6:7587241201445777327:2079]
2025-12-24T02:10:52.892281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241201445777327:2079] [node 4] ICS09 handshake done sender: [6:7587241201445777315:2116] self: [6:7587241201445777319:0] peer: [4:7587241203422568286:0] socket: 158 qp: -1
2025-12-24T02:10:52.892287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241201445777327:2079] [node 4] ICS10 traffic start
2025-12-24T02:10:52.892292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.892460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241201445777257:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.892480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.892520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777325:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.893181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241201445777324:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.893203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241201445777236:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.891455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365750:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.891467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199971365750:2102] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.892636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365775:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.892672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199971365750:2102] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.892772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199971365743:2073] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.892779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.892784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199971365743:2073] [node 5] ICP22 created new session: [2:7587241199971365778:2073]
2025-12-24T02:10:52.892788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199971365778:2073] [node 5] ICS09 handshake done sender: [2:7587241199971365750:2102] self: [2:7587241199971365749:0] peer: [5:7587241203522943818:0] socket: 141 qp: -1
2025-12-24T02:10:52.892793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199971365778:2073] [node 5] ICS10 traffic start
2025-12-24T02:10:52.892797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.893304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365752:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.893307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199971365752:2103] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.894925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203522943732:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.895070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777331:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
2025-12-24T02:10:52.894947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.894954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203522943732:2048] [node 1] ICP22 created new session: [5:7587241203522943820:2048]
2025-12-24T02:10:52.894960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203522943820:2048] [node 1] ICS09 handshake done sender: [5:7587241203522943791:2104] self: [5:7587241203522943814:0] peer: [1:7587241200521970026:0] socket: 143 qp: -1
2025-12-24T02:10:52.894967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203522943820:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.894977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.895217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943796:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.895228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241203522943796:2106] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.896269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200521970071:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.896332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241203422568284:2081] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.896351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.896358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241203422568284:2081] [node 6] ICP22 created new session: [4:7587241203422568320:2081]
2025-12-24T02:10:52.896363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241203422568320:2081] [node 6] ICS09 handshake done sender: [4:7587241203422568304:2112] self: [4:7587241203422568286:0] peer: [6:7587241201445777319:0] socket: 157 qp: -1
2025-12-24T02:10:52.896369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241203422568320:2081] [node 6] ICS10 traffic start
2025-12-24T02:10:52.896376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.895165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199971365752:2103] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.896954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200521970024:2068] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.896971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.896978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200521970024:2068] [node 5] ICP22 created new session: [1:7587241200521970094:2068]
2025-12-24T02:10:52.896984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200521970094:2068] [node 5] ICS09 handshake done sender: [1:7587241200521970027:2070] self: [1:7587241200521970026:0] peer: [5:7587241203522943814:0] socket: 131 qp: -1
2025-12-24T02:10:52.896988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200521970094:2068] [node 5] ICS10 traffic start
2025-12-24T02:10:52.896995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.897172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200521970031:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.897181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.897188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200521970031:2072] [node 6] ICP22 created new session: [1:7587241200521970097:2072]
2025-12-24T02:10:52.897192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200521970097:2072] [node 6] ICS09 handshake done sender: [1:7587241200521970071:2111] self: [1:7587241200521970034:0] peer: [6:7587241201445777323:0] socket: 159 qp: -1
2025-12-24T02:10:52.897196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200521970097:2072] [node 6] ICS10 traffic start
2025-12-24T02:10:52.897202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.897967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.898149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199971365744:2078] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.898168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.898174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199971365744:2078] [node 6] ICP22 created new session: [2:7587241199971365783:2078]
2025-12-24T02:10:52.898180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199971365783:2078] [node 6] ICS09 handshake done sender: [2:7587241199971365752:2103] self: [2:7587241199971365751:0] peer: [6:7587241201445777330:0] socket: 174 qp: -1
2025-12-24T02:10:52.898185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199971365783:2078] [node 6] ICS10 traffic start
2025-12-24T02:10:52.898193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.902935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199632064743:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.903186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241201445777324:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.903341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199632064736:2072] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.903355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.903361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199632064736:2072] [node 5] ICP22 created new session: [3:7587241199632064760:2072]
2025-12-24T02:10:52.903366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199632064760:2072] [node 5] ICS09 handshake done sender: [3:7587241199632064743:2113] self: [3:7587241199632064739:0] peer: [5:7587241203522943815:0] socket: 153 qp: -1
2025-12-24T02:10:52.903372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199632064760:2072] [node 5] ICS10 traffic start
2025-12-24T02:10:52.903379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.903483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199632064750:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.903572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241199632064669:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.903582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.903586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199632064669:2048] [node 1] ICP22 created new session: [3:7587241199632064762:2048]
2025-12-24T02:10:52.903590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199632064762:2048] [node 1] ICS09 handshake done sender: [3:7587241199632064750:2117] self: [3:7587241199632064756:0] peer: [1:7587241200521970075:0] socket: 162 qp: -1
2025-12-24T02:10:52.903594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199632064762:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.903599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.903720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064759:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.906262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241203522943819:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.906304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943822:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.907681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241200521970003:2049] [node 3] ICP18 handshake (actor [1:7587241200521970086:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.908170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568323:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.908494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241200521970096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63044
2025-12-24T02:10:52.908627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568315:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.908712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241203422568232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.908825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970096:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.908857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970078:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.908860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200521970078:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.910458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568325:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
2025-12-24T02:10:52.910576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200521970078:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.910768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241203422568315:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.910799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241201445777236:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.910818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.910826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241201445777236:2049] [node 2] ICP22 created new session: [6:7587241201445777332:2049]
2025-12-24T02:10:52.910832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241201445777332:2049] [node 2] ICS09 handshake done sender: [6:7587241201445777324:2120] self: [6:7587241201445777330:0] peer: [2:7587241199971365751:0] socket: 176 qp: -1
2025-12-24T02:10:52.910837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241201445777332:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.910846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.910879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365785:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.911857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365786:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.913171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064759:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.913271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241199632064670:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.913330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064765:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.913921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241203422568232:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.913937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.913944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241203422568232:2048] [node 1] ICP22 created new session: [4:7587241203422568326:2048]
2025-12-24T02:10:52.913950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241203422568326:2048] [node 1] ICS09 handshake done sender: [4:7587241203422568315:2118] self: [4:7587241203422568324:0] peer: [1:7587241200521970077:0] socket: 134 qp: -1
2025-12-24T02:10:52.913957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241203422568326:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.913964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.914107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568306:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.914110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241203422568306:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.911712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943823:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.911877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203522943796:2106] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:52.912007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203522943792:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.912057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241203522943794:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.912078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241203522943793:2067] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.912089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:52.912093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203522943793:2067] [node 6] ICP22 created new session: [5:7587241203522943828:2067]
2025-12-24T02:10:52.912098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203522943828:2067] [node 6] ICS09 handshake done sender: [5:7587241203522943796:2106] self: [5:7587241203522943795:0] peer: [6:7587241201445777329:0] socket: 167 qp: -1
2025-12-24T02:10:52.912104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203522943828:2067] [node 6] ICS10 traffic start
2025-12-24T02:10:52.912109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:52.912220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203522943733:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.912225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.912229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203522943733:2049] [node 2] ICP22 created new session: [5:7587241203522943830:2049]
2025-12-24T02:10:52.912233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203522943830:2049] [node 2] ICS09 handshake done sender: [5:7587241203522943792:2105] self: [5:7587241203522943818:0] peer: [2:7587241199971365749:0] socket: 144 qp: -1
2025-12-24T02:10:52.912236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203522943830:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.912240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.913471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203522943811:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.913666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203522943734:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.913674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.913679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203522943734:2050] [node 3] ICP22 created new session: [5:7587241203522943832:2050]
2025-12-24T02:10:52.913683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203522943832:2050] [node 3] ICS09 handshake done sender: [5:7587241203522943811:2118] self: [5:7587241203522943815:0] peer: [3:7587241199632064739:0] socket: 166 qp: -1
2025-12-24T02:10:52.913687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203522943832:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.913692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.915613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200521970074:2080] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.915633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.915640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200521970074:2080] [node 4] ICP22 created new session: [1:7587241200521970102:2080]
2025-12-24T02:10:52.915646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200521970102:2080] [node 4] ICS09 handshake done sender: [1:7587241200521970078:2115] self: [1:7587241200521970077:0] peer: [4:7587241203422568324:0] socket: 179 qp: -1
2025-12-24T02:10:52.915651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200521970102:2080] [node 4] ICS10 traffic start
2025-12-24T02:10:52.915659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.916165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365754:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.916179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241199971365754:2104] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.919896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241203422568306:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.919805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241199971365785:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.919906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241199971365689:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.919994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199971365754:2104] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.920002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241200521970080:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.920011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241200521970080:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.920076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241201445777257:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11316) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241201445777318:2119] held: no
2025-12-24T02:10:52.920114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241201445777316:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.921011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241200521970080:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.920868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241201445777236:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:52.920939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241201445777235:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17654) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241201445777316:2117] held: no
2025-12-24T02:10:52.920954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241201445777235:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.920963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.920968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241201445777235:2048] [node 1] ICP22 created new session: [6:7587241201445777335:2048]
2025-12-24T02:10:52.920975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241201445777335:2048] [node 1] ICS09 handshake done sender: [6:7587241201445777316:2117] self: [6:7587241201445777323:0] peer: [1:7587241200521970034:0] socket: 161 qp: -1
2025-12-24T02:10:52.920980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241201445777335:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.920986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:52.921139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241201445777334:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.921334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241199632064767:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32300
2025-12-24T02:10:52.921435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241203422568285:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.921451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.921456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241203422568285:2086] [node 5] ICP22 created new session: [4:7587241203422568329:2086]
2025-12-24T02:10:52.921461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241203422568329:2086] [node 5] ICS09 handshake done sender: [4:7587241203422568306:2113] self: [4:7587241203422568305:0] peer: [5:7587241203522943827:0] socket: 182 qp: -1
2025-12-24T02:10:52.921465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241203422568329:2086] [node 5] ICS10 traffic start
2025-12-24T02:10:52.921470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.921178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241200521970002:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.921197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.921204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241200521970002:2048] [node 2] ICP22 created new session: [1:7587241200521970105:2048]
2025-12-24T02:10:52.921210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241200521970105:2048] [node 2] ICS09 handshake done sender: [1:7587241200521970080:2116] self: [1:7587241200521970079:0] peer: [2:7587241199971365788:0] socket: 170 qp: -1
2025-12-24T02:10:52.921216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241200521970105:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:52.921629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199632064759:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:52.921223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.921916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241199632064670:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.921927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:52.921932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199632064670:2049] [node 2] ICP22 created new session: [3:7587241199632064768:2049]
2025-12-24T02:10:52.921937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199632064768:2049] [node 2] ICS09 handshake done sender: [3:7587241199632064759:2119] self: [3:7587241199632064766:0] peer: [2:7587241199971365753:0] socket: 193 qp: -1
2025-12-24T02:10:52.921942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199632064768:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:52.921947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:52.921776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241203422568323:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.920186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241203522943834:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13356
2025-12-24T02:10:52.921539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241203522943819:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.921704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241203522943794:2072] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.921713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.921718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241203522943794:2072] [node 4] ICP22 created new session: [5:7587241203522943835:2072]
2025-12-24T02:10:52.921723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241203522943835:2072] [node 4] ICS09 handshake done sender: [5:7587241203522943819:2120] self: [5:7587241203522943827:0] peer: [4:7587241203422568305:0] socket: 183 qp: -1
2025-12-24T02:10:52.921729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241203522943835:2072] [node 4] ICS10 traffic start
2025-12-24T02:10:52.921737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:52.924915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241201445777318:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:52.925059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241203422568234:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:52.925229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241199971365690:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.925246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.925251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199971365690:2049] [node 3] ICP22 created new session: [2:7587241199971365790:2049]
2025-12-24T02:10:52.925256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199971365790:2049] [node 3] ICS09 handshake done sender: [2:7587241199971365754:2104] self: [2:7587241199971365753:0] peer: [3:7587241199632064766:0] socket: 187 qp: -1
2025-12-24T02:10:52.925262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199971365790:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:52.925269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.927546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241201445777257:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.927561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:52.927566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241201445777257:2068] [node 5] ICP22 created new session: [6:7587241201445777337:2068]
2025-12-24T02:10:52.927572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241201445777337:2068] [node 5] ICS09 handshake done sender: [6:7587241201445777318:2119] self: [6:7587241201445777329:0] peer: [5:7587241203522943795:0] socket: 171 qp: -1
2025-12-24T02:10:52.927579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241201445777337:2068] [node 5] ICS10 traffic start
2025-12-24T02:10:52.927587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:52.928165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241199632064747:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:52.928177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241199632064747:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:52.929043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241199632064747:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:52.929136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241199632064737:2084] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:52.929151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:52.929157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241199632064737:2084] [node 4] ICP22 created new session: [3:7587241199632064771:2084]
2025-12-24T02:10:52.929163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241199632064771:2084] [node 4] ICS09 handshake done sender: [3:7587241199632064747:2115] self: [3:7587241199632064746:0] peer: [4:7587241203422568331:0] socket: 126 qp: -1
2025-12-24T02:10:52.929168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241199632064771:2084] [node 4] ICS10 traffic start
2025-12-24T02:10:52.929176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.935681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241199971365689:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17654) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241199971365785:2118] held: no
2025-12-24T02:10:52.935821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241199971365789:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57385
2025-12-24T02:10:52.935978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241199971365785:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:52.936025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241199971365689:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.936038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:52.936044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241199971365689:2048] [node 1] ICP22 created new session: [2:7587241199971365792:2048]
2025-12-24T02:10:52.936048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241199971365792:2048] [node 1] ICS09 handshake done sender: [2:7587241199971365785:2118] self: [2:7587241199971365788:0] peer: [1:7587241200521970079:0] socket: 195 qp: -1
2025-12-24T02:10:52.936054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241199971365792:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:52.936060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:52.936397Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241203422568234:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11390) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241203422568323:2119] held: no
2025-12-24T02:10:52.936439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241203422568332:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19571
2025-12-24T02:10:52.936563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241203422568323:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:52.936607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241203422568234:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:52.936622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:52.936625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241203422568234:2050] [node 3] ICP22 created new session: [4:7587241203422568333:2050]
2025-12-24T02:10:52.936629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241203422568333:2050] [node 3] ICS09 handshake done sender: [4:7587241203422568323:2119] self: [4:7587241203422568331:0] peer: [3:7587241199632064746:0] socket: 154 qp: -1
2025-12-24T02:10:52.936633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241203422568333:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:52.936638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:52.940892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:52.940980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241199971365792:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-12-24T02:10:52.940988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241199971365689:2048] [node 1] ICP30 unregister session Session# [2:7587241199971365792:2048] VirtualId# [2:7587241199971365788:0]
2025-12-24T02:10:52.940994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241199971365792:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.941044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.941064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.942520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:52.942601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241199632064762:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-12-24T02:10:52.942607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241199632064669:2048] [node 1] ICP30 unregister session Session# [3:7587241199632064762:2048] VirtualId# [3:7587241199632064756:0]
2025-12-24T02:10:52.942613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241199632064762:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.942669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.942870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.945669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:52.945757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203522943820:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-12-24T02:10:52.945766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203522943732:2048] [node 1] ICP30 unregister session Session# [5:7587241203522943820:2048] VirtualId# [5:7587241203522943814:0]
2025-12-24T02:10:52.945773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203522943820:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.945829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.945854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.947356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:52.947447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241203422568326:2048] [node 1] ICS01 socket: 134 reason# EndOfStream
2025-12-24T02:10:52.947454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241203422568232:2048] [node 1] ICP30 unregister session Session# [4:7587241203422568326:2048] VirtualId# [4:7587241203422568324:0]
2025-12-24T02:10:52.947459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241203422568326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.947512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.947538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.949383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:52.949460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241201445777335:2048] [node 1] ICS01 socket: 161 reason# EndOfStream
2025-12-24T02:10:52.949467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241201445777235:2048] [node 1] ICP30 unregister session Session# [6:7587241201445777335:2048] VirtualId# [6:7587241201445777323:0]
2025-12-24T02:10:52.949472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241201445777335:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.949518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:52.949540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:52.950558Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:52.950627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241199632064768:2049] [node 2] ICS01 socket: 193 reason# EndOfStream
2025-12-24T02:10:52.950633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241199632064670:2049] [node 2] ICP30 unregister session Session# [3:7587241199632064768:2049] VirtualId# [3:7587241199632064766:0]
2025-12-24T02:10:52.950639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241199632064768:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.950684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.950720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.957047Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:52.957149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241203422568316:2049] [node 2] ICS01 socket: 155 reason# EndOfStream
2025-12-24T02:10:52.957158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241203422568233:2049] [node 2] ICP30 unregister session Session# [4:7587241203422568316:2049] VirtualId# [4:7587241203422568311:0]
2025-12-24T02:10:52.957180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241203422568316:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.957235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.957262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.958420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:52.958490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241201445777332:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-12-24T02:10:52.958497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241201445777236:2049] [node 2] ICP30 unregister session Session# [6:7587241201445777332:2049] VirtualId# [6:7587241201445777330:0]
2025-12-24T02:10:52.958501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241201445777332:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.958541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.958561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.957390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:52.957484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203522943830:2049] [node 2] ICS01 socket: 144 reason# EndOfStream
2025-12-24T02:10:52.957489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203522943733:2049] [node 2] ICP30 unregister session Session# [5:7587241203522943830:2049] VirtualId# [5:7587241203522943818:0]
2025-12-24T02:10:52.957495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203522943830:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.957531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:52.957553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:52.972430Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:52.972495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241203422568333:2050] [node 3] ICS01 socket: 154 reason# EndOfStream
2025-12-24T02:10:52.972500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241203422568234:2050] [node 3] ICP30 unregister session Session# [4:7587241203422568333:2050] VirtualId# [4:7587241203422568331:0]
2025-12-24T02:10:52.972504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241203422568333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.972538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.972560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.973266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:52.973334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241201445777320:2050] [node 3] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:10:52.973340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241201445777237:2050] [node 3] ICP30 unregister session Session# [6:7587241201445777320:2050] VirtualId# [6:7587241201445777314:0]
2025-12-24T02:10:52.973344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241201445777320:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.973387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.973407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.972155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:52.972259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241203522943832:2050] [node 3] ICS01 socket: 166 reason# EndOfStream
2025-12-24T02:10:52.972269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241203522943734:2050] [node 3] ICP30 unregister session Session# [5:7587241203522943832:2050] VirtualId# [5:7587241203522943815:0]
2025-12-24T02:10:52.972275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241203522943832:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.972338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:52.972366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:52.978466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:52.978546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241201445777327:2079] [node 4] ICS01 socket: 158 reason# EndOfStream
2025-12-24T02:10:52.978551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241201445777303:2079] [node 4] ICP30 unregister session Session# [6:7587241201445777327:2079] VirtualId# [6:7587241201445777319:0]
2025-12-24T02:10:52.978556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241201445777327:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:52.978598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:52.978620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:53.012583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241205520992303:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.012598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241205520992304:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.018919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241204059245984:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:53.018933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241204059245985:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.029152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206554032005:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.029163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206554032006:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.029167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206554032007:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.026837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207381008840:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.026848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207381008841:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.042894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241204575719919:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.042909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241204575719920:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.042913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241204575719921:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.033549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241207513331506:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.033559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241207513331507:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.033563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241207513331508:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.071161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241204059246045:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:53.070973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241205520992365:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:53.071113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241205520992366:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:53.073482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241204575719943:2061] [node 4] ICP01 ready to work
2025-12-24T02:10:53.073575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241204575719944:2066] [node 5] ICP01 ready to work
2025-12-24T02:10:53.072867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241204059246050:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:53.079255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241205520992375:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.085705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032069:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.085779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032070:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.086266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032071:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.091110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241207513331574:2085] [node 4] ICP01 ready to work
2025-12-24T02:10:53.091726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241204059246055:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:53.095073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241204575719920:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22118519ns) {Configure:104(22117028ns) {ProcessConfigured:135(21716398ns) {StartInitialHandshake:157(21715580ns) {DropHandshakes:470(494ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(1641ns) {SwitchToState:179(858ns)}}}}}
2025-12-24T02:10:53.098778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207381008903:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:53.099010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207381008905:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.108777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241205520992381:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25400
2025-12-24T02:10:53.102914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241207381008840:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31212860ns) {Configure:104(31211539ns) {ProcessConfigured:135(30918608ns) {StartInitialHandshake:157(30918058ns) {DropHandshakes:470(265ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1577ns) {SwitchToState:179(1040ns)}}}}}
2025-12-24T02:10:53.103770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207381008911:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:53.110825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241206554032007:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24477480ns) {Configure:104(24476376ns) {ProcessConfigured:135(24162576ns) {StartInitialHandshake:157(24161982ns) {DropHandshakes:470(1069ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(317ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(2196ns) {SwitchToState:179(971ns)}}}}}
2025-12-24T02:10:53.110907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032070:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.110929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032069:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.111131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206554032077:2113] [node 6] ICP01 ready to work
2025-12-24T02:10:53.111180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032071:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.111712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206554032005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.112041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206554032006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.112653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992368:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.112667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241205520992368:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.113132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241205520992382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25400
2025-12-24T02:10:53.113254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241205520992368:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.113343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241205520992365:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.113358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.113365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241205520992365:2107] [node 4] ICP22 created new session: [2:7587241205520992384:2107]
2025-12-24T02:10:53.113371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241205520992384:2107] [node 4] ICS09 handshake done sender: [2:7587241205520992368:2109] self: [2:7587241205520992367:0] peer: [4:7587241206554032084:0] socket: 142 qp: -1
2025-12-24T02:10:53.113378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241205520992384:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:53.113386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.114860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241207513331574:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23599956ns) {Configure:104(23598726ns) {ProcessConfigured:135(23236002ns) {StartInitialHandshake:157(23235340ns) {DropHandshakes:470(1092ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(221ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1114ns) {SwitchToState:179(731ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.119514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241207513331573:2082] [node 6] ICP01 ready to work
2025-12-24T02:10:53.126812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992381:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.126807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206554032073:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.140911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241204059245985:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28397595ns) {Configure:104(28396736ns) {ProcessConfigured:135(28016186ns) {StartInitialHandshake:157(28015833ns) {DropHandshakes:470(381ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(38ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(849ns)}}}}}
2025-12-24T02:10:53.141014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246047:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.141020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241204059246047:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.141475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241204059246057:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58895
2025-12-24T02:10:53.141532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241204059246058:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58895
2025-12-24T02:10:53.141565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241204059246059:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58895
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.142212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575719997:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.140396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241205520992366:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(68722605ns) {Configure:104(68104145ns) {ProcessConfigured:135(67878228ns) {StartInitialHandshake:157(67877740ns) {DropHandshakes:470(305ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(1171ns) {SwitchToState:179(863ns)}}}}}
2025-12-24T02:10:53.142709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.140656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008919:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
2025-12-24T02:10:53.140691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008920:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
2025-12-24T02:10:53.140781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008921:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
2025-12-24T02:10:53.140824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008922:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.143021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241204059246047:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.143006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992382:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.143035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241205520992365:2107] [node 4] ICP18 handshake (actor [2:7587241205520992382:2115]) is too old
2025-12-24T02:10:53.142836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008920:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.142947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008923:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
2025-12-24T02:10:53.143182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246058:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.143209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246057:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.143216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241204059246045:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.143226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.143234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241204059246045:2107] [node 4] ICP22 created new session: [1:7587241204059246066:2107]
2025-12-24T02:10:53.143239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241204059246066:2107] [node 4] ICS09 handshake done sender: [1:7587241204059246047:2108] self: [1:7587241204059246046:0] peer: [4:7587241206554032080:0] socket: 132 qp: -1
2025-12-24T02:10:53.143245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241204059246066:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:53.143270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.144871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241204575719997:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.144902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241204575719920:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.146826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008921:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.146901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241207381008841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.146997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331548:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.162726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992380:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.162750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241205520992380:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.163037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992374:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.163043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241205520992374:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.163558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241205520992389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25400
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.163744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008926:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
2025-12-24T02:10:53.163848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241205520992374:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.164002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207381008921:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.164052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241207381008841:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.164068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.164073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207381008841:2049] [node 2] ICP22 created new session: [3:7587241207381008927:2049]
2025-12-24T02:10:53.164078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207381008927:2049] [node 2] ICS09 handshake done sender: [3:7587241207381008921:2117] self: [3:7587241207381008925:0] peer: [2:7587241205520992373:0] socket: 169 qp: -1
2025-12-24T02:10:53.164083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207381008927:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.164090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.164723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241205520992304:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.164739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.164743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241205520992304:2049] [node 3] ICP22 created new session: [2:7587241205520992391:2049]
2025-12-24T02:10:53.164747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241205520992391:2049] [node 3] ICS09 handshake done sender: [2:7587241205520992374:2112] self: [2:7587241205520992373:0] peer: [3:7587241207381008925:0] socket: 144 qp: -1
2025-12-24T02:10:53.164751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241205520992391:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.164755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.165185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206554032071:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.173643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331578:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.173697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331579:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.174288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008923:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.174344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241207381008840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.173952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241204059246055:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(82050031ns) {Configure:104(82049448ns) {ProcessConfigured:135(81738829ns) {StartInitialHandshake:157(81738636ns) {DropHandshakes:470(274ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(1762ns) {SwitchToState:179(731ns)}}}}}
2025-12-24T02:10:53.174456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331548:2087] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.174511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331578:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.176818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008919:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.176618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331550:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.176696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241207513331508:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.176708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241207513331506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.182817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241206554032073:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21728487ns) {Configure:104(21725283ns) {ProcessConfigured:135(21392734ns) {StartInitialHandshake:157(21392082ns) {DropHandshakes:470(1329ns) {DropIncomingHandshake:442(127ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(618ns) PrepareNewSessionHandshake:477(1859ns) {SwitchToState:179(1447ns)}}}}}
2025-12-24T02:10:53.183039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241206554032006:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241206554032071:2112] held: no
2025-12-24T02:10:53.183073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206554032006:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.183085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.183092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206554032006:2049] [node 2] ICP22 created new session: [4:7587241206554032093:2049]
2025-12-24T02:10:53.183098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206554032093:2049] [node 2] ICS09 handshake done sender: [4:7587241206554032071:2112] self: [4:7587241206554032084:0] peer: [2:7587241205520992367:0] socket: 148 qp: -1
2025-12-24T02:10:53.183106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206554032093:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.183114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.183444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032090:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.184358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032091:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.184410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032092:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.184528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206554032069:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.184595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206554032005:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.184608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.184614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206554032005:2048] [node 1] ICP22 created new session: [4:7587241206554032096:2048]
2025-12-24T02:10:53.184620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206554032096:2048] [node 1] ICS09 handshake done sender: [4:7587241206554032069:2110] self: [4:7587241206554032080:0] peer: [1:7587241204059246046:0] socket: 146 qp: -1
2025-12-24T02:10:53.184626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206554032096:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.184633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.187162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241205520992380:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.187120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575719999:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.187147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720000:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.187280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241205520992375:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.187293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.187301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241205520992375:2110] [node 6] ICP22 created new session: [2:7587241205520992395:2110]
2025-12-24T02:10:53.187306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241205520992395:2110] [node 6] ICS09 handshake done sender: [2:7587241205520992380:2113] self: [2:7587241205520992379:0] peer: [6:7587241204575719998:0] socket: 164 qp: -1
2025-12-24T02:10:53.187311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241205520992395:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:53.187318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.189434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241204575719997:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.189529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241204575719920:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.189548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.189553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241204575719920:2049] [node 2] ICP22 created new session: [6:7587241204575720001:2049]
2025-12-24T02:10:53.189559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241204575720001:2049] [node 2] ICS09 handshake done sender: [6:7587241204575719997:2115] self: [6:7587241204575719998:0] peer: [2:7587241205520992379:0] socket: 167 qp: -1
2025-12-24T02:10:53.189565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241204575720001:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.189573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.187037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008922:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.190920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008910:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.190929Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207381008910:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.191078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331579:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.191107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241207513331507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.190856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246061:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.190871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241204059246061:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.194751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331550:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.195311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246052:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.195324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241204059246052:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.195724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246059:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.195893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241204059246072:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58895
2025-12-24T02:10:53.195906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241204059246073:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58895
2025-12-24T02:10:53.196101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241204059246045:2107] [node 4] ICP18 handshake (actor [1:7587241204059246059:2114]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.199071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.199701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241204059246052:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.202771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992388:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.202787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241205520992388:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.202941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241207513331548:2087] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.202834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032092:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.202913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032091:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.203137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206554032007:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.207857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241205520992389:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.207888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241205520992303:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.208275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207381008910:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.208362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241204059246050:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.208380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.208385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241204059246050:2109] [node 5] ICP22 created new session: [1:7587241204059246076:2109]
2025-12-24T02:10:53.208391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241204059246076:2109] [node 5] ICS09 handshake done sender: [1:7587241204059246052:2110] self: [1:7587241204059246051:0] peer: [5:7587241207513331589:0] socket: 133 qp: -1
2025-12-24T02:10:53.208397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241204059246076:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:53.208406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.208429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241204059246061:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.208530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246064:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.208534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241204059246064:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.208540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241204059245985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.208551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.208556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241204059245985:2049] [node 3] ICP22 created new session: [1:7587241204059246078:2049]
2025-12-24T02:10:53.208560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241204059246078:2049] [node 3] ICS09 handshake done sender: [1:7587241204059246061:2115] self: [1:7587241204059246060:0] peer: [3:7587241207381008930:0] socket: 172 qp: -1
2025-12-24T02:10:53.208564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241204059246078:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.208568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.209387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241204059246064:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.209490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241204059245984:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.209508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.209515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241204059245984:2048] [node 2] ICP22 created new session: [1:7587241204059246082:2048]
2025-12-24T02:10:53.209522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241204059246082:2048] [node 2] ICS09 handshake done sender: [1:7587241204059246064:2116] self: [1:7587241204059246063:0] peer: [2:7587241205520992397:0] socket: 161 qp: -1
2025-12-24T02:10:53.209527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241204059246082:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:53.209533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.215073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241204575720000:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.215117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241204575719943:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.216068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241207513331506:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.216089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.216098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241207513331506:2048] [node 1] ICP22 created new session: [5:7587241207513331593:2048]
2025-12-24T02:10:53.216105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241207513331593:2048] [node 1] ICS09 handshake done sender: [5:7587241207513331548:2087] self: [5:7587241207513331589:0] peer: [1:7587241204059246051:0] socket: 140 qp: -1
2025-12-24T02:10:53.216112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241207513331593:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.216120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.215257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008913:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.215279Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207381008913:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.215796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207381008903:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.215815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.215824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207381008903:2107] [node 5] ICP22 created new session: [3:7587241207381008935:2107]
2025-12-24T02:10:53.215831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207381008935:2107] [node 5] ICS09 handshake done sender: [3:7587241207381008910:2112] self: [3:7587241207381008909:0] peer: [5:7587241207513331588:0] socket: 153 qp: -1
2025-12-24T02:10:53.215838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207381008935:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:53.215850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.216963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241207381008840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241207381008923:2119] held: no
2025-12-24T02:10:53.217101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207381008934:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.220143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241205520992303:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4070) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241205520992389:2117] held: no
2025-12-24T02:10:53.220190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241205520992398:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25400
2025-12-24T02:10:53.220296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207381008913:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.220943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246073:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.220391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207381008923:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.221308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241205520992389:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.221382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241205520992303:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.221395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.221400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241205520992303:2048] [node 1] ICP22 created new session: [2:7587241205520992399:2048]
2025-12-24T02:10:53.221406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241205520992399:2048] [node 1] ICS09 handshake done sender: [2:7587241205520992389:2117] self: [2:7587241205520992397:0] peer: [1:7587241204059246063:0] socket: 174 qp: -1
2025-12-24T02:10:53.221411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241205520992399:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.221419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.223529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241206554032007:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241206554032091:2120] held: no
2025-12-24T02:10:53.223599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206554032101:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21244
2025-12-24T02:10:53.223759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206554032091:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.223811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206554032007:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.223826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.223831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206554032007:2050] [node 3] ICP22 created new session: [4:7587241206554032102:2050]
2025-12-24T02:10:53.223837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206554032102:2050] [node 3] ICS09 handshake done sender: [4:7587241206554032091:2120] self: [4:7587241206554032100:0] peer: [3:7587241207381008912:0] socket: 181 qp: -1
2025-12-24T02:10:53.223843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206554032102:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.223849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.221690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207381008905:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.221717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.221726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207381008905:2108] [node 4] ICP22 created new session: [3:7587241207381008939:2108]
2025-12-24T02:10:53.221732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207381008939:2108] [node 4] ICS09 handshake done sender: [3:7587241207381008913:2113] self: [3:7587241207381008912:0] peer: [4:7587241206554032100:0] socket: 152 qp: -1
2025-12-24T02:10:53.221737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207381008939:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:53.221746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.222074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241207381008840:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.222085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.222092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207381008840:2048] [node 1] ICP22 created new session: [3:7587241207381008941:2048]
2025-12-24T02:10:53.222097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207381008941:2048] [node 1] ICS09 handshake done sender: [3:7587241207381008923:2119] self: [3:7587241207381008930:0] peer: [1:7587241204059246060:0] socket: 173 qp: -1
2025-12-24T02:10:53.222102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207381008941:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.222108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.226966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.227031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720006:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.227589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032082:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.227608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241206554032082:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.228486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241204575720006:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.228587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241204575719921:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.229061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206554032082:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.229141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241206554032077:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.229160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.229167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206554032077:2113] [node 6] ICP22 created new session: [4:7587241206554032105:2113]
2025-12-24T02:10:53.229174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206554032105:2113] [node 6] ICS09 handshake done sender: [4:7587241206554032082:2116] self: [4:7587241206554032081:0] peer: [6:7587241204575720004:0] socket: 184 qp: -1
2025-12-24T02:10:53.229181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206554032105:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:53.229188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.229289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720007:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.229355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207381008915:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.229368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207381008915:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.230200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720009:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.230368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241204575720000:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.230949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246072:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.230987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241204059246050:2109] [node 5] ICP18 handshake (actor [1:7587241204059246072:2118]) is too old
2025-12-24T02:10:53.231112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241205520992388:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.230458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241204575719943:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.230475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.230481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241204575719943:2061] [node 4] ICP22 created new session: [6:7587241204575720010:2061]
2025-12-24T02:10:53.230487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241204575720010:2061] [node 4] ICS09 handshake done sender: [6:7587241204575720000:2117] self: [6:7587241204575720004:0] peer: [4:7587241206554032081:0] socket: 186 qp: -1
2025-12-24T02:10:53.230494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241204575720010:2061] [node 4] ICS10 traffic start
2025-12-24T02:10:53.230503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.231289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241205520992366:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.231303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.231310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241205520992366:2108] [node 5] ICP22 created new session: [2:7587241205520992402:2108]
2025-12-24T02:10:53.231317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241205520992402:2108] [node 5] ICS09 handshake done sender: [2:7587241205520992388:2116] self: [2:7587241205520992371:0] peer: [5:7587241207513331590:0] socket: 155 qp: -1
2025-12-24T02:10:53.231324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241205520992402:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:53.231332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.231409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241204575720005:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.231448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241204575719919:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.231580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207381008915:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.231671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207381008911:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.231684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.231689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207381008911:2109] [node 6] ICP22 created new session: [3:7587241207381008944:2109]
2025-12-24T02:10:53.231694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207381008944:2109] [node 6] ICS09 handshake done sender: [3:7587241207381008915:2114] self: [3:7587241207381008914:0] peer: [6:7587241204575720008:0] socket: 151 qp: -1
2025-12-24T02:10:53.231703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207381008944:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:53.231712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.231567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241204059246071:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.231577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241204059246071:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.232387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241204575720006:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.232461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241204575719921:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.232476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.232482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241204575719921:2050] [node 3] ICP22 created new session: [6:7587241204575720014:2050]
2025-12-24T02:10:53.232488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241204575720014:2050] [node 3] ICS09 handshake done sender: [6:7587241204575720006:2119] self: [6:7587241204575720008:0] peer: [3:7587241207381008914:0] socket: 157 qp: -1
2025-12-24T02:10:53.232494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241204575720014:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.232502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.233471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720013:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.234914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241204059246071:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.235019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241204059246055:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.235034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.235042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241204059246055:2111] [node 6] ICP22 created new session: [1:7587241204059246086:2111]
2025-12-24T02:10:53.235048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241204059246086:2111] [node 6] ICS09 handshake done sender: [1:7587241204059246071:2117] self: [1:7587241204059246056:0] peer: [6:7587241204575720012:0] socket: 194 qp: -1
2025-12-24T02:10:53.235068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241204059246086:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:53.235077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.239548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241204575720005:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.247156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241204575719919:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.247179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.247186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241204575719919:2048] [node 1] ICP22 created new session: [6:7587241204575720016:2048]
2025-12-24T02:10:53.247193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241204575720016:2048] [node 1] ICS09 handshake done sender: [6:7587241204575720005:2118] self: [6:7587241204575720012:0] peer: [1:7587241204059246056:0] socket: 195 qp: -1
2025-12-24T02:10:53.247201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241204575720016:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.247209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.248135Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241204575719919:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.254896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241207513331508:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241207513331578:2114] held: no
2025-12-24T02:10:53.255006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331595:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.255069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331596:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.255137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331597:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.258998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331595:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.259023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241207513331578:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.259156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241207513331579:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.259292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720018:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.262948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241205520992404:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25400
2025-12-24T02:10:53.266798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241207513331574:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.266839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241207513331508:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.266853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.266860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241207513331508:2050] [node 3] ICP22 created new session: [5:7587241207513331601:2050]
2025-12-24T02:10:53.266865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241207513331601:2050] [node 3] ICS09 handshake done sender: [5:7587241207513331578:2114] self: [5:7587241207513331588:0] peer: [3:7587241207381008909:0] socket: 175 qp: -1
2025-12-24T02:10:53.266871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241207513331601:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.266879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.269693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241207513331507:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.269712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.269720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241207513331507:2049] [node 2] ICP22 created new session: [5:7587241207513331603:2049]
2025-12-24T02:10:53.269726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241207513331603:2049] [node 2] ICS09 handshake done sender: [5:7587241207513331579:2115] self: [5:7587241207513331590:0] peer: [2:7587241205520992371:0] socket: 156 qp: -1
2025-12-24T02:10:53.269733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241207513331603:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.269741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.270786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241204575720018:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.270825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241204575719944:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.271758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241205520992366:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:53.270793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206554032089:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.270808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241206554032089:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.271868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206554032089:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.271946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241206554032073:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.271959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.271967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206554032073:2108] [node 5] ICP22 created new session: [4:7587241206554032108:2108]
2025-12-24T02:10:53.271973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206554032108:2108] [node 5] ICS09 handshake done sender: [4:7587241206554032089:2118] self: [4:7587241206554032088:0] peer: [5:7587241207513331600:0] socket: 160 qp: -1
2025-12-24T02:10:53.271979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206554032108:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:53.271989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.274791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241207513331585:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.274802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241207513331585:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.276237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241207513331574:2085] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241207513331595:2120] held: no
2025-12-24T02:10:53.276413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241207513331605:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44625
2025-12-24T02:10:53.279048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241207513331595:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.279169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241207513331585:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.279231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241207513331574:2085] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.279242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.279250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241207513331574:2085] [node 4] ICP22 created new session: [5:7587241207513331607:2085]
2025-12-24T02:10:53.279256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241207513331607:2085] [node 4] ICS09 handshake done sender: [5:7587241207513331595:2120] self: [5:7587241207513331600:0] peer: [4:7587241206554032088:0] socket: 196 qp: -1
2025-12-24T02:10:53.279263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241207513331607:2085] [node 4] ICS10 traffic start
2025-12-24T02:10:53.279274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.279385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241207513331573:2082] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.279397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.279401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241207513331573:2082] [node 6] ICP22 created new session: [5:7587241207513331609:2082]
2025-12-24T02:10:53.279405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241207513331609:2082] [node 6] ICS09 handshake done sender: [5:7587241207513331585:2118] self: [5:7587241207513331584:0] peer: [6:7587241204575720019:0] socket: 138 qp: -1
2025-12-24T02:10:53.279409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241207513331609:2082] [node 6] ICS10 traffic start
2025-12-24T02:10:53.279414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.283844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241204575719944:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241204575720018:2123] held: no
2025-12-24T02:10:53.283912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241204575720020:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49442
2025-12-24T02:10:53.284075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241204575720018:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.284132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241204575719944:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.284144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.284150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241204575719944:2066] [node 5] ICP22 created new session: [6:7587241204575720021:2066]
2025-12-24T02:10:53.284156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241204575720021:2066] [node 5] ICS09 handshake done sender: [6:7587241204575720018:2123] self: [6:7587241204575720019:0] peer: [5:7587241207513331584:0] socket: 147 qp: -1
2025-12-24T02:10:53.284162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241204575720021:2066] [node 5] ICS10 traffic start
2025-12-24T02:10:53.284170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.298768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:53.298877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241205520992399:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-12-24T02:10:53.298889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241205520992303:2048] [node 1] ICP30 unregister session Session# [2:7587241205520992399:2048] VirtualId# [2:7587241205520992397:0]
2025-12-24T02:10:53.298898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241205520992399:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.298967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.298992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.299125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:53.299191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241207381008941:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:10:53.299198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241207381008840:2048] [node 1] ICP30 unregister session Session# [3:7587241207381008941:2048] VirtualId# [3:7587241207381008930:0]
2025-12-24T02:10:53.299202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241207381008941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.299235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.299256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.305162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:53.305247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241204575720016:2048] [node 1] ICS01 socket: 195 reason# EndOfStream
2025-12-24T02:10:53.305258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241204575719919:2048] [node 1] ICP30 unregister session Session# [6:7587241204575720016:2048] VirtualId# [6:7587241204575720012:0]
2025-12-24T02:10:53.305265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241204575720016:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.305322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.305346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.308555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:53.308648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241206554032096:2048] [node 1] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:10:53.308656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241206554032005:2048] [node 1] ICP30 unregister session Session# [4:7587241206554032096:2048] VirtualId# [4:7587241206554032080:0]
2025-12-24T02:10:53.308663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241206554032096:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.308729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.308757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.311280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:53.311368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241207513331593:2048] [node 1] ICS01 socket: 140 reason# EndOfStream
2025-12-24T02:10:53.311375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241207513331506:2048] [node 1] ICP30 unregister session Session# [5:7587241207513331593:2048] VirtualId# [5:7587241207513331589:0]
2025-12-24T02:10:53.311381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241207513331593:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.311421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.311444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.314847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:53.314943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241207381008927:2049] [node 2] ICS01 socket: 169 reason# EndOfStream
2025-12-24T02:10:53.314951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241207381008841:2049] [node 2] ICP30 unregister session Session# [3:7587241207381008927:2049] VirtualId# [3:7587241207381008925:0]
2025-12-24T02:10:53.314958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241207381008927:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.315017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.315038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.330688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:53.330792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241204575720001:2049] [node 2] ICS01 socket: 167 reason# EndOfStream
2025-12-24T02:10:53.330801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241204575719920:2049] [node 2] ICP30 unregister session Session# [6:7587241204575720001:2049] VirtualId# [6:7587241204575719998:0]
2025-12-24T02:10:53.330807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241204575720001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.330869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.330894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.332675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:53.332764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241206554032093:2049] [node 2] ICS01 socket: 148 reason# EndOfStream
2025-12-24T02:10:53.332772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241206554032006:2049] [node 2] ICP30 unregister session Session# [4:7587241206554032093:2049] VirtualId# [4:7587241206554032084:0]
2025-12-24T02:10:53.332778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241206554032093:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.332829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.332855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.332894Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:53.332961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241207513331603:2049] [node 2] ICS01 socket: 156 reason# EndOfStream
2025-12-24T02:10:53.332966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241207513331507:2049] [node 2] ICP30 unregister session Session# [5:7587241207513331603:2049] VirtualId# [5:7587241207513331590:0]
2025-12-24T02:10:53.332970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241207513331603:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.332998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.333020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.346762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:53.346850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241204575720014:2050] [node 3] ICS01 socket: 157 reason# EndOfStream
2025-12-24T02:10:53.346858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241204575719921:2050] [node 3] ICP30 unregister session Session# [6:7587241204575720014:2050] VirtualId# [6:7587241204575720008:0]
2025-12-24T02:10:53.346865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241204575720014:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.346914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.346936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:53.346778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:53.346867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241207513331601:2050] [node 3] ICS01 socket: 175 reason# EndOfStream
2025-12-24T02:10:53.346876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241207513331508:2050] [node 3] ICP30 unregister session Session# [5:7587241207513331601:2050] VirtualId# [5:7587241207513331588:0]
2025-12-24T02:10:53.346882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241207513331601:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.346938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.346963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:53.362745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:53.362833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241204575720010:2061] [node 4] ICS01 socket: 186 reason# EndOfStream
2025-12-24T02:10:53.362841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241204575719943:2061] [node 4] ICP30 unregister session Session# [6:7587241204575720010:2061] VirtualId# [6:7587241204575720004:0]
2025-12-24T02:10:53.362847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241204575720010:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.362908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:53.362930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:53.410762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241206524738838:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:53.410772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241206524738839:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.418772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241204141474663:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.418791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241204141474664:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.409736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241205390615654:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.409745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241205390615655:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.415590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241204269178002:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.415598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241204269178003:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.415603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241204269178004:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.422927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241206668038361:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.422934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241206668038362:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.422939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241206668038363:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.454865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241204956703374:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.458733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241204956703372:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.459396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241204956703373:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.466175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241205390615711:2103] [node 6] ICP01 ready to work
2025-12-24T02:10:53.466909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241204141474727:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:53.466378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241204956703400:2070] [node 4] ICP01 ready to work
2025-12-24T02:10:53.466458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241204956703401:2072] [node 6] ICP01 ready to work
2025-12-24T02:10:53.466613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241206524738900:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:53.466725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241206524738901:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:53.466942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241204269178025:2065] [node 5] ICP01 ready to work
2025-12-24T02:10:53.472023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241206524738907:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.488011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241205390615722:2105] [node 4] ICP01 ready to work
2025-12-24T02:10:53.488528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241205390615725:2106] [node 5] ICP01 ready to work
2025-12-24T02:10:53.489168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615720:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.489183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615721:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.489534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241206668038361:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22719704ns) {Configure:104(22718624ns) {ProcessConfigured:135(22263040ns) {StartInitialHandshake:157(22262461ns) {DropHandshakes:470(604ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(131ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(1236ns) {SwitchToState:179(640ns)}}}}}
2025-12-24T02:10:53.490515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241204141474727:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23546713ns) {Configure:104(23545539ns) {ProcessConfigured:135(23206313ns) {StartInitialHandshake:157(23205929ns) {DropHandshakes:470(448ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(123ns) PrepareNewSessionHandshake:477(1276ns) {SwitchToState:179(955ns)}}}}}
2025-12-24T02:10:53.490661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241204141474734:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:53.490793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241205390615711:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24550264ns) {Configure:104(24549658ns) {ProcessConfigured:135(23390530ns) {StartInitialHandshake:157(23389957ns) {DropHandshakes:470(881ns) {DropIncomingHandshake:442(163ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(1517ns) {SwitchToState:179(632ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.491785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241204956703373:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24500654ns) {Configure:104(24499717ns) {ProcessConfigured:135(24253076ns) {StartInitialHandshake:157(24252792ns) {DropHandshakes:470(228ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1539ns) {SwitchToState:179(516ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.498282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241204141474729:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:53.497077Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241204269178002:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30172400ns) {Configure:104(30164434ns) {ProcessConfigured:135(28675602ns) {StartInitialHandshake:157(28674777ns) {DropHandshakes:470(905ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(494ns)} GenerateSessionVirtualId:988(452ns) PrepareNewSessionHandshake:477(1889ns) {SwitchToState:179(971ns)}}}}}
2025-12-24T02:10:53.497895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178063:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.498554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615734:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.499222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178063:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.506846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241206524738901:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40049225ns) {Configure:104(40048511ns) {ProcessConfigured:135(34970531ns) {StartInitialHandshake:157(34969230ns) {DropHandshakes:470(443ns) {DropIncomingHandshake:442(113ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1299ns) {SwitchToState:179(644ns)}}}}}
2025-12-24T02:10:53.507668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241206524738912:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-12-24T02:10:53.507732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241206524738913:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-12-24T02:10:53.507773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241206524738914:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-12-24T02:10:53.507811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241206524738915:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-12-24T02:10:53.508063Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241206524738900:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41279486ns) {Configure:104(41277844ns) {ProcessConfigured:135(36136875ns) {StartInitialHandshake:157(36136082ns) {DropHandshakes:470(392ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(684ns) {SwitchToState:179(373ns)}}}}}
2025-12-24T02:10:53.510792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241204956703401:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43993459ns) {Configure:104(43991815ns) {ProcessConfigured:135(43691724ns) {StartInitialHandshake:157(43690665ns) {DropHandshakes:470(158ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(161ns) PrepareNewSessionHandshake:477(1702ns) {SwitchToState:179(686ns)}}}}}
2025-12-24T02:10:53.516708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738912:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.516754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738914:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.515044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241206668038394:2065] [node 5] ICP01 ready to work
2025-12-24T02:10:53.515958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241206668038395:2062] [node 4] ICP01 ready to work
2025-12-24T02:10:53.516130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241204269178036:2068] [node 6] ICP01 ready to work
2025-12-24T02:10:53.516171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241204269178004:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.514821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241205390615655:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16052510ns) {Configure:104(16040124ns) {ProcessConfigured:135(15628805ns) {StartInitialHandshake:157(15627824ns) {DropHandshakes:470(343ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1865ns) {SwitchToState:179(1041ns)}}}}}
2025-12-24T02:10:53.514833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615721:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.519107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738913:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.530813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241204956703400:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(64029494ns) {Configure:104(64026375ns) {ProcessConfigured:135(63670224ns) {StartInitialHandshake:157(63669367ns) {DropHandshakes:470(488ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:477(1761ns) {SwitchToState:179(885ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.532826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615737:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.533399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703450:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.533426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703451:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.531817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241204141474734:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33398055ns) {Configure:104(33396540ns) {ProcessConfigured:135(33050280ns) {StartInitialHandshake:157(33049743ns) {DropHandshakes:470(612ns) {DropIncomingHandshake:442(108ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(1531ns) {SwitchToState:179(1018ns)}}}}}
2025-12-24T02:10:53.531883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241204141474735:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56445
2025-12-24T02:10:53.532356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241204141474737:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56445
2025-12-24T02:10:53.532425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241204141474738:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56445
2025-12-24T02:10:53.532481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241204141474739:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56445
2025-12-24T02:10:53.533190Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241204141474663:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(65797738ns) {Configure:104(65797073ns) {ProcessConfigured:135(65566372ns) {StartInitialHandshake:157(65566001ns) {DropHandshakes:470(323ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(1063ns) {SwitchToState:179(867ns)}}}}}
2025-12-24T02:10:53.535170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474738:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.535197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474735:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.547787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038438:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.547818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038439:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.550856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474737:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.546952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241204269178003:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79925547ns) {Configure:104(79924961ns) {ProcessConfigured:135(78489394ns) {StartInitialHandshake:157(78488582ns) {DropHandshakes:470(522ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(1547ns) {SwitchToState:179(864ns)}}}}}
2025-12-24T02:10:53.547221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178081:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.547896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241204269178036:2068] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31674116ns) {Configure:104(31673515ns) {ProcessConfigured:135(31329854ns) {StartInitialHandshake:157(31329287ns) {DropHandshakes:470(587ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(188ns)}}}}}
2025-12-24T02:10:53.551102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615734:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.554826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241206668038395:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27435159ns) {Configure:104(27432983ns) {ProcessConfigured:135(27220878ns) {StartInitialHandshake:157(27219776ns) {DropHandshakes:470(284ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(2110ns) {SwitchToState:179(1540ns)}}}}}
2025-12-24T02:10:53.554942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241206524738839:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(82857648ns) {Configure:104(82856757ns) {ProcessConfigured:135(82650248ns) {StartInitialHandshake:157(82649992ns) {DropHandshakes:470(118ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(92ns) PrepareNewSessionHandshake:477(1070ns) {SwitchToState:179(207ns)}}}}}
2025-12-24T02:10:53.558739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615727:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.558767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241205390615727:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.562202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241205390615727:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.562472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241205390615722:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.562488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.562494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241205390615722:2105] [node 4] ICP22 created new session: [3:7587241205390615740:2105]
2025-12-24T02:10:53.562501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241205390615740:2105] [node 4] ICS09 handshake done sender: [3:7587241205390615727:2113] self: [3:7587241205390615726:0] peer: [4:7587241204269178078:0] socket: 144 qp: -1
2025-12-24T02:10:53.562508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241205390615740:2105] [node 4] ICS10 traffic start
2025-12-24T02:10:53.562518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.563099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241204956703451:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.563151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241204956703372:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.563152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703452:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.563314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738911:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.563327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241206524738911:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.564810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703454:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.564960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241206524738911:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.565100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241204956703451:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.565172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241204956703372:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.565187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.565200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241204956703372:2048] [node 1] ICP22 created new session: [5:7587241204956703455:2048]
2025-12-24T02:10:53.565206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241204956703455:2048] [node 1] ICS09 handshake done sender: [5:7587241204956703451:2116] self: [5:7587241204956703453:0] peer: [1:7587241206524738910:0] socket: 177 qp: -1
2025-12-24T02:10:53.565215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241204956703455:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.565227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.565969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178081:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.566010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241204269178003:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.566152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474733:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.566164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241204141474733:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.567072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178083:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.567132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241204141474733:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.567243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241204141474727:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.567259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.567267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241204141474727:2108] [node 4] ICP22 created new session: [2:7587241204141474750:2108]
2025-12-24T02:10:53.567273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241204141474750:2108] [node 4] ICS09 handshake done sender: [2:7587241204141474733:2111] self: [2:7587241204141474730:0] peer: [4:7587241204269178084:0] socket: 174 qp: -1
2025-12-24T02:10:53.567281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241204141474750:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:53.567290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.567787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615720:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.567821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615737:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.567852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241205390615655:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.567864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241205390615722:2105] [node 4] ICP18 handshake (actor [3:7587241205390615737:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.570427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.570645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241205390615720:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.568301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738915:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.568842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.569361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474724:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.569375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241204141474724:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.570386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241204141474724:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.570481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241204141474664:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.570503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.570512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241204141474664:2049] [node 3] ICP22 created new session: [2:7587241204141474754:2049]
2025-12-24T02:10:53.570519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241204141474754:2049] [node 3] ICS09 handshake done sender: [2:7587241204141474724:2107] self: [2:7587241204141474723:0] peer: [3:7587241205390615743:0] socket: 138 qp: -1
2025-12-24T02:10:53.570526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241204141474754:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.570548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.570746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241205390615655:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.570784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.570794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241205390615655:2049] [node 2] ICP22 created new session: [3:7587241205390615745:2049]
2025-12-24T02:10:53.570801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241205390615745:2049] [node 2] ICS09 handshake done sender: [3:7587241205390615720:2110] self: [3:7587241205390615743:0] peer: [2:7587241204141474723:0] socket: 139 qp: -1
2025-12-24T02:10:53.570812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241205390615745:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.570831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.570860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241206524738907:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.570881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.570888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241206524738907:2109] [node 5] ICP22 created new session: [1:7587241206524738926:2109]
2025-12-24T02:10:53.570893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241206524738926:2109] [node 5] ICS09 handshake done sender: [1:7587241206524738911:2112] self: [1:7587241206524738910:0] peer: [5:7587241204956703453:0] socket: 168 qp: -1
2025-12-24T02:10:53.570898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241206524738926:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:53.570906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.570841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241204269178063:2103] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.570988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241204956703450:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.571117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703458:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.571150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241204956703400:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.571291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241204269178004:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.571305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.571312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241204269178004:2050] [node 3] ICP22 created new session: [4:7587241204269178087:2050]
2025-12-24T02:10:53.571318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241204269178087:2050] [node 3] ICS09 handshake done sender: [4:7587241204269178063:2103] self: [4:7587241204269178078:0] peer: [3:7587241205390615726:0] socket: 156 qp: -1
2025-12-24T02:10:53.571324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241204269178087:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.571329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.571404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241204269178004:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:53.571550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178085:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.571868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474739:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.572028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241204141474663:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.572185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241206668038438:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.572191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241206668038441:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.572227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241206668038394:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.572234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241206668038361:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.572328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241204956703458:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.572390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241204956703452:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.574793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241204956703373:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.574819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241204956703374:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.576012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241206668038439:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.575640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615729:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.575654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241205390615729:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.576813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241205390615729:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.577414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703462:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.578602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241204956703452:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.579430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241204269178081:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.579655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.579596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241204956703374:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.579735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178062:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.579624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.579741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241204269178062:2102] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.579636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241204956703374:2050] [node 3] ICP22 created new session: [5:7587241204956703463:2050]
2025-12-24T02:10:53.579742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241204269178003:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.579643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241204956703463:2050] [node 3] ICS09 handshake done sender: [5:7587241204956703452:2117] self: [5:7587241204956703461:0] peer: [3:7587241205390615728:0] socket: 183 qp: -1
2025-12-24T02:10:53.579756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.579651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241204956703463:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.579765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241204269178003:2049] [node 2] ICP22 created new session: [4:7587241204269178089:2049]
2025-12-24T02:10:53.579663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.579771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241204269178089:2049] [node 2] ICS09 handshake done sender: [4:7587241204269178081:2115] self: [4:7587241204269178084:0] peer: [2:7587241204141474730:0] socket: 178 qp: -1
2025-12-24T02:10:53.579778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241204269178089:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.579786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.580469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474743:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.580495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241204141474743:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.576119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241206668038362:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.576206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.580804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241205390615725:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.580827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.580834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241205390615725:2106] [node 5] ICP22 created new session: [3:7587241205390615749:2106]
2025-12-24T02:10:53.580841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241205390615749:2106] [node 5] ICS09 handshake done sender: [3:7587241205390615729:2114] self: [3:7587241205390615728:0] peer: [5:7587241204956703461:0] socket: 182 qp: -1
2025-12-24T02:10:53.580848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241205390615749:2106] [node 5] ICS10 traffic start
2025-12-24T02:10:53.580856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.582581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241204956703448:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.582601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241204956703448:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.583665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241204956703448:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.584993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178092:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.585155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241204269178062:2102] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.585273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241204269178025:2065] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.585286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.585292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241204269178025:2065] [node 5] ICP22 created new session: [4:7587241204269178094:2065]
2025-12-24T02:10:53.585297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241204269178094:2065] [node 5] ICS09 handshake done sender: [4:7587241204269178062:2102] self: [4:7587241204269178061:0] peer: [5:7587241204956703459:0] socket: 158 qp: -1
2025-12-24T02:10:53.585302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241204269178094:2065] [node 5] ICS10 traffic start
2025-12-24T02:10:53.585309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.586198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241204141474741:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.586212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241204141474741:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.583405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241206668038444:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.583524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.583543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241206668038363:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.584169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241206668038441:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.584310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241206668038394:2065] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.584322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.584329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241206668038394:2065] [node 5] ICP22 created new session: [6:7587241206668038448:2065]
2025-12-24T02:10:53.584337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241206668038448:2065] [node 5] ICS09 handshake done sender: [6:7587241206668038441:2117] self: [6:7587241206668038442:0] peer: [5:7587241204956703410:0] socket: 152 qp: -1
2025-12-24T02:10:53.584344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241206668038448:2065] [node 5] ICS10 traffic start
2025-12-24T02:10:53.584352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.586230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.586804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738909:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.586820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241206524738909:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.587001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241204141474741:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.587762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241204141474734:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.587785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.587793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241204141474734:2110] [node 6] ICP22 created new session: [2:7587241204141474759:2110]
2025-12-24T02:10:53.587803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241204141474759:2110] [node 6] ICS09 handshake done sender: [2:7587241204141474741:2116] self: [2:7587241204141474736:0] peer: [6:7587241206668038445:0] socket: 176 qp: -1
2025-12-24T02:10:53.587812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241204141474759:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:53.587834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.588088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738904:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.588106Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241206524738904:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.586972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615733:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.586985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241205390615733:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.589437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241205390615733:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.591141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241205390615711:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.591166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.591175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241205390615711:2103] [node 6] ICP22 created new session: [3:7587241205390615752:2103]
2025-12-24T02:10:53.591182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241205390615752:2103] [node 6] ICS09 handshake done sender: [3:7587241205390615733:2115] self: [3:7587241205390615715:0] peer: [6:7587241206668038447:0] socket: 142 qp: -1
2025-12-24T02:10:53.591189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241205390615752:2103] [node 6] ICS10 traffic start
2025-12-24T02:10:53.591198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.591046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038452:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.591575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241206668038439:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.591647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038453:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.591692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241206668038362:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.591702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.591708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241206668038362:2049] [node 2] ICP22 created new session: [6:7587241206668038454:2049]
2025-12-24T02:10:53.591715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241206668038454:2049] [node 2] ICS09 handshake done sender: [6:7587241206668038439:2115] self: [6:7587241206668038445:0] peer: [2:7587241204141474736:0] socket: 179 qp: -1
2025-12-24T02:10:53.591720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241206668038454:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.591726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.591844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241206668038363:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241206668038444:2118] held: no
2025-12-24T02:10:53.592398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241206668038444:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.592450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241206668038363:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.592459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.592466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241206668038363:2050] [node 3] ICP22 created new session: [6:7587241206668038456:2050]
2025-12-24T02:10:53.592471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241206668038456:2050] [node 3] ICS09 handshake done sender: [6:7587241206668038444:2118] self: [6:7587241206668038447:0] peer: [3:7587241205390615715:0] socket: 197 qp: -1
2025-12-24T02:10:53.592476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241206668038456:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.592482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.592719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178086:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.592764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241204269178002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.593036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241206524738904:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.593526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241204141474762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56445
2025-12-24T02:10:53.593716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738919:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.593726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241206524738919:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.594262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241206524738838:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.594274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.594281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241206524738838:2048] [node 2] ICP22 created new session: [1:7587241206524738930:2048]
2025-12-24T02:10:53.594287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241206524738930:2048] [node 2] ICS09 handshake done sender: [1:7587241206524738904:2110] self: [1:7587241206524738903:0] peer: [2:7587241204141474757:0] socket: 167 qp: -1
2025-12-24T02:10:53.594296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241206524738930:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:53.594304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.594653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241206524738919:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.594836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178092:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.594990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.594983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178097:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
2025-12-24T02:10:53.596779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241204141474743:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.597286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241206524738909:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.597423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241206524738901:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.597441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.597447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241206524738901:2108] [node 6] ICP22 created new session: [1:7587241206524738936:2108]
2025-12-24T02:10:53.597453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241206524738936:2108] [node 6] ICS09 handshake done sender: [1:7587241206524738909:2111] self: [1:7587241206524738905:0] peer: [6:7587241206668038443:0] socket: 169 qp: -1
2025-12-24T02:10:53.597458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241206524738936:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:53.597464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.597896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038458:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.595870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241204956703401:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.595886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.595896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241204956703401:2072] [node 6] ICP22 created new session: [5:7587241204956703466:2072]
2025-12-24T02:10:53.595902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241204956703466:2072] [node 6] ICS09 handshake done sender: [5:7587241204956703448:2113] self: [5:7587241204956703410:0] peer: [6:7587241206668038442:0] socket: 186 qp: -1
2025-12-24T02:10:53.595908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241204956703466:2072] [node 6] ICS10 traffic start
2025-12-24T02:10:53.595916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.597926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703469:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.598007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241204956703470:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15462
2025-12-24T02:10:53.597863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241206524738935:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:94
2025-12-24T02:10:53.597300Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241205390615725:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(16459151ns) {HandleHandshakeStatus:329(16458261ns) {DropIncomingHandshake:442(5003ns) DropOutgoingHandshake:457(196ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(9341676ns) {ForwardSessionEventToSession:612(9328849ns) ForwardSessionEventToSession:612(1362ns) ForwardSessionEventToSession:612(1061ns) ForwardSessionEventToSession:612(857ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(1422ns) ForwardSessionEventToSession:612(861ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(1243ns)}}}
2025-12-24T02:10:53.599011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241204956703458:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.599205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241204269178086:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.599193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241204141474729:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.599220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.599227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241204141474729:2109] [node 5] ICP22 created new session: [2:7587241204141474763:2109]
2025-12-24T02:10:53.599232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241204141474763:2109] [node 5] ICS09 handshake done sender: [2:7587241204141474743:2117] self: [2:7587241204141474742:0] peer: [5:7587241204956703460:0] socket: 189 qp: -1
2025-12-24T02:10:53.599238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241204141474763:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:53.599251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.598842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241206524738900:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.598867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.598875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241206524738900:2107] [node 4] ICP22 created new session: [1:7587241206524738938:2107]
2025-12-24T02:10:53.598881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241206524738938:2107] [node 4] ICS09 handshake done sender: [1:7587241206524738919:2117] self: [1:7587241206524738906:0] peer: [4:7587241204269178096:0] socket: 192 qp: -1
2025-12-24T02:10:53.598888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241206524738938:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:53.598896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.599516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738935:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.599545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241206524738838:2048] [node 2] ICP18 handshake (actor [1:7587241206524738935:2119]) is too old
2025-12-24T02:10:53.600199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241204269178002:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.600221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.600226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241204269178002:2048] [node 1] ICP22 created new session: [4:7587241204269178099:2048]
2025-12-24T02:10:53.600232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241204269178099:2048] [node 1] ICS09 handshake done sender: [4:7587241204269178086:2119] self: [4:7587241204269178096:0] peer: [1:7587241206524738906:0] socket: 193 qp: -1
2025-12-24T02:10:53.600236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241204269178099:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.600242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.598867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241206668038438:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.599947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241206668038361:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241206668038438:2114] held: no
2025-12-24T02:10:53.599965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241206668038361:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.599975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.599979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241206668038361:2048] [node 1] ICP22 created new session: [6:7587241206668038459:2048]
2025-12-24T02:10:53.599985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241206668038459:2048] [node 1] ICS09 handshake done sender: [6:7587241206668038438:2114] self: [6:7587241206668038443:0] peer: [1:7587241206524738905:0] socket: 171 qp: -1
2025-12-24T02:10:53.599991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241206668038459:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.599998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.602885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241205390615754:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.602944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241205390615654:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.603134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241204141474739:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.604527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241204141474663:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.604549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.604561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241204141474663:2048] [node 1] ICP22 created new session: [2:7587241204141474765:2048]
2025-12-24T02:10:53.604568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241204141474765:2048] [node 1] ICS09 handshake done sender: [2:7587241204141474739:2115] self: [2:7587241204141474757:0] peer: [1:7587241206524738903:0] socket: 173 qp: -1
2025-12-24T02:10:53.604577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241204141474765:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.604585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.605454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241206668038451:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.605313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241204141474734:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(17521588ns) {HandleHandshakeStatus:329(17520819ns) {DropIncomingHandshake:442(4245ns) DropOutgoingHandshake:457(138ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(8604530ns) {ForwardSessionEventToSession:612(8592761ns) ForwardSessionEventToSession:612(1503ns) ForwardSessionEventToSession:612(1043ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(1021ns) ForwardSessionEventToSession:612(870ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(881ns)}}}
2025-12-24T02:10:53.605496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241206668038395:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.606100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241206524738922:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.606116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241206524738922:2118] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.606746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241204269178101:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.607939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241206524738922:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.608066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241206524738839:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.608082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.608088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241206524738839:2049] [node 3] ICP22 created new session: [1:7587241206524738941:2049]
2025-12-24T02:10:53.608094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241206524738941:2049] [node 3] ICS09 handshake done sender: [1:7587241206524738922:2118] self: [1:7587241206524738908:0] peer: [3:7587241205390615755:0] socket: 208 qp: -1
2025-12-24T02:10:53.608099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241206524738941:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.608106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.606988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241204956703373:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.607008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.607015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241204956703373:2049] [node 2] ICP22 created new session: [5:7587241204956703471:2049]
2025-12-24T02:10:53.607021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241204956703471:2049] [node 2] ICS09 handshake done sender: [5:7587241204956703458:2119] self: [5:7587241204956703460:0] peer: [2:7587241204141474742:0] socket: 191 qp: -1
2025-12-24T02:10:53.607027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241204956703471:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.607034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.607138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241204956703373:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:53.607227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241204956703450:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.607354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241205390615756:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61679
2025-12-24T02:10:53.609497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241205390615654:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24064) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241205390615754:2120] held: no
2025-12-24T02:10:53.609531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241205390615754:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.609729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178082:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.609745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241204269178082:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.609711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241204956703400:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.609728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.609737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241204956703400:2070] [node 4] ICP22 created new session: [5:7587241204956703473:2070]
2025-12-24T02:10:53.609743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241204956703473:2070] [node 4] ICS09 handshake done sender: [5:7587241204956703450:2115] self: [5:7587241204956703459:0] peer: [4:7587241204269178061:0] socket: 162 qp: -1
2025-12-24T02:10:53.609750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241204956703473:2070] [node 4] ICS10 traffic start
2025-12-24T02:10:53.609758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.610744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241205390615654:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.610765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.610773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241205390615654:2048] [node 1] ICP22 created new session: [3:7587241205390615757:2048]
2025-12-24T02:10:53.610780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241205390615757:2048] [node 1] ICS09 handshake done sender: [3:7587241205390615754:2120] self: [3:7587241205390615755:0] peer: [1:7587241206524738908:0] socket: 210 qp: -1
2025-12-24T02:10:53.610788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241205390615757:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.610795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.616908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241204269178101:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.616986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241204269178025:2065] [node 5] ICP18 handshake (actor [4:7587241204269178101:2122]) is too old
2025-12-24T02:10:53.617301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241204269178082:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.617376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241204269178036:2068] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.617398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.617407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241204269178036:2068] [node 6] ICP22 created new session: [4:7587241204269178103:2068]
2025-12-24T02:10:53.617413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241204269178103:2068] [node 6] ICS09 handshake done sender: [4:7587241204269178082:2116] self: [4:7587241204269178079:0] peer: [6:7587241206668038461:0] socket: 201 qp: -1
2025-12-24T02:10:53.617419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241204269178103:2068] [node 6] ICS10 traffic start
2025-12-24T02:10:53.617426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.619564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241206668038395:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241206668038451:2120] held: no
2025-12-24T02:10:53.619681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241206668038462:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34101
2025-12-24T02:10:53.619865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241206668038451:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.619951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241206668038395:2062] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.619962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.619976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241206668038395:2062] [node 4] ICP22 created new session: [6:7587241206668038463:2062]
2025-12-24T02:10:53.619982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241206668038463:2062] [node 4] ICS09 handshake done sender: [6:7587241206668038451:2120] self: [6:7587241206668038461:0] peer: [4:7587241204269178079:0] socket: 203 qp: -1
2025-12-24T02:10:53.619989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241206668038463:2062] [node 4] ICS10 traffic start
2025-12-24T02:10:53.619998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.638402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:53.638499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241204269178099:2048] [node 1] ICS01 socket: 193 reason# EndOfStream
2025-12-24T02:10:53.638509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241204269178002:2048] [node 1] ICP30 unregister session Session# [4:7587241204269178099:2048] VirtualId# [4:7587241204269178096:0]
2025-12-24T02:10:53.638516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241204269178099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.638568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.638595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.638630Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:53.638717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241206668038459:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-12-24T02:10:53.638722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241206668038361:2048] [node 1] ICP30 unregister session Session# [6:7587241206668038459:2048] VirtualId# [6:7587241206668038443:0]
2025-12-24T02:10:53.638727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241206668038459:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.638766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.638796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.640344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:53.640431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241205390615757:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:10:53.640439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241205390615654:2048] [node 1] ICP30 unregister session Session# [3:7587241205390615757:2048] VirtualId# [3:7587241205390615755:0]
2025-12-24T02:10:53.640444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241205390615757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.640503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.640532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.642055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:53.642137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241204956703455:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:10:53.642145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241204956703372:2048] [node 1] ICP30 unregister session Session# [5:7587241204956703455:2048] VirtualId# [5:7587241204956703453:0]
2025-12-24T02:10:53.642151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241204956703455:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.642217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.642245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.650790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:53.650904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241205390615745:2049] [node 2] ICS07 socket disconnect 139 reason# EndOfStream
2025-12-24T02:10:53.650911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241205390615745:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.650969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.650990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241205390615745:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:53.651431Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241205390615655:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:53.651448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241205390615745:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:53.651454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241205390615655:2049] [node 2] ICP30 unregister session Session# [3:7587241205390615745:2049] VirtualId# [3:7587241205390615743:0]
2025-12-24T02:10:53.651472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.651479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241205390615655:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.651482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.654151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:53.654236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241204269178089:2049] [node 2] ICS07 socket disconnect 178 reason# EndOfStream
2025-12-24T02:10:53.654242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241204269178089:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.654294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.654317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241204269178089:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:53.654770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241204269178003:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:53.654782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241204269178089:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:53.654787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241204269178003:2049] [node 2] ICP30 unregister session Session# [4:7587241204269178089:2049] VirtualId# [4:7587241204269178084:0]
2025-12-24T02:10:53.654802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.654808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241204269178003:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.654811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32220) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.654812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:53.654887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241204956703471:2049] [node 2] ICS01 socket: 191 reason# EndOfStream
2025-12-24T02:10:53.654894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241204956703373:2049] [node 2] ICP30 unregister session Session# [5:7587241204956703471:2049] VirtualId# [5:7587241204956703460:0]
2025-12-24T02:10:53.654899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241204956703471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.654947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.654975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.658777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:53.658868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241206668038454:2049] [node 2] ICS01 socket: 179 reason# EndOfStream
2025-12-24T02:10:53.658877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241206668038362:2049] [node 2] ICP30 unregister session Session# [6:7587241206668038454:2049] VirtualId# [6:7587241206668038445:0]
2025-12-24T02:10:53.658884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241206668038454:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.658946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.658973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.666755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241204269178003:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:53.677869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:53.678114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:53.678175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241206668038456:2050] [node 3] ICS01 socket: 197 reason# EndOfStream
2025-12-24T02:10:53.678179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241206668038363:2050] [node 3] ICP30 unregister session Session# [6:7587241206668038456:2050] VirtualId# [6:7587241206668038447:0]
2025-12-24T02:10:53.678183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241206668038456:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.678211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.678241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:53.677969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241204956703463:2050] [node 3] ICS01 socket: 183 reason# EndOfStream
2025-12-24T02:10:53.677980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241204956703374:2050] [node 3] ICP30 unregister session Session# [5:7587241204956703463:2050] VirtualId# [5:7587241204956703461:0]
2025-12-24T02:10:53.677988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241204956703463:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.678052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.678077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:53.698839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:53.698938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241204956703473:2070] [node 4] ICS07 socket disconnect 162 reason# EndOfStream
2025-12-24T02:10:53.698946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241204956703473:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.699009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:53.699033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241204956703473:2070] [node 4] ICS15 start handshake
2025-12-24T02:10:53.699118Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:53.699208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241206668038463:2062] [node 4] ICS01 socket: 203 reason# EndOfStream
2025-12-24T02:10:53.699214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241206668038395:2062] [node 4] ICP30 unregister session Session# [6:7587241206668038463:2062] VirtualId# [6:7587241206668038461:0]
2025-12-24T02:10:53.699222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241206668038463:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.699280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:53.699307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:53.744038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203682173911:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:53.744056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203682173912:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.740456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241207474177297:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.740487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241207474177298:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:53.758855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206001858372:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.758873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206001858373:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.758884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206001858374:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.754869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207420650520:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.754880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207420650521:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.758899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241205501680924:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.758909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241205501680925:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.758914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241205501680926:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.775301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241205496493161:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:53.775343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241205496493163:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:53.777487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241205496493162:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:53.782995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203682173973:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:53.783819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203682173977:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:53.783246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241205496493181:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.787030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241207474177358:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:53.788807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241203682173982:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:53.790615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858396:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.807337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241205496493222:2081] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.811798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241207474177364:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:53.812468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241207474177365:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34127
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.812937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207420650590:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:53.813057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650585:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.813130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650586:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.813623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650591:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.810913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206001858435:2074] [node 5] ICP01 ready to work
2025-12-24T02:10:53.811788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241206001858373:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21152433ns) {Configure:104(21151478ns) {ProcessConfigured:135(20830204ns) {StartInitialHandshake:157(20829966ns) {DropHandshakes:470(625ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(852ns) {SwitchToState:179(433ns)}}}}}
2025-12-24T02:10:53.812873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241205501680994:2060] [node 4] ICP01 ready to work
2025-12-24T02:10:53.813990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241205501680998:2074] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.814479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650585:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.816016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177365:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.816847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241203682173911:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24754344ns) {Configure:104(24753293ns) {ProcessConfigured:135(24417618ns) {StartInitialHandshake:157(24417129ns) {DropHandshakes:470(567ns) {DropIncomingHandshake:442(92ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(878ns) {SwitchToState:179(481ns)}}}}}
2025-12-24T02:10:53.815980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858396:2069] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.816406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858449:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.823365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241207474177298:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36262802ns) {Configure:104(36261522ns) {ProcessConfigured:135(35878905ns) {StartInitialHandshake:157(35878483ns) {DropHandshakes:470(941ns) {DropIncomingHandshake:442(138ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(1251ns) {SwitchToState:179(743ns)}}}}}
2025-12-24T02:10:53.823033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241207420650520:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20512639ns) {Configure:104(20511682ns) {ProcessConfigured:135(20166851ns) {StartInitialHandshake:157(20166008ns) {DropHandshakes:470(1272ns) {DropIncomingHandshake:442(469ns) DropOutgoingHandshake:457(142ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(1123ns) {SwitchToState:179(622ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.827173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203682173990:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59963
2025-12-24T02:10:53.827232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203682173991:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59963
2025-12-24T02:10:53.827267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203682173992:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59963
2025-12-24T02:10:53.827724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173990:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.827867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650591:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.828039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650586:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.828059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241207420650520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.828159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173986:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.828167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203682173986:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.828684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203682173986:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.828764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203682173912:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.828796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.828803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203682173912:2049] [node 3] ICP22 created new session: [1:7587241203682173995:2049]
2025-12-24T02:10:53.828809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203682173995:2049] [node 3] ICS09 handshake done sender: [1:7587241203682173986:2112] self: [1:7587241203682173985:0] peer: [3:7587241207420650598:0] socket: 155 qp: -1
2025-12-24T02:10:53.828817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203682173995:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.828835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.829254Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241205501680924:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25187622ns) {Configure:104(25186743ns) {ProcessConfigured:135(24821841ns) {StartInitialHandshake:157(24820922ns) {DropHandshakes:470(646ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(520ns)}}}}}
2025-12-24T02:10:53.829443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681001:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.829511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681002:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.829707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681002:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.829741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241205501680994:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.829833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858446:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.829848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241206001858446:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.830429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681001:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.830522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206001858446:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.830613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241206001858435:2074] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.830628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.830635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206001858435:2074] [node 5] ICP22 created new session: [4:7587241206001858452:2074]
2025-12-24T02:10:53.830640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206001858452:2074] [node 5] ICS09 handshake done sender: [4:7587241206001858446:2113] self: [4:7587241206001858445:0] peer: [5:7587241205501681003:0] socket: 160 qp: -1
2025-12-24T02:10:53.830646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206001858452:2074] [node 5] ICS10 traffic start
2025-12-24T02:10:53.830653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.830834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241205496493161:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23423532ns) {Configure:104(23422545ns) {ProcessConfigured:135(23088971ns) {StartInitialHandshake:157(23088443ns) {DropHandshakes:470(633ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(1149ns) {SwitchToState:179(595ns)}}}}}
2025-12-24T02:10:53.831508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493231:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.831573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493233:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.831659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493232:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.832927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207420650588:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:53.833373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241207420650589:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:53.832473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173975:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.832499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203682173975:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.833258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173991:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.834895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241207474177363:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:53.832646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241205501680924:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.838749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241206001858426:2072] [node 6] ICP01 ready to work
2025-12-24T02:10:53.839329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858449:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.839355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241206001858435:2074] [node 5] ICP18 handshake (actor [4:7587241206001858449:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.863613Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241205496493181:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(80287895ns) {Configure:104(80287275ns) {ProcessConfigured:135(71210666ns) {StartInitialHandshake:157(71210093ns) {DropHandshakes:470(669ns) {DropIncomingHandshake:442(150ns) DropOutgoingHandshake:457(130ns)} GenerateSessionVirtualId:988(500ns) PrepareNewSessionHandshake:477(1662ns) {SwitchToState:179(723ns)}}}}}
2025-12-24T02:10:53.863682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241205496493231:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.863994Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241205496493162:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(81032457ns) {Configure:104(81031569ns) {ProcessConfigured:135(71406831ns) {StartInitialHandshake:157(71406014ns) {DropHandshakes:470(403ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(35ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:477(786ns) {SwitchToState:179(507ns)}}}}}
2025-12-24T02:10:53.864032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241205496493162:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.862947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241207474177374:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34127
2025-12-24T02:10:53.862990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241207474177375:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34127
2025-12-24T02:10:53.864236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177369:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.864240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241207474177369:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.866998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858456:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.864885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241207474177369:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.864981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241207474177364:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.864995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.865003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241207474177364:2109] [node 6] ICP22 created new session: [2:7587241207474177377:2109]
2025-12-24T02:10:53.865009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241207474177377:2109] [node 6] ICS09 handshake done sender: [2:7587241207474177369:2112] self: [2:7587241207474177368:0] peer: [6:7587241205496493241:0] socket: 162 qp: -1
2025-12-24T02:10:53.865016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241207474177377:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:53.865027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.866828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241207474177358:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(77607102ns) {Configure:104(77605123ns) {ProcessConfigured:135(77348130ns) {StartInitialHandshake:157(77346724ns) {DropHandshakes:470(286ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(889ns) {SwitchToState:179(396ns)}}}}}
2025-12-24T02:10:53.869774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177375:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.872066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173988:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.872078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203682173988:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.872744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203682173988:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.872851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203682173982:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.872868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.872876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203682173982:2111] [node 6] ICP22 created new session: [1:7587241203682174000:2111]
2025-12-24T02:10:53.872883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203682174000:2111] [node 6] ICS09 handshake done sender: [1:7587241203682173988:2113] self: [1:7587241203682173983:0] peer: [6:7587241205496493242:0] socket: 175 qp: -1
2025-12-24T02:10:53.872889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203682174000:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:53.872898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.871918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241205496493233:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.871959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241205496493161:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.874847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858457:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.875461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173989:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.875470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203682173989:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.875149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241207474177382:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34127
2025-12-24T02:10:53.875336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177382:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.875359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241207474177297:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.878836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682173979:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.878858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241203682173979:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.878908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858457:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.878928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206001858372:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.878948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206001858373:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.878961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241206001858372:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(62916230ns) {IncomingHandshake:261(62915719ns) {DropIncomingHandshake:442(57ns) PrepareNewSessionHandshake:477(457ns) {SwitchToState:179(210ns)} GenerateSessionVirtualId:988(232ns) IssueIncomingHandshakeReply:193(62909958ns) {SwitchToState:179(717ns)}}}
2025-12-24T02:10:53.883037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241205501680994:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241205501681002:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.883884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203682174003:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59963
2025-12-24T02:10:53.885002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241205496493163:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(102032915ns) {Configure:104(102031064ns) {ProcessConfigured:135(92363584ns) {StartInitialHandshake:157(92362142ns) {DropHandshakes:470(354ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(1211ns) {SwitchToState:179(815ns)}}}}}
2025-12-24T02:10:53.885240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241206001858426:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46393271ns) {Configure:104(46392011ns) {ProcessConfigured:135(46067555ns) {StartInitialHandshake:157(46067265ns) {DropHandshakes:470(733ns) {DropIncomingHandshake:442(154ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(958ns) {SwitchToState:179(406ns)}}}}}
2025-12-24T02:10:53.885114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241205501680998:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(71068956ns) {Configure:104(71068455ns) {ProcessConfigured:135(70798299ns) {StartInitialHandshake:157(70797886ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(588ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.887030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858456:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:10:53.887376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493244:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.889255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241205496493232:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.889282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241205496493231:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.889318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241205496493163:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.890201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681007:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.890239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681006:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.889472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241205496493162:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.889490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.889497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241205496493162:2049] [node 2] ICP22 created new session: [6:7587241205496493247:2049]
2025-12-24T02:10:53.889502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241205496493247:2049] [node 2] ICS09 handshake done sender: [6:7587241205496493231:2111] self: [6:7587241205496493241:0] peer: [2:7587241207474177368:0] socket: 182 qp: -1
2025-12-24T02:10:53.889510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241205496493247:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.889519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.888668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177374:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.887317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650599:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.887634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650602:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.887672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207420650586:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.888027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650602:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.888037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241207420650520:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.888051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.888057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207420650520:2048] [node 1] ICP22 created new session: [3:7587241207420650603:2048]
2025-12-24T02:10:53.888063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207420650603:2048] [node 1] ICS09 handshake done sender: [3:7587241207420650586:2112] self: [3:7587241207420650598:0] peer: [1:7587241203682173985:0] socket: 157 qp: -1
2025-12-24T02:10:53.888071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207420650603:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.888080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.890884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682174003:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.891299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.891404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241205496493233:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.891481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241205496493161:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.891493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.891498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241205496493161:2048] [node 1] ICP22 created new session: [6:7587241205496493250:2048]
2025-12-24T02:10:53.891503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241205496493250:2048] [node 1] ICS09 handshake done sender: [6:7587241205496493233:2113] self: [6:7587241205496493242:0] peer: [1:7587241203682173983:0] socket: 184 qp: -1
2025-12-24T02:10:53.891509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241205496493250:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.891534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.891624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681007:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.891121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241203682173982:2111] [node 6] ICP18 handshake (actor [1:7587241203682174003:2118]) is too old
2025-12-24T02:10:53.894799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241205501681002:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.891186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177379:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.891199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241207474177379:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.895570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650593:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.895582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207420650593:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.896110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241207420650521:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.896290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650605:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.897120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207420650593:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.897998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203682173989:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.896434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177372:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.896445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241207474177372:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.897016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241207474177372:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.894884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241205501680994:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.894905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.894912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241205501680994:2060] [node 4] ICP22 created new session: [5:7587241205501681009:2060]
2025-12-24T02:10:53.894919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241205501681009:2060] [node 4] ICS09 handshake done sender: [5:7587241205501681002:2115] self: [5:7587241205501681003:0] peer: [4:7587241206001858445:0] socket: 176 qp: -1
2025-12-24T02:10:53.894927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241205501681009:2060] [node 4] ICS10 traffic start
2025-12-24T02:10:53.894945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.899123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203682173979:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.899261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241203682173975:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.899342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203682173911:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.899372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.899378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203682173911:2048] [node 2] ICP22 created new session: [1:7587241203682174007:2048]
2025-12-24T02:10:53.899383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203682174007:2048] [node 2] ICS09 handshake done sender: [1:7587241203682173989:2114] self: [1:7587241203682173984:0] peer: [2:7587241207474177383:0] socket: 197 qp: -1
2025-12-24T02:10:53.899389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203682174007:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:53.899397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.899493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203682173977:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.899499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.899503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203682173977:2108] [node 5] ICP22 created new session: [1:7587241203682174009:2108]
2025-12-24T02:10:53.899507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203682174009:2108] [node 5] ICS09 handshake done sender: [1:7587241203682173979:2110] self: [1:7587241203682173978:0] peer: [5:7587241205501681004:0] socket: 148 qp: -1
2025-12-24T02:10:53.899511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203682174009:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:53.899515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.899890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241203682173973:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.899905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.899910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241203682173973:2107] [node 4] ICP22 created new session: [1:7587241203682174011:2107]
2025-12-24T02:10:53.899916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241203682174011:2107] [node 4] ICS09 handshake done sender: [1:7587241203682173975:2109] self: [1:7587241203682173974:0] peer: [4:7587241206001858450:0] socket: 145 qp: -1
2025-12-24T02:10:53.899922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241203682174011:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:53.899927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.898593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241207420650588:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65574922ns) {Configure:104(65574642ns) {ProcessConfigured:135(19340755ns) {StartInitialHandshake:157(19328503ns) {DropHandshakes:470(1106ns) {DropIncomingHandshake:442(152ns) DropOutgoingHandshake:457(622ns)} GenerateSessionVirtualId:988(475ns) PrepareNewSessionHandshake:477(4185ns) {SwitchToState:179(440ns)}} IncomingHandshake:261(6147ns) {DropIncomingHandshake:442(132ns) PrepareNewSessionHandshake:477(154ns) {SwitchToState:179(96ns)} GenerateSessionVirtualId:988(197ns) IssueIncomingHandshakeReply:193(319ns)}}}}
2025-12-24T02:10:53.898648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207420650590:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.898656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.898662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207420650590:2109] [node 6] ICP22 created new session: [3:7587241207420650611:2109]
2025-12-24T02:10:53.898669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207420650611:2109] [node 6] ICS09 handshake done sender: [3:7587241207420650593:2114] self: [3:7587241207420650592:0] peer: [6:7587241205496493246:0] socket: 166 qp: -1
2025-12-24T02:10:53.898674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207420650611:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:53.898704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.898962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241207420650610:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18468
2025-12-24T02:10:53.899079Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241207420650589:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(65598323ns) {Configure:104(65598063ns) {ProcessConfigured:135(19629432ns) {StartInitialHandshake:157(19619578ns) {DropHandshakes:470(248ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:477(402ns) {SwitchToState:179(159ns)}} IncomingHandshake:261(3988ns) {DropIncomingHandshake:442(46ns) PrepareNewSessionHandshake:477(184ns) {SwitchToState:179(146ns)} GenerateSessionVirtualId:988(156ns) IssueIncomingHandshakeReply:193(187ns)}}}}
2025-12-24T02:10:53.900394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207420650602:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.900464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241207420650521:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.900474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.900482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207420650521:2049] [node 2] ICP22 created new session: [3:7587241207420650617:2049]
2025-12-24T02:10:53.900488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207420650617:2049] [node 2] ICS09 handshake done sender: [3:7587241207420650602:2117] self: [3:7587241207420650606:0] peer: [2:7587241207474177359:0] socket: 194 qp: -1
2025-12-24T02:10:53.900494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207420650617:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.900501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.902969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241206001858372:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241206001858396:2069] held: no
2025-12-24T02:10:53.903884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241207474177298:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.903904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.903911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241207474177298:2049] [node 3] ICP22 created new session: [2:7587241207474177387:2049]
2025-12-24T02:10:53.903917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241207474177387:2049] [node 3] ICS09 handshake done sender: [2:7587241207474177372:2113] self: [2:7587241207474177359:0] peer: [3:7587241207420650606:0] socket: 180 qp: -1
2025-12-24T02:10:53.903922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241207474177387:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:53.903929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.905258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.905439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241205496493232:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.905501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241205496493163:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.905516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.905522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241205496493163:2050] [node 3] ICP22 created new session: [6:7587241205496493253:2050]
2025-12-24T02:10:53.905528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241205496493253:2050] [node 3] ICS09 handshake done sender: [6:7587241205496493232:2112] self: [6:7587241205496493246:0] peer: [3:7587241207420650592:0] socket: 183 qp: -1
2025-12-24T02:10:53.905533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241205496493253:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.905538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.902887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241203682173912:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.907197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241203682174013:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59963
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.907678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241207474177297:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241207474177382:2117] held: no
2025-12-24T02:10:53.907855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241207474177389:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34127
2025-12-24T02:10:53.908079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241207474177379:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.908183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241207474177358:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.908203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.908209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241207474177358:2107] [node 4] ICP22 created new session: [2:7587241207474177391:2107]
2025-12-24T02:10:53.908216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241207474177391:2107] [node 4] ICS09 handshake done sender: [2:7587241207474177379:2116] self: [2:7587241207474177360:0] peer: [4:7587241206001858458:0] socket: 192 qp: -1
2025-12-24T02:10:53.908222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241207474177391:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:53.908227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.908326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241207474177382:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.908328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241207474177363:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(73332763ns) {Configure:104(73332361ns) {ProcessConfigured:135(13378464ns) {StartInitialHandshake:157(13362590ns) {DropHandshakes:470(504ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(459ns) PrepareNewSessionHandshake:477(879ns) {SwitchToState:179(392ns)}} IncomingHandshake:261(6415ns) {DropIncomingHandshake:442(73ns) PrepareNewSessionHandshake:477(151ns) {SwitchToState:179(96ns)} GenerateSessionVirtualId:988(288ns) IssueIncomingHandshakeReply:193(238ns)}}}}
2025-12-24T02:10:53.908434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241207474177297:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.908445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.908450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241207474177297:2048] [node 1] ICP22 created new session: [2:7587241207474177395:2048]
2025-12-24T02:10:53.908454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241207474177395:2048] [node 1] ICS09 handshake done sender: [2:7587241207474177382:2117] self: [2:7587241207474177383:0] peer: [1:7587241203682173984:0] socket: 199 qp: -1
2025-12-24T02:10:53.908459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241207474177395:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.908464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.909532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241203682174013:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.909567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241203682173977:2108] [node 5] ICP18 handshake (actor [1:7587241203682174013:2119]) is too old
2025-12-24T02:10:53.909917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241205501680924:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241205501681001:2114] held: no
2025-12-24T02:10:53.906962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650605:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.907000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241207420650590:2109] [node 6] ICP18 handshake (actor [3:7587241207420650605:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.910917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493255:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.914302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241206001858373:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241206001858457:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.915044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241205496493255:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.915102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241205496493222:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.915203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681005:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.915215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241205501681005:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.919320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.919374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858463:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.919437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858464:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.919658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493257:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.923385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241205501681005:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.922913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206001858396:2069] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.925966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206001858457:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.926784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206001858372:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.926819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.926828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206001858372:2048] [node 1] ICP22 created new session: [4:7587241206001858465:2048]
2025-12-24T02:10:53.926834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206001858465:2048] [node 1] ICS09 handshake done sender: [4:7587241206001858396:2069] self: [4:7587241206001858450:0] peer: [1:7587241203682173974:0] socket: 146 qp: -1
2025-12-24T02:10:53.926843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206001858465:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.926865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.926875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858463:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.926944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206001858373:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.926954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.926962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206001858373:2049] [node 2] ICP22 created new session: [4:7587241206001858467:2049]
2025-12-24T02:10:53.926967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206001858467:2049] [node 2] ICS09 handshake done sender: [4:7587241206001858457:2117] self: [4:7587241206001858458:0] peer: [2:7587241207474177360:0] socket: 198 qp: -1
2025-12-24T02:10:53.926974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206001858467:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.926981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:53.927130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241206001858374:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.927140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493258:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.925201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241205501680998:2074] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.925226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.925233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241205501680998:2074] [node 6] ICP22 created new session: [5:7587241205501681017:2074]
2025-12-24T02:10:53.925240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241205501681017:2074] [node 6] ICS09 handshake done sender: [5:7587241205501681005:2116] self: [5:7587241205501680999:0] peer: [6:7587241205496493256:0] socket: 161 qp: -1
2025-12-24T02:10:53.925246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241205501681017:2074] [node 6] ICS10 traffic start
2025-12-24T02:10:53.925253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.925571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.926786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681015:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.926867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681016:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.930946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241205496493257:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.931003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241205496493181:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.930931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650608:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.930945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207420650608:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.931264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241205496493222:2081] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20110) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241205496493255:2121] held: no
2025-12-24T02:10:53.931718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207420650608:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.931815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207420650588:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.931832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.931839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207420650588:2108] [node 4] ICP22 created new session: [3:7587241207420650620:2108]
2025-12-24T02:10:53.931846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207420650620:2108] [node 4] ICS09 handshake done sender: [3:7587241207420650608:2119] self: [3:7587241207420650600:0] peer: [4:7587241206001858469:0] socket: 209 qp: -1
2025-12-24T02:10:53.931851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207420650620:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:53.931858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:53.932718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241205496493255:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.931571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681015:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.931598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241205501681016:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.931614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241205501681001:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:53.932777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241205501680926:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.932790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241205501680925:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:53.934881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241205501680924:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.934897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:53.934904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241205501680924:2048] [node 1] ICP22 created new session: [5:7587241205501681021:2048]
2025-12-24T02:10:53.934910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241205501681021:2048] [node 1] ICS09 handshake done sender: [5:7587241205501681001:2114] self: [5:7587241205501681004:0] peer: [1:7587241203682173978:0] socket: 151 qp: -1
2025-12-24T02:10:53.934917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241205501681021:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:53.934929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:53.934850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241206001858459:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.934864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241206001858459:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.934936Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241206001858374:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241206001858463:2120] held: no
2025-12-24T02:10:53.936141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241205496493222:2081] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.936159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.936168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241205496493222:2081] [node 5] ICP22 created new session: [6:7587241205496493260:2081]
2025-12-24T02:10:53.936174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241205496493260:2081] [node 5] ICS09 handshake done sender: [6:7587241205496493255:2121] self: [6:7587241205496493256:0] peer: [5:7587241205501680999:0] socket: 164 qp: -1
2025-12-24T02:10:53.936181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241205496493260:2081] [node 5] ICS10 traffic start
2025-12-24T02:10:53.936195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.938841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241207474177393:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.938854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241207474177393:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:53.940540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206001858459:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:53.940693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241206001858426:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.940709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:53.940715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206001858426:2072] [node 6] ICP22 created new session: [4:7587241206001858471:2072]
2025-12-24T02:10:53.940721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206001858471:2072] [node 6] ICS09 handshake done sender: [4:7587241206001858459:2118] self: [4:7587241206001858455:0] peer: [6:7587241205496493259:0] socket: 173 qp: -1
2025-12-24T02:10:53.940726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206001858471:2072] [node 6] ICS10 traffic start
2025-12-24T02:10:53.940739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:53.940604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241207420650614:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:53.940622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241207420650614:2121] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.941540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241205496493262:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19951
2025-12-24T02:10:53.941740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241205496493257:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:53.941832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241205496493181:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.941843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:53.941850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241205496493181:2066] [node 4] ICP22 created new session: [6:7587241205496493263:2066]
2025-12-24T02:10:53.941855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241205496493263:2066] [node 4] ICS09 handshake done sender: [6:7587241205496493257:2122] self: [6:7587241205496493259:0] peer: [4:7587241206001858455:0] socket: 170 qp: -1
2025-12-24T02:10:53.941860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241205496493263:2066] [node 4] ICS10 traffic start
2025-12-24T02:10:53.941867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.942924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681024:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.942955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241205501681023:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36430
2025-12-24T02:10:53.943114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241205501681016:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:53.943187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241205501680925:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.943201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:53.943207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241205501680925:2049] [node 2] ICP22 created new session: [5:7587241205501681025:2049]
2025-12-24T02:10:53.943213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241205501681025:2049] [node 2] ICS09 handshake done sender: [5:7587241205501681016:2121] self: [5:7587241205501681020:0] peer: [2:7587241207474177385:0] socket: 212 qp: -1
2025-12-24T02:10:53.943220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241205501681025:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:53.943225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:53.945222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241206001858473:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26686
2025-12-24T02:10:53.945296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241207420650614:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.945424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241207420650589:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.945444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.945451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241207420650589:2107] [node 5] ICP22 created new session: [3:7587241207420650623:2107]
2025-12-24T02:10:53.945457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241207420650623:2107] [node 5] ICS09 handshake done sender: [3:7587241207420650614:2121] self: [3:7587241207420650601:0] peer: [5:7587241205501681019:0] socket: 210 qp: -1
2025-12-24T02:10:53.945462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241207420650623:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:53.945469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.945780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241206001858463:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.945890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241205501681015:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:53.945984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241206001858374:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.945994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.946000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241206001858374:2050] [node 3] ICP22 created new session: [4:7587241206001858474:2050]
2025-12-24T02:10:53.946005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241206001858474:2050] [node 3] ICS09 handshake done sender: [4:7587241206001858463:2120] self: [4:7587241206001858469:0] peer: [3:7587241207420650600:0] socket: 181 qp: -1
2025-12-24T02:10:53.946011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241206001858474:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.946020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.946328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241205501680926:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241205501681015:2120] held: no
2025-12-24T02:10:53.946509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241205501680926:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:53.946520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:53.946526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241205501680926:2050] [node 3] ICP22 created new session: [5:7587241205501681027:2050]
2025-12-24T02:10:53.946531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241205501681027:2050] [node 3] ICS09 handshake done sender: [5:7587241205501681015:2120] self: [5:7587241205501681019:0] peer: [3:7587241207420650601:0] socket: 211 qp: -1
2025-12-24T02:10:53.946537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241205501681027:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:53.946542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:53.943025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241207474177393:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:53.943126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241207474177363:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:53.943138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:53.943145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241207474177363:2108] [node 5] ICP22 created new session: [2:7587241207474177399:2108]
2025-12-24T02:10:53.943151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241207474177399:2108] [node 5] ICS09 handshake done sender: [2:7587241207474177393:2119] self: [2:7587241207474177385:0] peer: [5:7587241205501681020:0] socket: 185 qp: -1
2025-12-24T02:10:53.943156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241207474177399:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:53.943165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:53.963393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:53.963486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241207420650603:2048] [node 1] ICS01 socket: 157 reason# EndOfStream
2025-12-24T02:10:53.963497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241207420650520:2048] [node 1] ICP30 unregister session Session# [3:7587241207420650603:2048] VirtualId# [3:7587241207420650598:0]
2025-12-24T02:10:53.963505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241207420650603:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.963549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.963572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.963918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:53.964009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241207474177395:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-12-24T02:10:53.964015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241207474177297:2048] [node 1] ICP30 unregister session Session# [2:7587241207474177395:2048] VirtualId# [2:7587241207474177383:0]
2025-12-24T02:10:53.964020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241207474177395:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.964082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.964110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.965330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:53.965410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241205501681021:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-12-24T02:10:53.965418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241205501680924:2048] [node 1] ICP30 unregister session Session# [5:7587241205501681021:2048] VirtualId# [5:7587241205501681004:0]
2025-12-24T02:10:53.965423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241205501681021:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.965476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.965054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:53.965128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241205496493250:2048] [node 1] ICS07 socket disconnect 184 reason# EndOfStream
2025-12-24T02:10:53.965500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.965133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241205496493250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.965181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.965200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241205496493250:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:53.965661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241205496493161:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:53.965673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241205496493250:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:53.965677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241205496493161:2048] [node 1] ICP30 unregister session Session# [6:7587241205496493250:2048] VirtualId# [6:7587241205496493242:0]
2025-12-24T02:10:53.965692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.965698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241205496493161:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.965700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15338) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.965583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:53.965660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241206001858465:2048] [node 1] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:10:53.965666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241206001858372:2048] [node 1] ICP30 unregister session Session# [4:7587241206001858465:2048] VirtualId# [4:7587241206001858450:0]
2025-12-24T02:10:53.965671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241206001858465:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.965709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:53.965732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:53.973940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:53.974030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241206001858467:2049] [node 2] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:10:53.974038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241206001858373:2049] [node 2] ICP30 unregister session Session# [4:7587241206001858467:2049] VirtualId# [4:7587241206001858458:0]
2025-12-24T02:10:53.974045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241206001858467:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.974093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.974123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.974162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:53.974225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241205501681025:2049] [node 2] ICS07 socket disconnect 212 reason# EndOfStream
2025-12-24T02:10:53.974230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241205501681025:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.974258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.974275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241205501681025:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:53.974763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:53.974844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241207420650617:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-12-24T02:10:53.974852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241207420650521:2049] [node 2] ICP30 unregister session Session# [3:7587241207420650617:2049] VirtualId# [3:7587241207420650606:0]
2025-12-24T02:10:53.974857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241207420650617:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.974920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.974948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.974912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241205501680925:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:53.974924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241205501681025:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:53.974929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241205501680925:2049] [node 2] ICP30 unregister session Session# [5:7587241205501681025:2049] VirtualId# [5:7587241205501681020:0]
2025-12-24T02:10:53.974943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.974949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241205501680925:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.974951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:53.976855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:53.976940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241205496493247:2049] [node 2] ICS01 socket: 182 reason# EndOfStream
2025-12-24T02:10:53.976947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241205496493162:2049] [node 2] ICP30 unregister session Session# [6:7587241205496493247:2049] VirtualId# [6:7587241205496493241:0]
2025-12-24T02:10:53.976952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241205496493247:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.977003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:53.977029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:53.978759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241205496493161:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:53.985091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241205501680925:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:53.994790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:53.994895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241205501681027:2050] [node 3] ICS01 socket: 211 reason# EndOfStream
2025-12-24T02:10:53.994903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241205501680926:2050] [node 3] ICP30 unregister session Session# [5:7587241205501681027:2050] VirtualId# [5:7587241205501681019:0]
2025-12-24T02:10:53.994909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241205501681027:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.994970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.995002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:53.995039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:53.995112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241205496493253:2050] [node 3] ICS01 socket: 183 reason# EndOfStream
2025-12-24T02:10:53.995117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241205496493163:2050] [node 3] ICP30 unregister session Session# [6:7587241205496493253:2050] VirtualId# [6:7587241205496493246:0]
2025-12-24T02:10:53.995121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241205496493253:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:53.995148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:53.995167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:54.007110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:54.007207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241205496493263:2066] [node 4] ICS01 socket: 170 reason# EndOfStream
2025-12-24T02:10:54.007217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241205496493181:2066] [node 4] ICP30 unregister session Session# [6:7587241205496493263:2066] VirtualId# [6:7587241205496493259:0]
2025-12-24T02:10:54.007223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241205496493263:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.007286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:54.007314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:54.007353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:54.007422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241205501681009:2060] [node 4] ICS01 socket: 176 reason# EndOfStream
2025-12-24T02:10:54.007428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241205501680994:2060] [node 4] ICP30 unregister session Session# [5:7587241205501681009:2060] VirtualId# [5:7587241205501681003:0]
2025-12-24T02:10:54.007432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241205501681009:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.007462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:54.007483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:54.046389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241209683804814:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:54.046404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241209683804815:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.052235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241210382804746:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.052248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241210382804747:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.050020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211709709696:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.050030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211709709697:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.054240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210819379095:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.054249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210819379096:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.054254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210819379097:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.054442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211580828316:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.054449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211580828317:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.054454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211580828318:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.078842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208102038940:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.078858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208102038941:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.078862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208102038942:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.105814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211709709712:2062] [node 6] ICP01 ready to work
2025-12-24T02:10:54.106584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211709709718:2064] [node 4] ICP01 ready to work
2025-12-24T02:10:54.115125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210819379158:2078] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.116423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241209683804875:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.123790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039004:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.123876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241209683804814:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16242061ns) {Configure:104(16240418ns) {ProcessConfigured:135(15905564ns) {StartInitialHandshake:157(15905027ns) {DropHandshakes:470(360ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(944ns) {SwitchToState:179(549ns)}}}}}
2025-12-24T02:10:54.123902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241209683804881:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28184
2025-12-24T02:10:54.123918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241209683804882:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.122919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241210382804811:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.124812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211709709768:2068] [node 5] ICP01 ready to work
2025-12-24T02:10:54.125220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709771:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.126588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828383:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.126641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828384:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.124034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210819379165:2084] [node 5] ICP01 ready to work
2025-12-24T02:10:54.124553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379169:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.127095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241210382804818:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:54.127973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241211580828316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16379173ns) {Configure:104(16378442ns) {ProcessConfigured:135(16084938ns) {StartInitialHandshake:157(16084370ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(897ns) {SwitchToState:179(549ns)}}}}}
2025-12-24T02:10:54.130782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241208102038942:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19155307ns) {Configure:104(19154638ns) {ProcessConfigured:135(18839344ns) {StartInitialHandshake:157(18838622ns) {DropHandshakes:470(379ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1326ns) {SwitchToState:179(944ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.131188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709772:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.131968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828384:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.138956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709772:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.145509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241209683804880:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28184
2025-12-24T02:10:54.143794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241211580828318:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28839492ns) {Configure:104(28838639ns) {ProcessConfigured:135(28542491ns) {StartInitialHandshake:157(28541779ns) {DropHandshakes:470(775ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(1309ns) {SwitchToState:179(807ns)}}}}}
2025-12-24T02:10:54.143151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241211709709718:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36511700ns) {Configure:104(36511119ns) {ProcessConfigured:135(36202796ns) {StartInitialHandshake:157(36202094ns) {DropHandshakes:470(549ns) {DropIncomingHandshake:442(104ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(1158ns) {SwitchToState:179(608ns)}}}}}
2025-12-24T02:10:54.143214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709771:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.146875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208102039004:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.146921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208102038942:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.146823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241209683804891:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:54.147023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804882:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.147111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804880:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.154358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039016:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.154041Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241210382804747:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37907307ns) {Configure:104(37906122ns) {ProcessConfigured:135(37550767ns) {StartInitialHandshake:157(37550325ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(36ns) DropOutgoingHandshake:457(93ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:477(855ns) {SwitchToState:179(280ns)}}}}}
2025-12-24T02:10:54.153867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241210819379096:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47328000ns) {Configure:104(47327255ns) {ProcessConfigured:135(47045537ns) {StartInitialHandshake:157(47044882ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(328ns) PrepareNewSessionHandshake:477(1061ns) {SwitchToState:179(564ns)}}}}}
2025-12-24T02:10:54.155495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208102039001:2071] [node 5] ICP01 ready to work
2025-12-24T02:10:54.155866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208102039016:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.155089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828383:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.155132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211580828317:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.158712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709716:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.158729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211709709716:2065] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.163347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804881:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.163440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241209683804893:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28184
2025-12-24T02:10:54.163505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241209683804894:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28184
2025-12-24T02:10:54.166870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804893:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.167096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804894:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.167743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.167792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709780:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.167437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804823:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.167497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804824:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.167533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804825:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.167569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804826:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.170949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804824:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.170980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804823:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.171095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804825:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.170842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241209683804891:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23662098ns) {Configure:104(23661012ns) {ProcessConfigured:135(23340237ns) {StartInitialHandshake:157(23339582ns) {DropHandshakes:470(757ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(1247ns) {SwitchToState:179(480ns)}}}}}
2025-12-24T02:10:54.174972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379169:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.174982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241210819379158:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(59782330ns) {Configure:104(59781584ns) {ProcessConfigured:135(59483018ns) {StartInitialHandshake:157(59482375ns) {DropHandshakes:470(566ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(1412ns) {SwitchToState:179(562ns)}}}}}
2025-12-24T02:10:54.175017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210819379095:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.175083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208102038992:2061] [node 4] ICP01 ready to work
2025-12-24T02:10:54.174784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709779:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.174923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709780:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.174996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804820:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.175015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241210382804820:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.175469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804815:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.175481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241210382804815:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.175527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211580828370:2065] [node 4] ICP01 ready to work
2025-12-24T02:10:54.178747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211580828361:2062] [node 6] ICP01 ready to work
2025-12-24T02:10:54.179496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241209683804889:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:54.179394Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241208102039001:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23823724ns) {Configure:104(23822952ns) {ProcessConfigured:135(23530494ns) {StartInitialHandshake:157(23529883ns) {DropHandshakes:470(610ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(135ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:477(1119ns) {SwitchToState:179(501ns)}}}}}
2025-12-24T02:10:54.179725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208102038941:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.179745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241208102038941:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23838602ns) {IncomingHandshake:261(23838424ns) {DropIncomingHandshake:442(29ns) PrepareNewSessionHandshake:477(314ns) {SwitchToState:179(85ns)} GenerateSessionVirtualId:988(156ns) IssueIncomingHandshakeReply:193(23834222ns) {SwitchToState:179(46ns)}}}
2025-12-24T02:10:54.179626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241210382804810:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:54.182736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804888:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.182751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241209683804888:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.182810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804885:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.182815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241209683804885:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.183506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241209683804885:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.183648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241209683804875:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.183665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.183671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241209683804875:2107] [node 4] ICP22 created new session: [1:7587241209683804902:2107]
2025-12-24T02:10:54.183675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241209683804902:2107] [node 4] ICS09 handshake done sender: [1:7587241209683804885:2114] self: [1:7587241209683804884:0] peer: [4:7587241210819379174:0] socket: 171 qp: -1
2025-12-24T02:10:54.183681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241209683804902:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:54.183687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.184004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804826:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.184048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241210382804746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.184235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241210382804820:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.185229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039021:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.185472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208102039016:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.185581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208102038941:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.185598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.185605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208102038941:2049] [node 2] ICP22 created new session: [6:7587241208102039022:2049]
2025-12-24T02:10:54.185610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208102039022:2049] [node 2] ICS09 handshake done sender: [6:7587241208102039016:2114] self: [6:7587241208102039018:0] peer: [2:7587241210382804819:0] socket: 181 qp: -1
2025-12-24T02:10:54.185617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208102039022:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.185628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.184308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804883:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.184315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241209683804883:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.186164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241210382804818:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.186189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.186194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804833:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.186196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241210382804818:2109] [node 6] ICP22 created new session: [2:7587241210382804834:2109]
2025-12-24T02:10:54.186202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241210382804834:2109] [node 6] ICS09 handshake done sender: [2:7587241210382804820:2112] self: [2:7587241210382804819:0] peer: [6:7587241208102039018:0] socket: 179 qp: -1
2025-12-24T02:10:54.186209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241210382804834:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:54.186229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.186262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241209683804883:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.186345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241209683804814:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.186359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.186366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241209683804814:2048] [node 2] ICP22 created new session: [1:7587241209683804906:2048]
2025-12-24T02:10:54.186370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241209683804906:2048] [node 2] ICS09 handshake done sender: [1:7587241209683804883:2113] self: [1:7587241209683804877:0] peer: [2:7587241210382804832:0] socket: 170 qp: -1
2025-12-24T02:10:54.186375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241209683804906:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:54.186381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.186871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241210382804826:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.186900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241210819379095:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241210819379169:2111] held: no
2025-12-24T02:10:54.186931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828395:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.187771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211580828370:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.187465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241210382804746:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.187480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.187486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241210382804746:2048] [node 1] ICP22 created new session: [2:7587241210382804837:2048]
2025-12-24T02:10:54.187492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241210382804837:2048] [node 1] ICS09 handshake done sender: [2:7587241210382804826:2117] self: [2:7587241210382804832:0] peer: [1:7587241209683804877:0] socket: 190 qp: -1
2025-12-24T02:10:54.187498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241210382804837:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.187504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.191213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211709709716:2065] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.191299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211709709712:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.191318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.191324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211709709712:2062] [node 6] ICP22 created new session: [3:7587241211709709785:2062]
2025-12-24T02:10:54.191330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211709709785:2062] [node 6] ICS09 handshake done sender: [3:7587241211709709716:2065] self: [3:7587241211709709715:0] peer: [6:7587241208102039015:0] socket: 151 qp: -1
2025-12-24T02:10:54.191335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211709709785:2062] [node 6] ICS10 traffic start
2025-12-24T02:10:54.191352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.194366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.194490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208102039004:2104] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.194569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208102038942:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.194586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.191346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828395:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.194594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208102038942:2050] [node 3] ICP22 created new session: [6:7587241208102039026:2050]
2025-12-24T02:10:54.194601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208102039026:2050] [node 3] ICS09 handshake done sender: [6:7587241208102039004:2104] self: [6:7587241208102039015:0] peer: [3:7587241211709709715:0] socket: 162 qp: -1
2025-12-24T02:10:54.194610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208102039026:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.194628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.194800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804898:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.194810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241209683804898:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.191405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211580828316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.194851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828400:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.198971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828400:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.199018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211580828318:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.198812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709783:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.199897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241211709709696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.199941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241211709709696:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24971283ns) {IncomingHandshake:261(24971061ns) {DropIncomingHandshake:442(106ns) PrepareNewSessionHandshake:477(250ns) {SwitchToState:179(131ns)} GenerateSessionVirtualId:988(173ns) IssueIncomingHandshakeReply:193(24968098ns) {SwitchToState:179(43ns)}}}
2025-12-24T02:10:54.202730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379175:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.202804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379167:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.202814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241210819379167:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.202998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210819379169:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.203102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210819379095:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.203116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.203123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210819379095:2048] [node 1] ICP22 created new session: [4:7587241210819379176:2048]
2025-12-24T02:10:54.203131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210819379176:2048] [node 1] ICS09 handshake done sender: [4:7587241210819379169:2111] self: [4:7587241210819379174:0] peer: [1:7587241209683804884:0] socket: 173 qp: -1
2025-12-24T02:10:54.203144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210819379176:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.203178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.202812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241208102038941:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17198217ns) {HandleHandshakeStatus:329(17197521ns) {DropIncomingHandshake:442(233ns) DropOutgoingHandshake:457(3433ns) SwitchToState:179(103ns) ProcessPendingSessionEvents:495(17101118ns) {ForwardSessionEventToSession:612(17088731ns) ForwardSessionEventToSession:612(1856ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(978ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(1047ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(863ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(790ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.203835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828402:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.205061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828402:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.209152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241209683804888:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.209335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241209683804815:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.209357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.209365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241209683804815:2049] [node 3] ICP22 created new session: [1:7587241209683804912:2049]
2025-12-24T02:10:54.209370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241209683804912:2049] [node 3] ICS09 handshake done sender: [1:7587241209683804888:2115] self: [1:7587241209683804878:0] peer: [3:7587241211709709782:0] socket: 186 qp: -1
2025-12-24T02:10:54.209378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241209683804912:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.209396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.210891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241211709709696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6254) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241211709709780:2118] held: no
2025-12-24T02:10:54.210952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.210888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.211002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211580828383:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.210793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241210382804815:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.211111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211580828317:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.211132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.211140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211580828317:2049] [node 2] ICP22 created new session: [5:7587241211580828405:2049]
2025-12-24T02:10:54.211146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211580828405:2049] [node 2] ICS09 handshake done sender: [5:7587241211580828383:2108] self: [5:7587241211580828392:0] peer: [2:7587241210382804814:0] socket: 176 qp: -1
2025-12-24T02:10:54.211152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211580828405:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.211170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.211652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211709709780:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.211672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709770:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.211679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211709709770:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.214868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241209683804898:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.215063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241209683804891:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.215090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.215097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241209683804891:2109] [node 5] ICP22 created new session: [1:7587241209683804915:2109]
2025-12-24T02:10:54.215103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241209683804915:2109] [node 5] ICS09 handshake done sender: [1:7587241209683804898:2118] self: [1:7587241209683804892:0] peer: [5:7587241211580828399:0] socket: 159 qp: -1
2025-12-24T02:10:54.215109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241209683804915:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:54.215118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.215325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828407:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.219534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210819379167:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.219665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241210819379165:2084] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.219680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.219689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210819379165:2084] [node 5] ICP22 created new session: [4:7587241210819379179:2084]
2025-12-24T02:10:54.219695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210819379179:2084] [node 5] ICS09 handshake done sender: [4:7587241210819379167:2110] self: [4:7587241210819379166:0] peer: [5:7587241211580828397:0] socket: 172 qp: -1
2025-12-24T02:10:54.219702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210819379179:2084] [node 5] ICS10 traffic start
2025-12-24T02:10:54.219727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.222862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241210382804840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4136
2025-12-24T02:10:54.218846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211580828395:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.223035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211580828316:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.223055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.223063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211580828316:2048] [node 1] ICP22 created new session: [5:7587241211580828408:2048]
2025-12-24T02:10:54.223081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211580828408:2048] [node 1] ICS09 handshake done sender: [5:7587241211580828395:2115] self: [5:7587241211580828399:0] peer: [1:7587241209683804892:0] socket: 199 qp: -1
2025-12-24T02:10:54.223089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211580828408:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.223112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.223427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828410:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.226947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804840:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.226830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709783:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.226856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241211709709696:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.226871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.226886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211709709696:2048] [node 1] ICP22 created new session: [3:7587241211709709789:2048]
2025-12-24T02:10:54.226895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211709709789:2048] [node 1] ICS09 handshake done sender: [3:7587241211709709780:2118] self: [3:7587241211709709782:0] peer: [1:7587241209683804878:0] socket: 191 qp: -1
2025-12-24T02:10:54.226904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211709709789:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.226914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.227111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241211709709697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.233116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211709709774:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.233128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211709709774:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.233857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211709709774:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.234008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211709709718:2064] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.234022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.234029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211709709718:2064] [node 4] ICP22 created new session: [3:7587241211709709794:2064]
2025-12-24T02:10:54.234034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211709709794:2064] [node 4] ICS09 handshake done sender: [3:7587241211709709774:2113] self: [3:7587241211709709723:0] peer: [4:7587241210819379183:0] socket: 161 qp: -1
2025-12-24T02:10:54.234040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211709709794:2064] [node 4] ICS10 traffic start
2025-12-24T02:10:54.234048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.231484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.233000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379182:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.233032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210819379097:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.233783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.234757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210819379182:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.235123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804822:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.235140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241210382804822:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.243591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211709709770:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.243674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211709709768:2068] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.243691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.243700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211709709768:2068] [node 5] ICP22 created new session: [3:7587241211709709797:2068]
2025-12-24T02:10:54.243706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211709709797:2068] [node 5] ICS09 handshake done sender: [3:7587241211709709770:2110] self: [3:7587241211709709769:0] peer: [5:7587241211580828401:0] socket: 202 qp: -1
2025-12-24T02:10:54.243713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211709709797:2068] [node 5] ICS10 traffic start
2025-12-24T02:10:54.243721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.248532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241209683804908:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.248545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241209683804908:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.249514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241209683804908:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.250020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241209683804889:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.250033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.250040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241209683804889:2108] [node 6] ICP22 created new session: [1:7587241209683804918:2108]
2025-12-24T02:10:54.250047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241209683804918:2108] [node 6] ICS09 handshake done sender: [1:7587241209683804908:2119] self: [1:7587241209683804900:0] peer: [6:7587241208102039030:0] socket: 205 qp: -1
2025-12-24T02:10:54.250053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241209683804918:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:54.250079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.246827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241210382804810:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(67093196ns) {Configure:104(67090995ns) {ProcessConfigured:135(66816368ns) {StartInitialHandshake:157(66815881ns) {DropHandshakes:470(443ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:477(1640ns) {SwitchToState:179(976ns)}}}}}
2025-12-24T02:10:54.246867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210819379097:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.246918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.246926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210819379097:2050] [node 3] ICP22 created new session: [4:7587241210819379185:2050]
2025-12-24T02:10:54.246932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210819379185:2050] [node 3] ICS09 handshake done sender: [4:7587241210819379182:2117] self: [4:7587241210819379183:0] peer: [3:7587241211709709723:0] socket: 168 qp: -1
2025-12-24T02:10:54.246941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210819379185:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.246948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.247634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211580828384:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.249646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211580828370:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.249663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.249671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211580828370:2065] [node 4] ICP22 created new session: [5:7587241211580828412:2065]
2025-12-24T02:10:54.249677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211580828412:2065] [node 4] ICS09 handshake done sender: [5:7587241211580828384:2109] self: [5:7587241211580828397:0] peer: [4:7587241210819379166:0] socket: 177 qp: -1
2025-12-24T02:10:54.249684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211580828412:2065] [node 4] ICS10 traffic start
2025-12-24T02:10:54.249694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.247754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039029:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.248387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208102039029:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.248426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208102038940:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.249388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039031:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.249954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208102039029:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.250301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208102038940:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.250314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.250321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208102038940:2048] [node 1] ICP22 created new session: [6:7587241208102039032:2048]
2025-12-24T02:10:54.250326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208102039032:2048] [node 1] ICS09 handshake done sender: [6:7587241208102039029:2119] self: [6:7587241208102039030:0] peer: [1:7587241209683804900:0] socket: 206 qp: -1
2025-12-24T02:10:54.250332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208102039032:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.250338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.252765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379187:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.251102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241210382804811:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.251129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.251136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241210382804811:2107] [node 5] ICP22 created new session: [2:7587241210382804844:2107]
2025-12-24T02:10:54.251143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241210382804844:2107] [node 5] ICS09 handshake done sender: [2:7587241210382804815:2111] self: [2:7587241210382804814:0] peer: [5:7587241211580828392:0] socket: 169 qp: -1
2025-12-24T02:10:54.251149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241210382804844:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:54.251156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.252817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241210382804822:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.252946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241210382804747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.252962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.252967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241210382804747:2049] [node 3] ICP22 created new session: [2:7587241210382804847:2049]
2025-12-24T02:10:54.252973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241210382804847:2049] [node 3] ICS09 handshake done sender: [2:7587241210382804822:2113] self: [2:7587241210382804809:0] peer: [3:7587241211709709791:0] socket: 197 qp: -1
2025-12-24T02:10:54.252979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241210382804847:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.252985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.253151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211580828411:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15707
2025-12-24T02:10:54.253302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211580828400:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.254276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241211709709697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241211709709783:2119] held: no
2025-12-24T02:10:54.254320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211709709799:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23159
2025-12-24T02:10:54.254513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211709709783:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.254628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241211709709697:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.254641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.254647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211709709697:2049] [node 2] ICP22 created new session: [3:7587241211709709800:2049]
2025-12-24T02:10:54.254652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211709709800:2049] [node 2] ICS09 handshake done sender: [3:7587241211709709783:2119] self: [3:7587241211709709791:0] peer: [2:7587241210382804809:0] socket: 200 qp: -1
2025-12-24T02:10:54.254657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211709709800:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.254663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.255588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039034:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.256268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211580828318:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.256293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.256300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211580828318:2050] [node 3] ICP22 created new session: [5:7587241211580828414:2050]
2025-12-24T02:10:54.256307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211580828414:2050] [node 3] ICS09 handshake done sender: [5:7587241211580828400:2118] self: [5:7587241211580828401:0] peer: [3:7587241211709709769:0] socket: 204 qp: -1
2025-12-24T02:10:54.256314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211580828414:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.256323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.255162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241210382804811:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(43209586ns) {HandleHandshakeStatus:329(43209041ns) {DropIncomingHandshake:442(4895ns) DropOutgoingHandshake:457(780ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(35971ns) {ForwardSessionEventToSession:612(24571ns) ForwardSessionEventToSession:612(1106ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(516ns) ForwardSessionEventToSession:612(1387ns) ForwardSessionEventToSession:612(505ns) ForwardSessionEventToSession:612(964ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(1139ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.258324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379187:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.258361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210819379096:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.258520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.258589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241210382804841:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.258601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241210382804841:2120] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.259692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241210382804841:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.260140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241210382804810:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.260155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.260161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241210382804810:2108] [node 4] ICP22 created new session: [2:7587241210382804850:2108]
2025-12-24T02:10:54.260165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241210382804850:2108] [node 4] ICS09 handshake done sender: [2:7587241210382804841:2120] self: [2:7587241210382804830:0] peer: [4:7587241210819379189:0] socket: 217 qp: -1
2025-12-24T02:10:54.260169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241210382804850:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:54.260175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.259578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210819379191:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22366
2025-12-24T02:10:54.259779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210819379187:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.259926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210819379096:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.259939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.259945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210819379096:2049] [node 2] ICP22 created new session: [4:7587241210819379192:2049]
2025-12-24T02:10:54.259951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210819379192:2049] [node 2] ICS09 handshake done sender: [4:7587241210819379187:2119] self: [4:7587241210819379189:0] peer: [2:7587241210382804830:0] socket: 220 qp: -1
2025-12-24T02:10:54.259958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210819379192:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.259965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.261275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379173:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.261280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241210819379173:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.258423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208102039034:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.260006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039036:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.260060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208102038992:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.263852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210819379173:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.265723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241210819379158:2078] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.265735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.265748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210819379158:2078] [node 6] ICP22 created new session: [4:7587241210819379195:2078]
2025-12-24T02:10:54.265754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210819379195:2078] [node 6] ICS09 handshake done sender: [4:7587241210819379173:2115] self: [4:7587241210819379159:0] peer: [6:7587241208102039037:0] socket: 222 qp: -1
2025-12-24T02:10:54.265762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210819379195:2078] [node 6] ICS10 traffic start
2025-12-24T02:10:54.265770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.267134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208102039036:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.267177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208102039001:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.267269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039038:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.267502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208102039034:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.267613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208102038992:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.267638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.267650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208102038992:2061] [node 4] ICP22 created new session: [6:7587241208102039040:2061]
2025-12-24T02:10:54.267659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208102039040:2061] [node 4] ICS09 handshake done sender: [6:7587241208102039034:2121] self: [6:7587241208102039037:0] peer: [4:7587241210819379159:0] socket: 156 qp: -1
2025-12-24T02:10:54.267671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208102039040:2061] [node 4] ICS10 traffic start
2025-12-24T02:10:54.267684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.269011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211580828398:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.269022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241211580828398:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.269633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211580828398:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.269711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241211580828361:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.269729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.269737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211580828361:2062] [node 6] ICP22 created new session: [5:7587241211580828418:2062]
2025-12-24T02:10:54.269743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211580828418:2062] [node 6] ICS09 handshake done sender: [5:7587241211580828398:2117] self: [5:7587241211580828394:0] peer: [6:7587241208102039039:0] socket: 180 qp: -1
2025-12-24T02:10:54.269749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211580828418:2062] [node 6] ICS10 traffic start
2025-12-24T02:10:54.269756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.267824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210819379190:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.267870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241210819379158:2078] [node 6] ICP18 handshake (actor [4:7587241210819379190:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.271615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208102039001:2071] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241208102039036:2122] held: no
2025-12-24T02:10:54.271688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208102039042:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5941
2025-12-24T02:10:54.271923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208102039036:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.272026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208102039001:2071] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.272041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.272048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208102039001:2071] [node 5] ICP22 created new session: [6:7587241208102039043:2071]
2025-12-24T02:10:54.272055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208102039043:2071] [node 5] ICS09 handshake done sender: [6:7587241208102039036:2122] self: [6:7587241208102039039:0] peer: [5:7587241211580828394:0] socket: 187 qp: -1
2025-12-24T02:10:54.272062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208102039043:2071] [node 5] ICS10 traffic start
2025-12-24T02:10:54.272072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.277190Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:54.277274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241210382804837:2048] [node 1] ICS01 socket: 190 reason# EndOfStream
2025-12-24T02:10:54.277282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241210382804746:2048] [node 1] ICP30 unregister session Session# [2:7587241210382804837:2048] VirtualId# [2:7587241210382804832:0]
2025-12-24T02:10:54.277289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241210382804837:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.277335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.277357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.290772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:54.290879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241211709709789:2048] [node 1] ICS01 socket: 191 reason# EndOfStream
2025-12-24T02:10:54.290889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241211709709696:2048] [node 1] ICP30 unregister session Session# [3:7587241211709709789:2048] VirtualId# [3:7587241211709709782:0]
2025-12-24T02:10:54.290895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241211709709789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.290948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.290969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.315711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:54.315797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208102039032:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-12-24T02:10:54.315804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208102038940:2048] [node 1] ICP30 unregister session Session# [6:7587241208102039032:2048] VirtualId# [6:7587241208102039030:0]
2025-12-24T02:10:54.315811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208102039032:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.315863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.315886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.318810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:54.318894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211580828408:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-12-24T02:10:54.318902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211580828316:2048] [node 1] ICP30 unregister session Session# [5:7587241211580828408:2048] VirtualId# [5:7587241211580828399:0]
2025-12-24T02:10:54.318908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211580828408:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.318950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.318981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.318812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:54.318910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241210819379176:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:10:54.318918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241210819379095:2048] [node 1] ICP30 unregister session Session# [4:7587241210819379176:2048] VirtualId# [4:7587241210819379174:0]
2025-12-24T02:10:54.318925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241210819379176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.318978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.319003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.326728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:54.326820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241211709709800:2049] [node 2] ICS01 socket: 200 reason# EndOfStream
2025-12-24T02:10:54.326828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241211709709697:2049] [node 2] ICP30 unregister session Session# [3:7587241211709709800:2049] VirtualId# [3:7587241211709709791:0]
2025-12-24T02:10:54.326835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241211709709800:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.326888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.326909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.334752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:54.334865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241210819379192:2049] [node 2] ICS01 socket: 220 reason# EndOfStream
2025-12-24T02:10:54.334875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241210819379096:2049] [node 2] ICP30 unregister session Session# [4:7587241210819379192:2049] VirtualId# [4:7587241210819379189:0]
2025-12-24T02:10:54.334882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241210819379192:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.334957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.334987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.334880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:54.334962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211580828405:2049] [node 2] ICS01 socket: 176 reason# EndOfStream
2025-12-24T02:10:54.334969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211580828317:2049] [node 2] ICP30 unregister session Session# [5:7587241211580828405:2049] VirtualId# [5:7587241211580828392:0]
2025-12-24T02:10:54.334976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211580828405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.335026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.335049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.335021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:54.335104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241208102039022:2049] [node 2] ICS07 socket disconnect 181 reason# EndOfStream
2025-12-24T02:10:54.335110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208102039022:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.335157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.335189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241208102039022:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:54.337646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208102038941:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10256) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.337665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208102039022:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.337671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208102038941:2049] [node 2] ICP30 unregister session Session# [6:7587241208102039022:2049] VirtualId# [6:7587241208102039018:0]
2025-12-24T02:10:54.337689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.337695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241208102038941:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10256) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.337698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10256) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.346812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241208102038941:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:54.349679Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:54.349775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211580828414:2050] [node 3] ICS01 socket: 204 reason# EndOfStream
2025-12-24T02:10:54.349788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211580828318:2050] [node 3] ICP30 unregister session Session# [5:7587241211580828414:2050] VirtualId# [5:7587241211580828401:0]
2025-12-24T02:10:54.349795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211580828414:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.349857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:54.349880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:54.350738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:54.350829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241208102039026:2050] [node 3] ICS07 socket disconnect 162 reason# EndOfStream
2025-12-24T02:10:54.350834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208102039026:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.350892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:54.350915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241208102039026:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:54.351320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208102038942:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30554) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.351329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208102039026:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.351334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208102038942:2050] [node 3] ICP30 unregister session Session# [6:7587241208102039026:2050] VirtualId# [6:7587241208102039015:0]
2025-12-24T02:10:54.351352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:54.351358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241208102038942:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.351361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30554) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.358742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:54.358846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208102039040:2061] [node 4] ICS01 socket: 156 reason# EndOfStream
2025-12-24T02:10:54.358855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208102038992:2061] [node 4] ICP30 unregister session Session# [6:7587241208102039040:2061] VirtualId# [6:7587241208102039037:0]
2025-12-24T02:10:54.358861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208102039040:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.358910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:54.358936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:54.362833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241208102038942:2050] [node 3] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:54.402856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241211812310913:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.402879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241211812310914:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.403428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210463364889:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.403436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210463364890:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.403441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210463364891:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.406826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211569810346:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.406843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211569810347:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.406083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211451075890:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.406091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211451075891:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.406096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211451075892:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.414749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210639673669:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:54.414793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210639673670:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.443530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241209803095754:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.443543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241209803095755:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.443549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241209803095756:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.458521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241211812310974:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:54.459175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241209803095771:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.459824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364950:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.458884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210639673730:2106] [node 6] ICP01 ready to work
2025-12-24T02:10:54.458995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210639673731:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.460180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364951:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.460586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810406:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.460441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210639673738:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46634
2025-12-24T02:10:54.460788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241209803095782:2066] [node 5] ICP01 ready to work
2025-12-24T02:10:54.461943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211451075950:2103] [node 6] ICP01 ready to work
2025-12-24T02:10:54.459423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241211812310980:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:54.459811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241211812310981:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35197
2025-12-24T02:10:54.466282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364951:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.466769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210463364890:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.467274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211569810411:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:54.467835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211569810416:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.468420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210639673742:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46634
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.481181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210639673747:2115] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.481649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095832:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.481732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095833:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.481776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095834:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.483182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241211812310989:2113] [node 6] ICP01 ready to work
2025-12-24T02:10:54.483251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310979:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.483258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241211812310979:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.484535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241211812310988:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35197
2025-12-24T02:10:54.486175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810406:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.486359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241211569810421:2113] [node 5] ICP01 ready to work
2025-12-24T02:10:54.486972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241211812310979:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.490945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673738:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.491170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241209803095832:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.491211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241209803095754:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.496672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241211569810346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.497746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673733:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.497764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210639673733:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.497547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241209803095834:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.497587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241209803095756:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.497134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241211812310974:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.497155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.497163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241211812310974:2107] [node 4] ICP22 created new session: [2:7587241211812310995:2107]
2025-12-24T02:10:54.497171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241211812310995:2107] [node 4] ICS09 handshake done sender: [2:7587241211812310979:2110] self: [2:7587241211812310978:0] peer: [4:7587241210463364952:0] socket: 161 qp: -1
2025-12-24T02:10:54.497180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241211812310995:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:54.497191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.500762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211451075960:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:54.500809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075956:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.500882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075958:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.500895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075959:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.501086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241209803095833:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.501161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241209803095782:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.502421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075959:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.502539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075952:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.502545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241211451075952:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.504429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810413:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.504449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211569810413:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.511217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673737:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.511234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210639673737:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.514884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095838:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.514927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095839:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.515082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210639673733:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.515355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210639673737:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.515471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210639673730:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.515485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.515491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210639673730:2106] [node 6] ICP22 created new session: [1:7587241210639673754:2106]
2025-12-24T02:10:54.515497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210639673754:2106] [node 6] ICS09 handshake done sender: [1:7587241210639673733:2108] self: [1:7587241210639673732:0] peer: [6:7587241209803095835:0] socket: 165 qp: -1
2025-12-24T02:10:54.515503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210639673754:2106] [node 6] ICS10 traffic start
2025-12-24T02:10:54.515513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.515638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210639673670:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.515646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.515651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210639673670:2049] [node 3] ICP22 created new session: [1:7587241210639673756:2049]
2025-12-24T02:10:54.515655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210639673756:2049] [node 3] ICS09 handshake done sender: [1:7587241210639673737:2110] self: [1:7587241210639673736:0] peer: [3:7587241211569810426:0] socket: 167 qp: -1
2025-12-24T02:10:54.515659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210639673756:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.515664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.515923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210639673753:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46634
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.515400Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241211812310974:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(18238680ns) {HandleHandshakeStatus:329(18237492ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(309ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(10998415ns) {ForwardSessionEventToSession:612(10984679ns) ForwardSessionEventToSession:612(2577ns) ForwardSessionEventToSession:612(1339ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(1378ns)}}}
2025-12-24T02:10:54.515579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310981:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.516939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310988:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.517453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241211812310997:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35197
2025-12-24T02:10:54.517467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241211812310998:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35197
2025-12-24T02:10:54.518068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075956:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.518140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211451075952:2107] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.526039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210463364960:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:54.526139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241210463364961:2112] [node 5] ICP01 ready to work
2025-12-24T02:10:54.527389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364958:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.527452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364959:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.531686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241209803095833:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.531781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241209803095782:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.531795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.531801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241209803095782:2066] [node 5] ICP22 created new session: [6:7587241209803095841:2066]
2025-12-24T02:10:54.531806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241209803095841:2066] [node 5] ICS09 handshake done sender: [6:7587241209803095833:2116] self: [6:7587241209803095837:0] peer: [5:7587241211451075951:0] socket: 186 qp: -1
2025-12-24T02:10:54.531812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241209803095841:2066] [node 5] ICS10 traffic start
2025-12-24T02:10:54.531822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.531528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241211451075950:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.531557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.531565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211451075950:2103] [node 6] ICP22 created new session: [5:7587241211451075973:2103]
2025-12-24T02:10:54.531573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211451075973:2103] [node 6] ICS09 handshake done sender: [5:7587241211451075952:2107] self: [5:7587241211451075951:0] peer: [6:7587241209803095837:0] socket: 170 qp: -1
2025-12-24T02:10:54.531582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211451075973:2103] [node 6] ICS10 traffic start
2025-12-24T02:10:54.531593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.539412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810427:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.539611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211569810413:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.539670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211569810411:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.539681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.539687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211569810411:2109] [node 6] ICP22 created new session: [3:7587241211569810429:2109]
2025-12-24T02:10:54.539693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211569810429:2109] [node 6] ICS09 handshake done sender: [3:7587241211569810413:2110] self: [3:7587241211569810412:0] peer: [6:7587241209803095836:0] socket: 179 qp: -1
2025-12-24T02:10:54.539699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211569810429:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:54.539707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.539818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211569810406:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.540355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810418:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.540359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211569810418:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.540971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241211569810346:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.540980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.540987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211569810346:2048] [node 1] ICP22 created new session: [3:7587241211569810432:2048]
2025-12-24T02:10:54.540992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211569810432:2048] [node 1] ICS09 handshake done sender: [3:7587241211569810406:2107] self: [3:7587241211569810426:0] peer: [1:7587241210639673736:0] socket: 168 qp: -1
2025-12-24T02:10:54.540999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211569810432:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.541005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.541319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673742:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.539058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364950:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.539116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364962:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.539160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210463364951:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.539869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210463364890:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.539879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.539883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210463364890:2049] [node 2] ICP22 created new session: [4:7587241210463364969:2049]
2025-12-24T02:10:54.539887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210463364969:2049] [node 2] ICS09 handshake done sender: [4:7587241210463364951:2109] self: [4:7587241210463364952:0] peer: [2:7587241211812310978:0] socket: 162 qp: -1
2025-12-24T02:10:54.539892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210463364969:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.539896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.540032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364958:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.540185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210463364891:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.541663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364962:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.543116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310998:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.543206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241211812310913:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.541563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810431:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.544080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211569810418:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.549716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211451075890:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.551467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673744:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.551490Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210639673744:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.549970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075958:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.550184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211451075891:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.550212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075976:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.555592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241210463364889:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.555837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.559759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211569810416:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.559780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.559788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211569810416:2111] [node 4] ICP22 created new session: [3:7587241211569810436:2111]
2025-12-24T02:10:54.559796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211569810436:2111] [node 4] ICS09 handshake done sender: [3:7587241211569810418:2112] self: [3:7587241211569810417:0] peer: [4:7587241210463364974:0] socket: 182 qp: -1
2025-12-24T02:10:54.559803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211569810436:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:54.559817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.559183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241209803095832:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.562799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310983:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.562827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241211812310983:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.563389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310997:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.563696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241211812310983:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.567800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673735:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.567816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210639673735:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.568402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673749:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.568405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210639673749:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.569427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210639673744:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.578839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241211569810416:2111] [node 4] ICP35 event processing took too much time PendingConnection:279(19021202ns) {HandleHandshakeStatus:329(19020106ns) {DropIncomingHandshake:442(122ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(18919617ns) {ForwardSessionEventToSession:612(18902667ns) ForwardSessionEventToSession:612(2773ns) ForwardSessionEventToSession:612(1149ns) ForwardSessionEventToSession:612(727ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(2788ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(1107ns) ForwardSessionEventToSession:612(779ns)}}}
2025-12-24T02:10:54.580552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.580767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.580953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210639673735:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.581117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210463364958:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.582772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095840:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.582886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095843:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.583152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.583201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241209803095754:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.583217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.583224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241209803095754:2048] [node 1] ICP22 created new session: [6:7587241209803095846:2048]
2025-12-24T02:10:54.583231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241209803095846:2048] [node 1] ICS09 handshake done sender: [6:7587241209803095832:2115] self: [6:7587241209803095835:0] peer: [1:7587241210639673732:0] socket: 185 qp: -1
2025-12-24T02:10:54.583238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241209803095846:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.583246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.583635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241209803095834:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.583691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241209803095756:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.583698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.583704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241209803095756:2050] [node 3] ICP22 created new session: [6:7587241209803095848:2050]
2025-12-24T02:10:54.583709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241209803095848:2050] [node 3] ICS09 handshake done sender: [6:7587241209803095834:2117] self: [6:7587241209803095836:0] peer: [3:7587241211569810412:0] socket: 187 qp: -1
2025-12-24T02:10:54.583714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241209803095848:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.583719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.586941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241211812310980:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.586969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.586977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241211812310980:2108] [node 5] ICP22 created new session: [2:7587241211812311006:2108]
2025-12-24T02:10:54.586984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241211812311006:2108] [node 5] ICS09 handshake done sender: [2:7587241211812310983:2112] self: [2:7587241211812310982:0] peer: [5:7587241211451075978:0] socket: 175 qp: -1
2025-12-24T02:10:54.586989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241211812311006:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:54.586997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.587338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810431:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.587462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241211569810416:2111] [node 4] ICP18 handshake (actor [3:7587241211569810431:2117]) is too old
2025-12-24T02:10:54.587600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210463364891:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.587620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.587626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210463364891:2050] [node 3] ICP22 created new session: [4:7587241210463364978:2050]
2025-12-24T02:10:54.587631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210463364978:2050] [node 3] ICS09 handshake done sender: [4:7587241210463364958:2113] self: [4:7587241210463364974:0] peer: [3:7587241211569810417:0] socket: 183 qp: -1
2025-12-24T02:10:54.587636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210463364978:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.587643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.587887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211451075958:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.588852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075980:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.588999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211451075891:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.589013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.589018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211451075891:2049] [node 2] ICP22 created new session: [5:7587241211451075981:2049]
2025-12-24T02:10:54.589023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211451075981:2049] [node 2] ICS09 handshake done sender: [5:7587241211451075958:2112] self: [5:7587241211451075978:0] peer: [2:7587241211812310982:0] socket: 177 qp: -1
2025-12-24T02:10:54.589028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211451075981:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.589034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.590198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364977:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
2025-12-24T02:10:54.590975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241209803095840:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.591058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241209803095844:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.591107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241209803095755:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.591120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241209803095771:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.591557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210639673749:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.591728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364966:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.591737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241210463364966:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.594755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241211812311005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35197
2025-12-24T02:10:54.595130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310992:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.595143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241211812310992:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.596061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095853:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
2025-12-24T02:10:54.595901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241211812310998:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.596191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241211812310913:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.596205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.596212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241211812310913:2048] [node 1] ICP22 created new session: [2:7587241211812311010:2048]
2025-12-24T02:10:54.596219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241211812311010:2048] [node 1] ICS09 handshake done sender: [2:7587241211812310998:2118] self: [2:7587241211812311002:0] peer: [1:7587241210639673743:0] socket: 203 qp: -1
2025-12-24T02:10:54.596228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241211812311010:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.596235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.597205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241211812310992:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.597430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241209803095840:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.597506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241209803095755:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.597522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.597528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241209803095755:2049] [node 2] ICP22 created new session: [6:7587241209803095854:2049]
2025-12-24T02:10:54.597534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241209803095854:2049] [node 2] ICS09 handshake done sender: [6:7587241209803095840:2120] self: [6:7587241209803095851:0] peer: [2:7587241211812310991:0] socket: 207 qp: -1
2025-12-24T02:10:54.597540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241209803095854:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.597547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.598586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241211812310989:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.598605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.598610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241211812310989:2113] [node 6] ICP22 created new session: [2:7587241211812311012:2113]
2025-12-24T02:10:54.598614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241211812311012:2113] [node 6] ICS09 handshake done sender: [2:7587241211812310992:2115] self: [2:7587241211812310991:0] peer: [6:7587241209803095851:0] socket: 204 qp: -1
2025-12-24T02:10:54.598619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241211812311012:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:54.598624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.598262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210639673669:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.598287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.598294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210639673669:2048] [node 2] ICP22 created new session: [1:7587241210639673764:2048]
2025-12-24T02:10:54.598300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210639673764:2048] [node 2] ICS09 handshake done sender: [1:7587241210639673744:2113] self: [1:7587241210639673743:0] peer: [2:7587241211812311002:0] socket: 200 qp: -1
2025-12-24T02:10:54.598306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210639673764:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:54.598315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.598558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210639673753:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.598640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210639673731:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.598648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.598651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210639673731:2107] [node 5] ICP22 created new session: [1:7587241210639673767:2107]
2025-12-24T02:10:54.598656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210639673767:2107] [node 5] ICS09 handshake done sender: [1:7587241210639673735:2109] self: [1:7587241210639673734:0] peer: [5:7587241211451075975:0] socket: 166 qp: -1
2025-12-24T02:10:54.598660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210639673767:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:54.598665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.598754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210639673747:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.598762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.598766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210639673747:2115] [node 4] ICP22 created new session: [1:7587241210639673769:2115]
2025-12-24T02:10:54.598769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210639673769:2115] [node 4] ICS09 handshake done sender: [1:7587241210639673749:2116] self: [1:7587241210639673748:0] peer: [4:7587241210463364975:0] socket: 201 qp: -1
2025-12-24T02:10:54.598773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210639673769:2115] [node 4] ICS10 traffic start
2025-12-24T02:10:54.598777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.599859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810438:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.600048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210463364962:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.600223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241210463364889:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.600236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.600240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210463364889:2048] [node 1] ICP22 created new session: [4:7587241210463364982:2048]
2025-12-24T02:10:54.600246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210463364982:2048] [node 1] ICS09 handshake done sender: [4:7587241210463364962:2115] self: [4:7587241210463364975:0] peer: [1:7587241210639673748:0] socket: 205 qp: -1
2025-12-24T02:10:54.600252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210463364982:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.600259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.607354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241209803095856:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18246
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.610773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211451075956:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.610938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.614653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241209803095844:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.611369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210463364966:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.614803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810439:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.614852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241211569810347:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.614931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211451075890:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.614949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.614957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211451075890:2048] [node 1] ICP22 created new session: [5:7587241211451075984:2048]
2025-12-24T02:10:54.614963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211451075984:2048] [node 1] ICS09 handshake done sender: [5:7587241211451075956:2110] self: [5:7587241211451075975:0] peer: [1:7587241210639673734:0] socket: 171 qp: -1
2025-12-24T02:10:54.614970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211451075984:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.614977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.614776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241209803095771:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.614794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.614806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241209803095771:2061] [node 4] ICP22 created new session: [6:7587241209803095857:2061]
2025-12-24T02:10:54.614815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241209803095857:2061] [node 4] ICS09 handshake done sender: [6:7587241209803095844:2122] self: [6:7587241209803095852:0] peer: [4:7587241210463364965:0] socket: 217 qp: -1
2025-12-24T02:10:54.614823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241209803095857:2061] [node 4] ICS10 traffic start
2025-12-24T02:10:54.614834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.615577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241211812310994:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.615591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241211812310994:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.615844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210639673766:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46634
2025-12-24T02:10:54.616216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241210639673731:2107] [node 5] ICP18 handshake (actor [1:7587241210639673753:2117]) is too old
2025-12-24T02:10:54.619622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241211812310994:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.619930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241211569810442:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54348
2025-12-24T02:10:54.620088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075976:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.620126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211451075892:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.622781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241211812310914:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.622808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.622817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241211812310914:2049] [node 3] ICP22 created new session: [2:7587241211812311015:2049]
2025-12-24T02:10:54.622823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241211812311015:2049] [node 3] ICS09 handshake done sender: [2:7587241211812310994:2116] self: [2:7587241211812310993:0] peer: [3:7587241211569810441:0] socket: 196 qp: -1
2025-12-24T02:10:54.622831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241211812311015:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.622841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.626943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241211569810423:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.626957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241211569810423:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.630868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241211569810347:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32137) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241211569810439:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.654954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075983:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.658822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211569810439:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.659040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241210463364960:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.659066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.659083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210463364960:2111] [node 6] ICP22 created new session: [4:7587241210463364985:2111]
2025-12-24T02:10:54.659089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210463364985:2111] [node 6] ICS09 handshake done sender: [4:7587241210463364966:2117] self: [4:7587241210463364965:0] peer: [6:7587241209803095852:0] socket: 209 qp: -1
2025-12-24T02:10:54.659097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210463364985:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:54.659105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.659547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211451075983:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.659579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211451075960:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.659909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241211569810347:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.659922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.659927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211569810347:2049] [node 2] ICP22 created new session: [3:7587241211569810444:2049]
2025-12-24T02:10:54.659933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211569810444:2049] [node 2] ICS09 handshake done sender: [3:7587241211569810439:2119] self: [3:7587241211569810441:0] peer: [2:7587241211812310993:0] socket: 210 qp: -1
2025-12-24T02:10:54.659938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211569810444:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.659943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.660261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075989:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.660345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241211569810423:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.660483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211451075976:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.660533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211451075892:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.660546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.660550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211451075892:2050] [node 3] ICP22 created new session: [5:7587241211451075990:2050]
2025-12-24T02:10:54.660554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211451075990:2050] [node 3] ICS09 handshake done sender: [5:7587241211451075976:2118] self: [5:7587241211451075986:0] peer: [3:7587241211569810422:0] socket: 213 qp: -1
2025-12-24T02:10:54.660558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211451075990:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.660564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.661321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241211569810421:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.661337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.661342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241211569810421:2113] [node 5] ICP22 created new session: [3:7587241211569810446:2113]
2025-12-24T02:10:54.661346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241211569810446:2113] [node 5] ICS09 handshake done sender: [3:7587241211569810423:2114] self: [3:7587241211569810422:0] peer: [5:7587241211451075986:0] socket: 211 qp: -1
2025-12-24T02:10:54.661350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241211569810446:2113] [node 5] ICS10 traffic start
2025-12-24T02:10:54.661357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.661730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364968:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.661741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241210463364968:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.662177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241210463364984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14430
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.662683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211451075992:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33092
2025-12-24T02:10:54.662840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241210463364968:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.662977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211451075983:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.663058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241210463364984:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.663082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241210463364961:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.663090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.663094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241210463364961:2112] [node 5] ICP22 created new session: [4:7587241210463364988:2112]
2025-12-24T02:10:54.663098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241210463364988:2112] [node 5] ICS09 handshake done sender: [4:7587241210463364968:2118] self: [4:7587241210463364967:0] peer: [5:7587241211451075988:0] socket: 208 qp: -1
2025-12-24T02:10:54.663103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241210463364988:2112] [node 5] ICS10 traffic start
2025-12-24T02:10:54.663109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.663200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241210463364961:2112] [node 5] ICP18 handshake (actor [4:7587241210463364984:2121]) is too old
2025-12-24T02:10:54.663338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211451075960:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.663349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.663353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211451075960:2111] [node 4] ICP22 created new session: [5:7587241211451075993:2111]
2025-12-24T02:10:54.663356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211451075993:2111] [node 4] ICS09 handshake done sender: [5:7587241211451075983:2121] self: [5:7587241211451075988:0] peer: [4:7587241210463364967:0] socket: 157 qp: -1
2025-12-24T02:10:54.663360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211451075993:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:54.663365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.678793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:54.678891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241211812311010:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-12-24T02:10:54.678902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241211812310913:2048] [node 1] ICP30 unregister session Session# [2:7587241211812311010:2048] VirtualId# [2:7587241211812311002:0]
2025-12-24T02:10:54.678911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241211812311010:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.678970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.678996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.679035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:54.679103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241211569810432:2048] [node 1] ICS01 socket: 168 reason# EndOfStream
2025-12-24T02:10:54.679108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241211569810346:2048] [node 1] ICP30 unregister session Session# [3:7587241211569810432:2048] VirtualId# [3:7587241211569810426:0]
2025-12-24T02:10:54.679113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241211569810432:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.679138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.679159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.686150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:54.686512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:54.686576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241210463364982:2048] [node 1] ICS01 socket: 205 reason# EndOfStream
2025-12-24T02:10:54.686581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241210463364889:2048] [node 1] ICP30 unregister session Session# [4:7587241210463364982:2048] VirtualId# [4:7587241210463364975:0]
2025-12-24T02:10:54.686585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241210463364982:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.686616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.686636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.686232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:54.686300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241209803095846:2048] [node 1] ICS01 socket: 185 reason# EndOfStream
2025-12-24T02:10:54.686305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241209803095754:2048] [node 1] ICP30 unregister session Session# [6:7587241209803095846:2048] VirtualId# [6:7587241209803095835:0]
2025-12-24T02:10:54.686309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241209803095846:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.686354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.686377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.686252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211451075984:2048] [node 1] ICS01 socket: 171 reason# EndOfStream
2025-12-24T02:10:54.686266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211451075890:2048] [node 1] ICP30 unregister session Session# [5:7587241211451075984:2048] VirtualId# [5:7587241211451075975:0]
2025-12-24T02:10:54.686272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211451075984:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.686349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.686378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.687868Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:54.687945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241211569810444:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:10:54.687951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241211569810347:2049] [node 2] ICP30 unregister session Session# [3:7587241211569810444:2049] VirtualId# [3:7587241211569810441:0]
2025-12-24T02:10:54.687957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241211569810444:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.687993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.688010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.700439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:54.700545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211451075981:2049] [node 2] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:10:54.700554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211451075891:2049] [node 2] ICP30 unregister session Session# [5:7587241211451075981:2049] VirtualId# [5:7587241211451075978:0]
2025-12-24T02:10:54.700562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211451075981:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.700627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.700655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.701554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:54.701622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241210463364969:2049] [node 2] ICS07 socket disconnect 162 reason# EndOfStream
2025-12-24T02:10:54.701628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241210463364969:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.701668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.701686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241210463364969:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:54.701873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:54.701932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241209803095854:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:10:54.701937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241209803095755:2049] [node 2] ICP30 unregister session Session# [6:7587241209803095854:2049] VirtualId# [6:7587241209803095851:0]
2025-12-24T02:10:54.701943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241209803095854:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.701977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.701996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.702363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241210463364890:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32137) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.702375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241210463364969:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.702378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241210463364890:2049] [node 2] ICP30 unregister session Session# [4:7587241210463364969:2049] VirtualId# [4:7587241210463364952:0]
2025-12-24T02:10:54.702388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.702393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241210463364890:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.702396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32137) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.710767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241210463364890:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:54.713340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:54.713420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241209803095848:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-12-24T02:10:54.713429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241209803095756:2050] [node 3] ICP30 unregister session Session# [6:7587241209803095848:2050] VirtualId# [6:7587241209803095836:0]
2025-12-24T02:10:54.713435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241209803095848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.713486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:54.713509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:54.713661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:54.713755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211451075990:2050] [node 3] ICS01 socket: 213 reason# EndOfStream
2025-12-24T02:10:54.713765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211451075892:2050] [node 3] ICP30 unregister session Session# [5:7587241211451075990:2050] VirtualId# [5:7587241211451075986:0]
2025-12-24T02:10:54.713772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211451075990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.713834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:54.713862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:54.724974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:54.725058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241209803095857:2061] [node 4] ICS01 socket: 217 reason# EndOfStream
2025-12-24T02:10:54.725067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241209803095771:2061] [node 4] ICP30 unregister session Session# [6:7587241209803095857:2061] VirtualId# [6:7587241209803095852:0]
2025-12-24T02:10:54.725073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241209803095857:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.725127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:54.725150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:54.734769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:54.734865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241209803095841:2066] [node 5] ICS01 socket: 186 reason# EndOfStream
2025-12-24T02:10:54.734874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241209803095782:2066] [node 5] ICP30 unregister session Session# [6:7587241209803095841:2066] VirtualId# [6:7587241209803095837:0]
2025-12-24T02:10:54.734879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241209803095841:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.734928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:54.734952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:54.774852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241208679578055:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.774869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241208679578056:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.774730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210022290303:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:54.774775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210022290304:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:54.776442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241210213279913:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.777250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241210213279912:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.785373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211235444431:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.785386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211235444432:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.785391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211235444433:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.786876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241209802574672:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.787007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241209802574673:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.787013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241209802574674:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.788647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208191801448:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:54.788658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208191801449:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:54.788663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208191801450:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:54.811339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208191801498:2096] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.811747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241209802574724:2098] [node 6] ICP01 ready to work
2025-12-24T02:10:54.812385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801504:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.814026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210022290365:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.814125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210022290366:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:54.814672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444471:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.815363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241209802574730:2101] [node 5] ICP01 ready to work
2025-12-24T02:10:54.815257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241208679578119:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:54.815340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241208679578120:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:54.819036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241208191801511:2102] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.824103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574738:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.825814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208191801504:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.825642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241208679578126:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:54.825702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578125:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
2025-12-24T02:10:54.826529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801513:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.827017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213279976:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.827150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241210022290377:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:54.830856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241210213279980:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.834985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241210022290303:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17080846ns) {Configure:104(17080111ns) {ProcessConfigured:135(16829631ns) {StartInitialHandshake:157(16829185ns) {DropHandshakes:470(357ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(1283ns) {SwitchToState:179(453ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.835092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208191801511:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.835112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211235444499:2093] [node 6] ICP01 ready to work
2025-12-24T02:10:54.835270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241211235444500:2105] [node 4] ICP01 ready to work
2025-12-24T02:10:54.835322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444496:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.841533Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241211235444432:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26804076ns) {Configure:104(26802785ns) {ProcessConfigured:135(26415531ns) {StartInitialHandshake:157(26414925ns) {DropHandshakes:470(527ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1519ns) {SwitchToState:179(801ns)}}}}}
2025-12-24T02:10:54.841589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444498:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.842244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444471:2087] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.842282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444496:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.842964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290371:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.842980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210022290371:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.843448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241209802574738:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.843482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241209802574726:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.843485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241209802574726:2099] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.844702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574746:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.844718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574747:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.844739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241209802574672:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.845077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801524:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.845110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801525:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.846576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241209802574726:2099] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.842805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211235444431:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.846888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208191801504:2100] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.846707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241210213279984:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:54.846792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241210213279985:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:54.846839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213279983:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.847006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241209802574724:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.847021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.847027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241209802574724:2098] [node 6] ICP22 created new session: [4:7587241209802574753:2098]
2025-12-24T02:10:54.847032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241209802574753:2098] [node 6] ICS09 handshake done sender: [4:7587241209802574726:2099] self: [4:7587241209802574725:0] peer: [6:7587241208191801521:0] socket: 163 qp: -1
2025-12-24T02:10:54.847038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241209802574753:2098] [node 6] ICS10 traffic start
2025-12-24T02:10:54.847070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.848026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444497:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:10:54.852508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208191801511:2102] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.852531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.852539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208191801511:2102] [node 4] ICP22 created new session: [6:7587241208191801528:2102]
2025-12-24T02:10:54.852547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208191801528:2102] [node 4] ICS09 handshake done sender: [6:7587241208191801504:2100] self: [6:7587241208191801521:0] peer: [4:7587241209802574725:0] socket: 164 qp: -1
2025-12-24T02:10:54.852555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208191801528:2102] [node 4] ICS10 traffic start
2025-12-24T02:10:54.852563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.854790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241210213279912:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37528455ns) {Configure:104(37527648ns) {ProcessConfigured:135(37210106ns) {StartInitialHandshake:157(37209575ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:477(1209ns) {SwitchToState:179(607ns)}}}}}
2025-12-24T02:10:54.856287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290368:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.856300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210022290368:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.856831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210022290382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.857692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444513:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.858055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241209802574724:2098] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:54.858090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.858124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.859253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210022290371:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.858818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241208679578055:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28281320ns) {Configure:104(28280195ns) {ProcessConfigured:135(27915657ns) {StartInitialHandshake:157(27915139ns) {DropHandshakes:470(434ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(968ns) {SwitchToState:179(558ns)}}}}}
2025-12-24T02:10:54.862948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241209802574756:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.864446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241209802574674:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.864674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801527:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.863339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578125:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.864300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210022290368:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.871411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241209802574738:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.871512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241209802574672:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.871533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.871540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241209802574672:2048] [node 1] ICP22 created new session: [4:7587241209802574761:2048]
2025-12-24T02:10:54.871546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241209802574761:2048] [node 1] ICS09 handshake done sender: [4:7587241209802574738:2108] self: [4:7587241209802574752:0] peer: [1:7587241210022290367:0] socket: 173 qp: -1
2025-12-24T02:10:54.871555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241209802574761:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.871564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.880112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578136:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
2025-12-24T02:10:54.880156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
2025-12-24T02:10:54.880186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578135:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
2025-12-24T02:10:54.880603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578128:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.880608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241208679578128:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.881023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
2025-12-24T02:10:54.881228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241208679578128:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.881310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241208679578056:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.881323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.881331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241208679578056:2049] [node 3] ICP22 created new session: [2:7587241208679578141:2049]
2025-12-24T02:10:54.881337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241208679578141:2049] [node 3] ICS09 handshake done sender: [2:7587241208679578128:2113] self: [2:7587241208679578127:0] peer: [3:7587241210213279994:0] socket: 177 qp: -1
2025-12-24T02:10:54.881343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241208679578141:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.881349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.879011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210022290366:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.879034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.879043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210022290366:2108] [node 5] ICP22 created new session: [1:7587241210022290385:2108]
2025-12-24T02:10:54.879050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210022290385:2108] [node 5] ICS09 handshake done sender: [1:7587241210022290371:2110] self: [1:7587241210022290369:0] peer: [5:7587241211235444510:0] socket: 169 qp: -1
2025-12-24T02:10:54.879060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210022290385:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:54.879069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.879278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210022290365:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.879286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.879290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210022290365:2107] [node 4] ICP22 created new session: [1:7587241210022290387:2107]
2025-12-24T02:10:54.879295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210022290387:2107] [node 4] ICS09 handshake done sender: [1:7587241210022290368:2109] self: [1:7587241210022290367:0] peer: [4:7587241209802574752:0] socket: 168 qp: -1
2025-12-24T02:10:54.879299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210022290387:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:54.879303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.880271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279983:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.880301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279976:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.880325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241210213279912:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.880334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241210213279913:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.882797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279987:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.882809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241210213279987:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.883951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241208191801448:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19137504ns) {Configure:104(19136533ns) {ProcessConfigured:135(18809178ns) {StartInitialHandshake:157(18808406ns) {DropHandshakes:470(794ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1399ns) {SwitchToState:179(249ns)}}}}}
2025-12-24T02:10:54.884018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208191801527:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.884201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801533:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.890354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210022290391:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-12-24T02:10:54.888062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444498:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.888113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211235444496:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.886790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241208191801450:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19881692ns) {Configure:104(19880363ns) {ProcessConfigured:135(19593880ns) {StartInitialHandshake:157(19593316ns) {DropHandshakes:470(709ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(573ns) PrepareNewSessionHandshake:477(1100ns) {SwitchToState:179(629ns)}}}}}
2025-12-24T02:10:54.886826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208191801450:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.887038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574763:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.890569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241210213279913:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241210213279976:2108] held: no
2025-12-24T02:10:54.890632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279982:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.890635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241210213279982:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.890910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290391:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.894790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208191801533:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.894843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208191801448:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.895028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578136:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.895061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578137:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.898862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290379:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.898876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210022290379:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.900579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213279995:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.899427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290374:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.899437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210022290374:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.900727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210022290374:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.900806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210022290304:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.900821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.900828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210022290304:2049] [node 3] ICP22 created new session: [1:7587241210022290394:2049]
2025-12-24T02:10:54.900835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210022290394:2049] [node 3] ICS09 handshake done sender: [1:7587241210022290374:2111] self: [1:7587241210022290373:0] peer: [3:7587241210213279993:0] socket: 183 qp: -1
2025-12-24T02:10:54.900841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210022290394:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:54.900850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.903239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208191801513:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.903303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801538:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.903318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208191801449:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.903936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241209802574763:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.903977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241209802574673:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.905170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574766:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.906680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241209802574763:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.903027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578139:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.903066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241208679578055:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.903458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578124:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.903462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241208679578124:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.904160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578130:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.904165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241208679578130:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.904577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241208679578124:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.904756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241208679578120:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.904769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.904775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241208679578120:2109] [node 6] ICP22 created new session: [2:7587241208679578150:2109]
2025-12-24T02:10:54.904780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241208679578150:2109] [node 6] ICS09 handshake done sender: [2:7587241208679578124:2111] self: [2:7587241208679578123:0] peer: [6:7587241208191801539:0] socket: 176 qp: -1
2025-12-24T02:10:54.904785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241208679578150:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:54.904792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.906523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241208679578130:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.904138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211235444431:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.904158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.904165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211235444431:2048] [node 1] ICP22 created new session: [5:7587241211235444514:2048]
2025-12-24T02:10:54.904172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211235444514:2048] [node 1] ICS09 handshake done sender: [5:7587241211235444496:2111] self: [5:7587241211235444510:0] peer: [1:7587241210022290369:0] socket: 171 qp: -1
2025-12-24T02:10:54.904178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211235444514:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.904199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.906813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241209802574673:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.906827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.906833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241209802574673:2049] [node 2] ICP22 created new session: [4:7587241209802574767:2049]
2025-12-24T02:10:54.906838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241209802574767:2049] [node 2] ICS09 handshake done sender: [4:7587241209802574763:2120] self: [4:7587241209802574765:0] peer: [2:7587241208679578129:0] socket: 216 qp: -1
2025-12-24T02:10:54.906845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241209802574767:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.906852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.906984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241208679578126:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.907002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.907006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241208679578126:2107] [node 4] ICP22 created new session: [2:7587241208679578152:2107]
2025-12-24T02:10:54.907010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241208679578152:2107] [node 4] ICS09 handshake done sender: [2:7587241208679578130:2114] self: [2:7587241208679578129:0] peer: [4:7587241209802574765:0] socket: 174 qp: -1
2025-12-24T02:10:54.907013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241208679578152:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:54.907018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.910208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290381:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.910221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241210022290381:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.910739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241210022290365:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.915561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241208679578154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19313
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.919528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208191801449:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29003) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241208191801513:2107] held: no
2025-12-24T02:10:54.919590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801541:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.919610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801540:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.920084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213279996:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.920286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241210213279976:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.920458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241210022290396:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35184
2025-12-24T02:10:54.920867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210022290379:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.921195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241210022290381:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.920698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241211235444431:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16532158ns) {HandleHandshakeStatus:329(16531517ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(120ns) SwitchToState:179(92ns) ProcessPendingSessionEvents:495(9517051ns) {ForwardSessionEventToSession:612(9506609ns) ForwardSessionEventToSession:612(1704ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(421ns) ForwardSessionEventToSession:612(745ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(1031ns) ForwardSessionEventToSession:612(461ns) ForwardSessionEventToSession:612(982ns)}}}
2025-12-24T02:10:54.920737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444497:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.920903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211235444500:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.920952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211235444432:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.920996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.921688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241208679578122:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.921700Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241208679578122:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.922639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213279997:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.922721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210022290303:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.922736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.922744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210022290303:2048] [node 2] ICP22 created new session: [1:7587241210022290399:2048]
2025-12-24T02:10:54.922751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210022290399:2048] [node 2] ICS09 handshake done sender: [1:7587241210022290381:2114] self: [1:7587241210022290375:0] peer: [2:7587241208679578146:0] socket: 211 qp: -1
2025-12-24T02:10:54.922758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210022290399:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:54.922765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.922884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241210022290396:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.922918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241210022290303:2048] [node 2] ICP18 handshake (actor [1:7587241210022290396:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.924063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241209802574769:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37483
2025-12-24T02:10:54.924366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241210022290377:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.924382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.924388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241210022290377:2112] [node 6] ICP22 created new session: [1:7587241210022290401:2112]
2025-12-24T02:10:54.924393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241210022290401:2112] [node 6] ICS09 handshake done sender: [1:7587241210022290379:2113] self: [1:7587241210022290378:0] peer: [6:7587241208191801537:0] socket: 210 qp: -1
2025-12-24T02:10:54.924399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241210022290401:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:54.924405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.922450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444519:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.923773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211235444497:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.924052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211235444432:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.924060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.924066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211235444432:2049] [node 2] ICP22 created new session: [5:7587241211235444520:2049]
2025-12-24T02:10:54.924070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211235444520:2049] [node 2] ICS09 handshake done sender: [5:7587241211235444497:2112] self: [5:7587241211235444518:0] peer: [2:7587241208679578121:0] socket: 184 qp: -1
2025-12-24T02:10:54.924074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211235444520:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.924078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.923623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241208679578122:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.923838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241208679578119:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.923845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.923851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241208679578119:2108] [node 5] ICP22 created new session: [2:7587241208679578156:2108]
2025-12-24T02:10:54.923857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241208679578156:2108] [node 5] ICS09 handshake done sender: [2:7587241208679578122:2110] self: [2:7587241208679578121:0] peer: [5:7587241211235444518:0] socket: 175 qp: -1
2025-12-24T02:10:54.923864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241208679578156:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:54.923871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.924233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241208679578139:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.924560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241208679578055:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.924568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.924571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241208679578055:2048] [node 1] ICP22 created new session: [2:7587241208679578158:2048]
2025-12-24T02:10:54.924582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241208679578158:2048] [node 1] ICS09 handshake done sender: [2:7587241208679578139:2119] self: [2:7587241208679578146:0] peer: [1:7587241210022290375:0] socket: 212 qp: -1
2025-12-24T02:10:54.924588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241208679578158:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.924594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.927069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241209802574739:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.927096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241209802574739:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.931238Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241208679578055:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:54.935332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208191801513:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:54.936373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208191801449:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.936397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.936404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208191801449:2049] [node 2] ICP22 created new session: [6:7587241208191801542:2049]
2025-12-24T02:10:54.936410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208191801542:2049] [node 2] ICS09 handshake done sender: [6:7587241208191801513:2107] self: [6:7587241208191801539:0] peer: [2:7587241208679578123:0] socket: 178 qp: -1
2025-12-24T02:10:54.936418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208191801542:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.936428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.937160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208191801533:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:54.934910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241210213279982:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.935120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279996:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.935182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241210213279987:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.936709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241210213279913:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.936720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:54.936727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241210213279913:2049] [node 2] ICP22 created new session: [3:7587241210213280002:2049]
2025-12-24T02:10:54.936732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241210213280002:2049] [node 2] ICS09 handshake done sender: [3:7587241210213279976:2108] self: [3:7587241210213279994:0] peer: [2:7587241208679578127:0] socket: 180 qp: -1
2025-12-24T02:10:54.936737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241210213280002:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:54.936744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:54.936853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241210213279980:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.936860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.936864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241210213279980:2111] [node 6] ICP22 created new session: [3:7587241210213280004:2111]
2025-12-24T02:10:54.936869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241210213280004:2111] [node 6] ICS09 handshake done sender: [3:7587241210213279982:2112] self: [3:7587241210213279981:0] peer: [6:7587241208191801536:0] socket: 199 qp: -1
2025-12-24T02:10:54.936873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241210213280004:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:54.936878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:54.937336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241210213279983:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.938511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213280001:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
2025-12-24T02:10:54.938538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241210213279984:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.938548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.938553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241210213279984:2109] [node 4] ICP22 created new session: [3:7587241210213280006:2109]
2025-12-24T02:10:54.938558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241210213280006:2109] [node 4] ICS09 handshake done sender: [3:7587241210213279987:2114] self: [3:7587241210213279986:0] peer: [4:7587241209802574760:0] socket: 200 qp: -1
2025-12-24T02:10:54.938564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241210213280006:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:54.938570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.938640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241210213279984:2109] [node 4] ICP18 handshake (actor [3:7587241210213279996:2118]) is too old
2025-12-24T02:10:54.938686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241210213279912:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.938709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.938714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241210213279912:2048] [node 1] ICP22 created new session: [3:7587241210213280008:2048]
2025-12-24T02:10:54.938718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241210213280008:2048] [node 1] ICS09 handshake done sender: [3:7587241210213279983:2113] self: [3:7587241210213279993:0] peer: [1:7587241210022290373:0] socket: 198 qp: -1
2025-12-24T02:10:54.938723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241210213280008:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.938727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.940181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241209802574739:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.939960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241210213280011:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42050
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.940643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444523:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.940710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211235444498:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:54.943015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208191801448:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.943037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:54.943046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208191801448:2048] [node 1] ICP22 created new session: [6:7587241208191801545:2048]
2025-12-24T02:10:54.943054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208191801545:2048] [node 1] ICS09 handshake done sender: [6:7587241208191801533:2119] self: [6:7587241208191801537:0] peer: [1:7587241210022290378:0] socket: 214 qp: -1
2025-12-24T02:10:54.943061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208191801545:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:54.943070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:54.943174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241208191801448:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:54.943297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208191801527:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.942837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241209802574674:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241209802574756:2117] held: no
2025-12-24T02:10:54.942926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241209802574730:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.942936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.942941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241209802574730:2101] [node 5] ICP22 created new session: [4:7587241209802574771:2101]
2025-12-24T02:10:54.942947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241209802574771:2101] [node 5] ICS09 handshake done sender: [4:7587241209802574739:2109] self: [4:7587241209802574731:0] peer: [5:7587241211235444517:0] socket: 179 qp: -1
2025-12-24T02:10:54.942953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241209802574771:2101] [node 5] ICS10 traffic start
2025-12-24T02:10:54.942960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.943641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241209802574756:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.943609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213280001:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.943905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241209802574674:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.943919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.943925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241209802574674:2050] [node 3] ICP22 created new session: [4:7587241209802574773:2050]
2025-12-24T02:10:54.943930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241209802574773:2050] [node 3] ICS09 handshake done sender: [4:7587241209802574756:2117] self: [4:7587241209802574760:0] peer: [3:7587241210213279986:0] socket: 204 qp: -1
2025-12-24T02:10:54.943936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241209802574773:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.943942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.945451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241209802574730:2101] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:54.943273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444516:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.943292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211235444500:2105] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.943302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:54.943308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211235444500:2105] [node 4] ICP22 created new session: [5:7587241211235444524:2105]
2025-12-24T02:10:54.943312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211235444524:2105] [node 4] ICS09 handshake done sender: [5:7587241211235444498:2113] self: [5:7587241211235444517:0] peer: [4:7587241209802574731:0] socket: 185 qp: -1
2025-12-24T02:10:54.943319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211235444524:2105] [node 4] ICS10 traffic start
2025-12-24T02:10:54.943326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:54.943752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241211235444433:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.944351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241208191801524:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.944376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208191801450:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.944384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.944389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208191801450:2050] [node 3] ICP22 created new session: [6:7587241208191801547:2050]
2025-12-24T02:10:54.944393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208191801547:2050] [node 3] ICS09 handshake done sender: [6:7587241208191801527:2117] self: [6:7587241208191801536:0] peer: [3:7587241210213279981:0] socket: 201 qp: -1
2025-12-24T02:10:54.944398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208191801547:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.944402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.951009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241208191801498:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:54.950978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241210213279989:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.950992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241210213279989:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.951671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241210213279989:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.951747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241210213279985:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.951763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.951769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241210213279985:2110] [node 5] ICP22 created new session: [3:7587241210213280014:2110]
2025-12-24T02:10:54.951774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241210213280014:2110] [node 5] ICS09 handshake done sender: [3:7587241210213279989:2115] self: [3:7587241210213279988:0] peer: [5:7587241211235444526:0] socket: 197 qp: -1
2025-12-24T02:10:54.951780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241210213280014:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:54.951788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.954974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241211235444503:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:54.954987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241211235444503:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:54.956430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241211235444433:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17060) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241211235444516:2119] held: no
2025-12-24T02:10:54.956615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241211235444527:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32043
2025-12-24T02:10:54.956849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211235444516:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:54.956960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241211235444503:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:54.957037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241211235444433:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.957050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:54.957058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211235444433:2050] [node 3] ICP22 created new session: [5:7587241211235444529:2050]
2025-12-24T02:10:54.957064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211235444529:2050] [node 3] ICS09 handshake done sender: [5:7587241211235444516:2119] self: [5:7587241211235444526:0] peer: [3:7587241210213279988:0] socket: 230 qp: -1
2025-12-24T02:10:54.957072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211235444529:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:54.957078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:54.957199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241211235444499:2093] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:54.957209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:54.957214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241211235444499:2093] [node 6] ICP22 created new session: [5:7587241211235444531:2093]
2025-12-24T02:10:54.957218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241211235444531:2093] [node 6] ICS09 handshake done sender: [5:7587241211235444503:2115] self: [5:7587241211235444502:0] peer: [6:7587241208191801549:0] socket: 187 qp: -1
2025-12-24T02:10:54.957223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241211235444531:2093] [node 6] ICS10 traffic start
2025-12-24T02:10:54.957227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:54.960631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208191801498:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11133) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241208191801524:2115] held: no
2025-12-24T02:10:54.960691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241208191801550:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:10:54.962788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241208191801524:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:54.962947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241208191801498:2096] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:54.962965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:54.962975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241208191801498:2096] [node 5] ICP22 created new session: [6:7587241208191801551:2096]
2025-12-24T02:10:54.962982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241208191801551:2096] [node 5] ICS09 handshake done sender: [6:7587241208191801524:2115] self: [6:7587241208191801549:0] peer: [5:7587241211235444502:0] socket: 195 qp: -1
2025-12-24T02:10:54.962990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241208191801551:2096] [node 5] ICS10 traffic start
2025-12-24T02:10:54.963000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:54.975475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:54.975564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241210213280008:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:10:54.975572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241210213279912:2048] [node 1] ICP30 unregister session Session# [3:7587241210213280008:2048] VirtualId# [3:7587241210213279993:0]
2025-12-24T02:10:54.975579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241210213280008:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.975621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.975641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.976178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:54.976251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241208679578158:2048] [node 1] ICS07 socket disconnect 212 reason# EndOfStream
2025-12-24T02:10:54.976256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241208679578158:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.976302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.976322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241208679578158:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:54.977735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241208679578055:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.977750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241208679578158:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.977754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241208679578055:2048] [node 1] ICP30 unregister session Session# [2:7587241208679578158:2048] VirtualId# [2:7587241208679578146:0]
2025-12-24T02:10:54.977770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.977778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241208679578055:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.977780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.982778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:54.982880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208191801545:2048] [node 1] ICS01 socket: 214 reason# EndOfStream
2025-12-24T02:10:54.982888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208191801448:2048] [node 1] ICP30 unregister session Session# [6:7587241208191801545:2048] VirtualId# [6:7587241208191801537:0]
2025-12-24T02:10:54.982896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208191801545:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.982953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.982988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.986739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:54.986815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241209802574761:2048] [node 1] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:10:54.986823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241209802574672:2048] [node 1] ICP30 unregister session Session# [4:7587241209802574761:2048] VirtualId# [4:7587241209802574752:0]
2025-12-24T02:10:54.986828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241209802574761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.986885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.986912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.987993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:54.988058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241210213280002:2049] [node 2] ICS01 socket: 180 reason# EndOfStream
2025-12-24T02:10:54.988063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241210213279913:2049] [node 2] ICP30 unregister session Session# [3:7587241210213280002:2049] VirtualId# [3:7587241210213279994:0]
2025-12-24T02:10:54.988068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241210213280002:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.988102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.988121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.986360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:54.986445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241211235444514:2048] [node 1] ICS07 socket disconnect 171 reason# EndOfStream
2025-12-24T02:10:54.986451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211235444514:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.986509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:54.986530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241211235444514:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:54.986881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241211235444431:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.986891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211235444514:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.986896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211235444431:2048] [node 1] ICP30 unregister session Session# [5:7587241211235444514:2048] VirtualId# [5:7587241211235444510:0]
2025-12-24T02:10:54.986913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:54.986919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241211235444431:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.986922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28809) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.990773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:54.990865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208191801542:2049] [node 2] ICS01 socket: 178 reason# EndOfStream
2025-12-24T02:10:54.990872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208191801449:2049] [node 2] ICP30 unregister session Session# [6:7587241208191801542:2049] VirtualId# [6:7587241208191801539:0]
2025-12-24T02:10:54.990879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208191801542:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.990933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.990958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.991040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:54.991146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241209802574767:2049] [node 2] ICS07 socket disconnect 216 reason# EndOfStream
2025-12-24T02:10:54.991151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241209802574767:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.991185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.991207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241209802574767:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:54.992342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:54.992410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211235444520:2049] [node 2] ICS01 socket: 184 reason# EndOfStream
2025-12-24T02:10:54.992417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211235444432:2049] [node 2] ICP30 unregister session Session# [5:7587241211235444520:2049] VirtualId# [5:7587241211235444518:0]
2025-12-24T02:10:54.992422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211235444520:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:54.992466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:54.992488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.992722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241209802574673:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29003) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:54.992735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241209802574767:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:54.992739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241209802574673:2049] [node 2] ICP30 unregister session Session# [4:7587241209802574767:2049] VirtualId# [4:7587241209802574765:0]
2025-12-24T02:10:54.992751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:54.992757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241209802574673:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.992759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29003) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:54.998748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241211235444431:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:55.000744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241209802574673:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:55.000833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:55.000908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241209802574773:2050] [node 3] ICS01 socket: 204 reason# EndOfStream
2025-12-24T02:10:55.000916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241209802574674:2050] [node 3] ICP30 unregister session Session# [4:7587241209802574773:2050] VirtualId# [4:7587241209802574760:0]
2025-12-24T02:10:55.000922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241209802574773:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.000971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.000993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.002732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:55.002801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241211235444529:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-12-24T02:10:55.002808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241211235444433:2050] [node 3] ICP30 unregister session Session# [5:7587241211235444529:2050] VirtualId# [5:7587241211235444526:0]
2025-12-24T02:10:55.002813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241211235444529:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.002865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.002894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.002811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:55.002882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208191801547:2050] [node 3] ICS01 socket: 201 reason# EndOfStream
2025-12-24T02:10:55.002889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208191801450:2050] [node 3] ICP30 unregister session Session# [6:7587241208191801547:2050] VirtualId# [6:7587241208191801536:0]
2025-12-24T02:10:55.002894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208191801547:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.002942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.002964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.021125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:55.021222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241208191801528:2102] [node 4] ICS07 socket disconnect 164 reason# EndOfStream
2025-12-24T02:10:55.021228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241208191801528:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.021298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:55.021320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241208191801528:2102] [node 4] ICS15 start handshake
2025-12-24T02:10:55.021706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241208191801511:2102] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27538) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:55.021718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241208191801528:2102] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:55.021723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241208191801511:2102] [node 4] ICP30 unregister session Session# [6:7587241208191801528:2102] VirtualId# [6:7587241208191801521:0]
2025-12-24T02:10:55.021738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:55.021744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241208191801511:2102] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.021746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27538) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.031155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241208191801511:2102] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:55.064857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241213024038414:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:55.064865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241213024038415:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.066607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241215981574451:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.066618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241215981574452:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.067736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213494586889:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.067742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213494586890:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.070785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241213573946855:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.070793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241213573946856:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.070798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241213573946857:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.069176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241215957845508:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.069183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241215957845509:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.069187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241215957845510:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.102210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241216173565346:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.102221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241216173565347:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.102227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241216173565348:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.112833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241216173565407:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.113654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946915:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.113956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946916:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.112855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241215981574512:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:55.114056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241215981574516:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62513
2025-12-24T02:10:55.112973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241215957845569:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:55.124917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241215981574518:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.125507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241213024038483:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:55.125593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241213024038484:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:55.125660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241213024038485:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:55.126902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213494586950:2106] [node 4] ICP01 ready to work
2025-12-24T02:10:55.126997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213494586952:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:55.127075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213494586953:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.129492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946915:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.129674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241213573946922:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:55.129763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241213573946924:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.127483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241215981574526:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:55.128687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241215981574530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62513
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.130968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241215957845574:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:55.132230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845585:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
2025-12-24T02:10:55.131524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241216173565417:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:55.132245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845583:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
2025-12-24T02:10:55.132298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845584:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
2025-12-24T02:10:55.130658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946925:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.135319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946916:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.136968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946936:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.137022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241213573946856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.138029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565411:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.135101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845583:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.140198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946936:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.140237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241213573946922:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.141620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241213024038492:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53024
2025-12-24T02:10:55.141668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241213024038493:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53024
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.145957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586966:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
2025-12-24T02:10:55.145984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586967:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.146030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586968:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.142804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845585:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.146872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038492:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.148272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574514:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.148292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241215981574514:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.146067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586969:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
2025-12-24T02:10:55.146801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586968:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.146890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586969:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.150886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241215981574451:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25911909ns) {Configure:104(25911019ns) {ProcessConfigured:135(25603080ns) {StartInitialHandshake:157(25602444ns) {DropHandshakes:470(957ns) {DropIncomingHandshake:442(228ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(846ns) {SwitchToState:179(410ns)}}}}}
2025-12-24T02:10:55.150726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845578:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.150749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241215957845578:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.154891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586967:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.154791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241215957845509:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.154846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241215957845509:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17158790ns) {Configure:104(17157952ns) {ProcessConfigured:135(16845244ns) {StartInitialHandshake:157(16791275ns) {DropHandshakes:470(869ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(403ns) PrepareNewSessionHandshake:477(665ns) {SwitchToState:179(385ns)}} IncomingHandshake:261(43595ns) {DropIncomingHandshake:442(109ns) PrepareNewSessionHandshake:477(128ns) {SwitchToState:179(79ns)} GenerateSessionVirtualId:988(276ns) IssueIncomingHandshakeReply:193(36978ns) {SwitchToState:179(53ns)}}}}}
2025-12-24T02:10:55.155259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241216173565417:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23459323ns) {Configure:104(23458910ns) {ProcessConfigured:135(23249607ns) {StartInitialHandshake:157(23249313ns) {DropHandshakes:470(199ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(735ns) {SwitchToState:179(79ns)}}}}}
2025-12-24T02:10:55.154925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586966:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.154955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241213494586890:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.154966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241213494586889:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.158874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574530:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.158792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038478:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.158808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241213024038478:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.160960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241213024038478:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.160875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
2025-12-24T02:10:55.162877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241216173565411:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.162902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241216173565417:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.162755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845584:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.162794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241215957845510:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.166799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845571:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.166816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241215957845571:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.169404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574522:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.169428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241215981574522:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.170764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574525:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.170776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241215981574525:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.174908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213494586966:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.175538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241215981574532:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62513
2025-12-24T02:10:55.175619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574516:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.175767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241215981574451:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.178912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241213024038415:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.178936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.178945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241213024038415:2049] [node 3] ICP22 created new session: [1:7587241213024038497:2049]
2025-12-24T02:10:55.178955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241213024038497:2049] [node 3] ICS09 handshake done sender: [1:7587241213024038478:2108] self: [1:7587241213024038477:0] peer: [3:7587241213494586975:0] socket: 178 qp: -1
2025-12-24T02:10:55.178964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241213024038497:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.178985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.178924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241215981574514:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.178823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586957:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.178840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213494586957:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.180324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038493:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.183565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565426:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.184132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241216173565411:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.184198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241216173565417:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.184206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.184211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241216173565417:2110] [node 4] ICP22 created new session: [6:7587241216173565427:2110]
2025-12-24T02:10:55.184216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241216173565427:2110] [node 4] ICS09 handshake done sender: [6:7587241216173565411:2109] self: [6:7587241216173565425:0] peer: [4:7587241215957845570:0] socket: 179 qp: -1
2025-12-24T02:10:55.184220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241216173565427:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:55.184226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.183672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241215957845571:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.183749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241215957845569:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.183759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.183765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241215957845569:2107] [node 6] ICP22 created new session: [4:7587241215957845593:2107]
2025-12-24T02:10:55.183770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241215957845593:2107] [node 6] ICS09 handshake done sender: [4:7587241215957845571:2108] self: [4:7587241215957845570:0] peer: [6:7587241216173565425:0] socket: 171 qp: -1
2025-12-24T02:10:55.183778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241215957845593:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:55.183786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.182881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946940:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.182954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946941:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.186836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241213573946916:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.186934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241213573946856:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.186953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.186960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241213573946856:2049] [node 2] ICP22 created new session: [5:7587241213573946942:2049]
2025-12-24T02:10:55.186965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241213573946942:2049] [node 2] ICS09 handshake done sender: [5:7587241213573946916:2108] self: [5:7587241213573946937:0] peer: [2:7587241215981574513:0] socket: 173 qp: -1
2025-12-24T02:10:55.186974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241213573946942:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.186982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.189682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241215981574512:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.189704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.189713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241215981574512:2107] [node 5] ICP22 created new session: [2:7587241215981574536:2107]
2025-12-24T02:10:55.189720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241215981574536:2107] [node 5] ICS09 handshake done sender: [2:7587241215981574514:2108] self: [2:7587241215981574513:0] peer: [5:7587241213573946937:0] socket: 172 qp: -1
2025-12-24T02:10:55.189728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241215981574536:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:55.189736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.189918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574532:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.191057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241213024038499:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53024
2025-12-24T02:10:55.190834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845596:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
2025-12-24T02:10:55.190922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.195246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241215957845578:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.195485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241215957845574:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.195502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.195510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241215957845574:2109] [node 5] ICP22 created new session: [4:7587241215957845598:2109]
2025-12-24T02:10:55.195517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241215957845598:2109] [node 5] ICS09 handshake done sender: [4:7587241215957845578:2111] self: [4:7587241215957845577:0] peer: [5:7587241213573946939:0] socket: 194 qp: -1
2025-12-24T02:10:55.195524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241215957845598:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:55.195533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.195883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241215957845583:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.195955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241215957845509:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.195963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.195970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241215957845509:2049] [node 2] ICP22 created new session: [4:7587241215957845601:2049]
2025-12-24T02:10:55.195976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241215957845601:2049] [node 2] ICS09 handshake done sender: [4:7587241215957845583:2113] self: [4:7587241215957845590:0] peer: [2:7587241215981574524:0] socket: 195 qp: -1
2025-12-24T02:10:55.195980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241215957845601:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.195985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.198879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038474:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.198900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241213024038474:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.195163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241215981574525:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:10:55.199182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946940:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:10:55.200137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241215981574518:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.200154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.200162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241215981574518:2110] [node 4] ICP22 created new session: [2:7587241215981574542:2110]
2025-12-24T02:10:55.200168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241215981574542:2110] [node 4] ICS09 handshake done sender: [2:7587241215981574525:2113] self: [2:7587241215981574524:0] peer: [4:7587241215957845590:0] socket: 183 qp: -1
2025-12-24T02:10:55.200175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241215981574542:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:55.200183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.200506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241215981574541:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62513
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.201109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241213494586889:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.201130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.201137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213494586889:2048] [node 1] ICP22 created new session: [3:7587241213494586977:2048]
2025-12-24T02:10:55.201143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213494586977:2048] [node 1] ICS09 handshake done sender: [3:7587241213494586966:2114] self: [3:7587241213494586975:0] peer: [1:7587241213024038477:0] socket: 180 qp: -1
2025-12-24T02:10:55.201149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213494586977:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.201156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.201519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
2025-12-24T02:10:55.201891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.204636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241216173565430:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.204684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241216173565347:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.202853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241213573946922:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7269) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241213573946936:2117] held: no
2025-12-24T02:10:55.202932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946925:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.202991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241213573946855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.203018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946944:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.206852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586979:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.219191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845603:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.222755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213494586957:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.223575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241213024038502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53024
2025-12-24T02:10:55.221931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241215981574522:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.227010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241215981574452:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.227037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.227045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241215981574452:2049] [node 3] ICP22 created new session: [2:7587241215981574547:2049]
2025-12-24T02:10:55.227051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241215981574547:2049] [node 3] ICS09 handshake done sender: [2:7587241215981574522:2112] self: [2:7587241215981574521:0] peer: [3:7587241213494586974:0] socket: 182 qp: -1
2025-12-24T02:10:55.227058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241215981574547:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.227065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.227513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241215981574546:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62513
2025-12-24T02:10:55.229767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241215981574516:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.230802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241213573946856:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(43830853ns) {HandleHandshakeStatus:329(43830394ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(2232ns) SwitchToState:179(280ns) ProcessPendingSessionEvents:495(31786775ns) {ForwardSessionEventToSession:612(31772753ns) ForwardSessionEventToSession:612(2165ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(1072ns) ForwardSessionEventToSession:612(1020ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(1384ns)}}}
2025-12-24T02:10:55.230909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565433:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.226819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241213024038415:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(47862157ns) {HandleHandshakeStatus:329(47860758ns) {DropIncomingHandshake:442(113ns) DropOutgoingHandshake:457(349ns) SwitchToState:179(118ns) ProcessPendingSessionEvents:495(27828756ns) {ForwardSessionEventToSession:612(27814083ns) ForwardSessionEventToSession:612(2941ns) ForwardSessionEventToSession:612(1476ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(618ns) ForwardSessionEventToSession:612(1248ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(1116ns)}}}
2025-12-24T02:10:55.227116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038502:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.227170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241213024038474:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.227284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241213024038414:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.227294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.227303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241213024038414:2048] [node 2] ICP22 created new session: [1:7587241213024038504:2048]
2025-12-24T02:10:55.227309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241213024038504:2048] [node 2] ICS09 handshake done sender: [1:7587241213024038474:2106] self: [1:7587241213024038473:0] peer: [2:7587241215981574535:0] socket: 170 qp: -1
2025-12-24T02:10:55.227315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241213024038504:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:55.227322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.227473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241213024038414:2048] [node 2] ICP18 handshake (actor [1:7587241213024038502:2118]) is too old
2025-12-24T02:10:55.230883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038499:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.231040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241216173565433:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.231112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241216173565346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.231175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.231127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241213573946857:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.231173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241213573946936:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.235003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241215981574451:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.235048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.235060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241215981574451:2048] [node 1] ICP22 created new session: [2:7587241215981574549:2048]
2025-12-24T02:10:55.235067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241215981574549:2048] [node 1] ICS09 handshake done sender: [2:7587241215981574516:2109] self: [2:7587241215981574535:0] peer: [1:7587241213024038473:0] socket: 174 qp: -1
2025-12-24T02:10:55.235074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241215981574549:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.235100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.235186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241215981574451:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:55.235654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241215981574529:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.235668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241215981574529:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.235693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038491:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.235706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241213024038491:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.238807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038489:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.238818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241213024038489:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.240451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845604:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
2025-12-24T02:10:55.240561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241213494586890:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241213494586967:2115] held: no
2025-12-24T02:10:55.240599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213494586983:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25363
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.241698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586959:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.244091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565436:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.241710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213494586959:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.242326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213494586950:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.242346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.242353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213494586950:2106] [node 4] ICP22 created new session: [3:7587241213494586984:2106]
2025-12-24T02:10:55.244111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565437:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.242359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213494586984:2106] [node 4] ICS09 handshake done sender: [3:7587241213494586957:2110] self: [3:7587241213494586956:0] peer: [4:7587241215957845591:0] socket: 186 qp: -1
2025-12-24T02:10:55.242367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213494586984:2106] [node 4] ICS10 traffic start
2025-12-24T02:10:55.242375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.246881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241215981574529:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.247016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241213024038491:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.247061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213494586967:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.247269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213494586959:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.247653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241213024038489:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.242902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241215957845603:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.242938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241215957845584:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.246729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241215957845510:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.246745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.246752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241215957845510:2050] [node 3] ICP22 created new session: [4:7587241215957845605:2050]
2025-12-24T02:10:55.246758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241215957845605:2050] [node 3] ICS09 handshake done sender: [4:7587241215957845584:2114] self: [4:7587241215957845591:0] peer: [3:7587241213494586956:0] socket: 196 qp: -1
2025-12-24T02:10:55.246767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241215957845605:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.246774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.247221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241215981574526:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.247245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.247253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241215981574526:2111] [node 6] ICP22 created new session: [2:7587241215981574552:2111]
2025-12-24T02:10:55.247260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241215981574552:2111] [node 6] ICS09 handshake done sender: [2:7587241215981574529:2114] self: [2:7587241215981574528:0] peer: [6:7587241216173565432:0] socket: 219 qp: -1
2025-12-24T02:10:55.247099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241213573946922:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.247266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241215981574552:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:55.247273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.247132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.247140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241213573946922:2110] [node 4] ICP22 created new session: [5:7587241213573946948:2110]
2025-12-24T02:10:55.247146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241213573946948:2110] [node 4] ICS09 handshake done sender: [5:7587241213573946936:2117] self: [5:7587241213573946939:0] peer: [4:7587241215957845577:0] socket: 199 qp: -1
2025-12-24T02:10:55.247156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241213573946948:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:55.247176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.247167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241216173565431:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.250785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241216173565433:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.250730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241213024038485:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.250756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.250764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241213024038485:2111] [node 6] ICP22 created new session: [1:7587241213024038510:2111]
2025-12-24T02:10:55.250770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241213024038510:2111] [node 6] ICS09 handshake done sender: [1:7587241213024038491:2114] self: [1:7587241213024038490:0] peer: [6:7587241216173565435:0] socket: 187 qp: -1
2025-12-24T02:10:55.250776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241213024038510:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:55.250785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.254872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241213024038484:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.254893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.254898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241213024038484:2110] [node 5] ICP22 created new session: [1:7587241213024038512:2110]
2025-12-24T02:10:55.254903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241213024038512:2110] [node 5] ICS09 handshake done sender: [1:7587241213024038489:2113] self: [1:7587241213024038488:0] peer: [5:7587241213573946946:0] socket: 205 qp: -1
2025-12-24T02:10:55.254908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241213024038512:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:55.254916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.254839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241216173565346:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.254859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.254867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241216173565346:2048] [node 1] ICP22 created new session: [6:7587241216173565439:2048]
2025-12-24T02:10:55.254873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241216173565439:2048] [node 1] ICS09 handshake done sender: [6:7587241216173565433:2119] self: [6:7587241216173565435:0] peer: [1:7587241213024038490:0] socket: 233 qp: -1
2025-12-24T02:10:55.254880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241216173565439:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.254888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.254896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241215957845508:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.261156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241213494586890:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.261181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.261192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213494586890:2049] [node 2] ICP22 created new session: [3:7587241213494586987:2049]
2025-12-24T02:10:55.261204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213494586987:2049] [node 2] ICS09 handshake done sender: [3:7587241213494586967:2115] self: [3:7587241213494586974:0] peer: [2:7587241215981574521:0] socket: 207 qp: -1
2025-12-24T02:10:55.261214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213494586987:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.261224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.261344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213494586952:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.261356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.261360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213494586952:2107] [node 5] ICP22 created new session: [3:7587241213494586989:2107]
2025-12-24T02:10:55.261366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213494586989:2107] [node 5] ICS09 handshake done sender: [3:7587241213494586959:2111] self: [3:7587241213494586958:0] peer: [5:7587241213573946947:0] socket: 185 qp: -1
2025-12-24T02:10:55.261370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213494586989:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:55.261376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.262876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241213024038487:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.262890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241213024038487:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.263997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241213024038487:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.264075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241213024038483:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.264088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.264092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241213024038483:2109] [node 4] ICP22 created new session: [1:7587241213024038515:2109]
2025-12-24T02:10:55.264097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241213024038515:2109] [node 4] ICS09 handshake done sender: [1:7587241213024038487:2112] self: [1:7587241213024038486:0] peer: [4:7587241215957845607:0] socket: 223 qp: -1
2025-12-24T02:10:55.264102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241213024038515:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:55.264108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.263920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241215957845608:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25884
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.269810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241216173565348:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.269859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241216173565348:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(18924697ns) {IncomingHandshake:261(18924255ns) {DropIncomingHandshake:442(189ns) PrepareNewSessionHandshake:477(511ns) {SwitchToState:179(119ns)} GenerateSessionVirtualId:988(221ns) IssueIncomingHandshakeReply:193(18918125ns) {SwitchToState:179(891ns)}}}
2025-12-24T02:10:55.267005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946950:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.267267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241213573946925:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.267336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241213573946951:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28260
2025-12-24T02:10:55.269931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241216173565430:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.270136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241213573946857:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.270153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.270160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241213573946857:2050] [node 3] ICP22 created new session: [5:7587241213573946952:2050]
2025-12-24T02:10:55.270165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241213573946952:2050] [node 3] ICS09 handshake done sender: [5:7587241213573946925:2113] self: [5:7587241213573946947:0] peer: [3:7587241213494586958:0] socket: 188 qp: -1
2025-12-24T02:10:55.270171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241213573946952:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.270188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.272646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241215957845508:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241215957845603:2119] held: no
2025-12-24T02:10:55.272682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241215957845603:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.272788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241215957845508:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.272802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.272808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241215957845508:2048] [node 1] ICP22 created new session: [4:7587241215957845609:2048]
2025-12-24T02:10:55.272814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241215957845609:2048] [node 1] ICS09 handshake done sender: [4:7587241215957845603:2119] self: [4:7587241215957845607:0] peer: [1:7587241213024038486:0] socket: 224 qp: -1
2025-12-24T02:10:55.272820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241215957845609:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.272829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.273796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241216173565347:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.273816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.273822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241216173565347:2049] [node 2] ICP22 created new session: [6:7587241216173565441:2049]
2025-12-24T02:10:55.273827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241216173565441:2049] [node 2] ICS09 handshake done sender: [6:7587241216173565430:2117] self: [6:7587241216173565432:0] peer: [2:7587241215981574528:0] socket: 221 qp: -1
2025-12-24T02:10:55.273833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241216173565441:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.273840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.274757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213494586961:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.274780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213494586961:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.290923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241216173565347:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17087953ns) {HandleHandshakeStatus:329(17087202ns) {DropIncomingHandshake:442(171ns) DropOutgoingHandshake:457(1000ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(40625ns) {ForwardSessionEventToSession:612(29983ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(1056ns) ForwardSessionEventToSession:612(729ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(1075ns) ForwardSessionEventToSession:612(696ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(575ns) ForwardSessionEventToSession:612(798ns)}}}
2025-12-24T02:10:55.290945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241216173565347:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.298803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241216173565434:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.298949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241216173565407:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.298974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565443:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.298831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241213573946940:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.299488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241213573946855:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241213573946940:2119] held: no
2025-12-24T02:10:55.299514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241213573946855:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.299526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.299532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241213573946855:2048] [node 1] ICP22 created new session: [5:7587241213573946954:2048]
2025-12-24T02:10:55.299539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241213573946954:2048] [node 1] ICS09 handshake done sender: [5:7587241213573946940:2119] self: [5:7587241213573946946:0] peer: [1:7587241213024038488:0] socket: 206 qp: -1
2025-12-24T02:10:55.299547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241213573946954:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.299555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.299774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241213573946931:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.299778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241213573946931:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.300540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241213573946931:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.300626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241213573946924:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.300644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.300649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241213573946924:2112] [node 6] ICP22 created new session: [5:7587241213573946957:2112]
2025-12-24T02:10:55.300654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241213573946957:2112] [node 6] ICS09 handshake done sender: [5:7587241213573946931:2115] self: [5:7587241213573946930:0] peer: [6:7587241216173565444:0] socket: 236 qp: -1
2025-12-24T02:10:55.300659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241213573946957:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:55.300665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.304054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241216173565407:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25710) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241216173565434:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.306092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241216173565445:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6706
2025-12-24T02:10:55.302884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213494586961:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.308306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241216173565434:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.308365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241216173565407:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.308376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.308382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241216173565407:2107] [node 5] ICP22 created new session: [6:7587241216173565446:2107]
2025-12-24T02:10:55.308388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241216173565446:2107] [node 5] ICS09 handshake done sender: [6:7587241216173565434:2120] self: [6:7587241216173565444:0] peer: [5:7587241213573946930:0] socket: 237 qp: -1
2025-12-24T02:10:55.308394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241216173565446:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:55.308415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.308751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241216173565431:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.308871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241216173565348:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.308883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.308888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241216173565348:2050] [node 3] ICP22 created new session: [6:7587241216173565448:2050]
2025-12-24T02:10:55.308893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241216173565448:2050] [node 3] ICS09 handshake done sender: [6:7587241216173565431:2118] self: [6:7587241216173565438:0] peer: [3:7587241213494586960:0] socket: 229 qp: -1
2025-12-24T02:10:55.308897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241216173565448:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.308901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.310806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213494586953:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.310828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.310834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213494586953:2108] [node 6] ICP22 created new session: [3:7587241213494586992:2108]
2025-12-24T02:10:55.310839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213494586992:2108] [node 6] ICS09 handshake done sender: [3:7587241213494586961:2112] self: [3:7587241213494586960:0] peer: [6:7587241216173565438:0] socket: 228 qp: -1
2025-12-24T02:10:55.310843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213494586992:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:55.310850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.321328Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:55.321418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241215981574549:2048] [node 1] ICS01 socket: 174 reason# EndOfStream
2025-12-24T02:10:55.321427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241215981574451:2048] [node 1] ICP30 unregister session Session# [2:7587241215981574549:2048] VirtualId# [2:7587241215981574535:0]
2025-12-24T02:10:55.321434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241215981574549:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.321484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.321508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.326774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:55.326869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241215957845609:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-12-24T02:10:55.326879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241215957845508:2048] [node 1] ICP30 unregister session Session# [4:7587241215957845609:2048] VirtualId# [4:7587241215957845607:0]
2025-12-24T02:10:55.326886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241215957845609:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.326945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.326972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.323038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:55.323127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241213494586977:2048] [node 1] ICS01 socket: 180 reason# EndOfStream
2025-12-24T02:10:55.323134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241213494586889:2048] [node 1] ICP30 unregister session Session# [3:7587241213494586977:2048] VirtualId# [3:7587241213494586975:0]
2025-12-24T02:10:55.323140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241213494586977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.323182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.323203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.328625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:55.328697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241216173565439:2048] [node 1] ICS07 socket disconnect 233 reason# EndOfStream
2025-12-24T02:10:55.328702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241216173565439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.328751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.328770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241216173565439:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:55.329123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241216173565346:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:55.329130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241216173565439:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:55.329135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241216173565346:2048] [node 1] ICP30 unregister session Session# [6:7587241216173565439:2048] VirtualId# [6:7587241216173565435:0]
2025-12-24T02:10:55.329150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.329154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241216173565346:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.329157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.330750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:55.330840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241213573946954:2048] [node 1] ICS01 socket: 206 reason# EndOfStream
2025-12-24T02:10:55.330847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241213573946855:2048] [node 1] ICP30 unregister session Session# [5:7587241213573946954:2048] VirtualId# [5:7587241213573946946:0]
2025-12-24T02:10:55.330853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241213573946954:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.330908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.330929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.332467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:55.332546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241213494586987:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:10:55.332552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241213494586890:2049] [node 2] ICP30 unregister session Session# [3:7587241213494586987:2049] VirtualId# [3:7587241213494586974:0]
2025-12-24T02:10:55.332559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241213494586987:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.332603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.332625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.337842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241216173565346:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:55.344659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:55.344755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241216173565441:2049] [node 2] ICS01 socket: 221 reason# EndOfStream
2025-12-24T02:10:55.344764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241216173565347:2049] [node 2] ICP30 unregister session Session# [6:7587241216173565441:2049] VirtualId# [6:7587241216173565432:0]
2025-12-24T02:10:55.344771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241216173565441:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.344828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.344853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.346898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:55.346995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241215957845601:2049] [node 2] ICS01 socket: 195 reason# EndOfStream
2025-12-24T02:10:55.347002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241215957845509:2049] [node 2] ICP30 unregister session Session# [4:7587241215957845601:2049] VirtualId# [4:7587241215957845590:0]
2025-12-24T02:10:55.347009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241215957845601:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.347073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.347111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.347150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:55.347214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241213573946942:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-12-24T02:10:55.347217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241213573946942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.347243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.347265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241213573946942:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:55.347745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241213573946856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:55.347753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241213573946942:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:55.347757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241213573946856:2049] [node 2] ICP30 unregister session Session# [5:7587241213573946942:2049] VirtualId# [5:7587241213573946937:0]
2025-12-24T02:10:55.347767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.347772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241213573946856:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.347775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.357571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241213573946856:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:55.358531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:55.358641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241216173565448:2050] [node 3] ICS01 socket: 229 reason# EndOfStream
2025-12-24T02:10:55.358650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241216173565348:2050] [node 3] ICP30 unregister session Session# [6:7587241216173565448:2050] VirtualId# [6:7587241216173565438:0]
2025-12-24T02:10:55.358656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241216173565448:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.358801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.358834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.362769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:55.362872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241213573946952:2050] [node 3] ICS01 socket: 188 reason# EndOfStream
2025-12-24T02:10:55.362880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241213573946857:2050] [node 3] ICP30 unregister session Session# [5:7587241213573946952:2050] VirtualId# [5:7587241213573946947:0]
2025-12-24T02:10:55.362887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241213573946952:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.362950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.362977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.370786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:55.370905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241216173565427:2110] [node 4] ICS01 socket: 179 reason# EndOfStream
2025-12-24T02:10:55.370914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241216173565417:2110] [node 4] ICP30 unregister session Session# [6:7587241216173565427:2110] VirtualId# [6:7587241216173565425:0]
2025-12-24T02:10:55.370922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241216173565427:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.370987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:55.371013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:55.404075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212866778219:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:55.404082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212866778220:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.409797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241215688308902:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.409806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241215688308903:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.411984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241212848724850:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.412092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241212848724852:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.412115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241212848724851:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.414418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241216241230385:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.414426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241216241230386:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.414430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241216241230387:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.407165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241216133268865:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.407179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241216133268866:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.417979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241215112486198:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.417987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241215112486199:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.417992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241215112486200:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.464962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212866778286:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:55.465064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212866778287:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:55.465141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212866778288:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:55.466398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241216241230446:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:55.467023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241216133268926:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.474824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230443:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.475755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308962:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.475927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486258:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.478956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268932:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.482974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241215112486263:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:55.486799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241212866778286:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21595905ns) {Configure:104(21594604ns) {ProcessConfigured:135(21328275ns) {StartInitialHandshake:157(21327701ns) {DropHandshakes:470(636ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1581ns) {SwitchToState:179(1150ns)}}}}}
2025-12-24T02:10:55.486895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241215688308968:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:55.486996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241215688308969:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:55.487006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241216133268939:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.487706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212866778298:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4181
2025-12-24T02:10:55.491126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241212848724916:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:55.491231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241212848724917:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.491916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724914:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.491952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724915:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.492440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.493109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.492921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230457:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.494420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212866778299:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4181
2025-12-24T02:10:55.496096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241216241230462:2114] [node 6] ICP01 ready to work
2025-12-24T02:10:55.499716Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241215112486200:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16851387ns) {Configure:104(16850189ns) {ProcessConfigured:135(16485869ns) {StartInitialHandshake:157(16485530ns) {DropHandshakes:470(1420ns) {DropIncomingHandshake:442(426ns) DropOutgoingHandshake:457(228ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(850ns) {SwitchToState:179(118ns)}}}}}
2025-12-24T02:10:55.499902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241215112486269:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:55.501181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241215112486263:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18146304ns) {Configure:104(18145551ns) {ProcessConfigured:135(17855573ns) {StartInitialHandshake:157(17855082ns) {DropHandshakes:470(705ns) {DropIncomingHandshake:442(134ns) DropOutgoingHandshake:457(108ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(1167ns) {SwitchToState:179(829ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.499107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241216133268937:2111] [node 5] ICP01 ready to work
2025-12-24T02:10:55.502420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212866778300:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4181
2025-12-24T02:10:55.501814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241215688308902:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17863789ns) {Configure:104(17862598ns) {ProcessConfigured:135(17444452ns) {StartInitialHandshake:157(17444103ns) {DropHandshakes:470(841ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(488ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(588ns)}}}}}
2025-12-24T02:10:55.502336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308973:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.502482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241212848724850:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18252428ns) {Configure:104(18251772ns) {ProcessConfigured:135(17777126ns) {StartInitialHandshake:157(17776931ns) {DropHandshakes:470(356ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(972ns) {SwitchToState:179(116ns)}}}}}
2025-12-24T02:10:55.506340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241215688308966:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:55.506896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241215112486258:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.506940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241215112486198:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.511429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212866778301:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4181
2025-12-24T02:10:55.515020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778301:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.515103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268932:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.515152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241216133268865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.515798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778280:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.515807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212866778280:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.516459Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241216241230462:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18042039ns) {Configure:104(18040703ns) {ProcessConfigured:135(17701779ns) {StartInitialHandshake:157(17700931ns) {DropHandshakes:470(1214ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(1325ns) {SwitchToState:179(575ns)}}}}}
2025-12-24T02:10:55.518805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724915:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.518868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241215688308968:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31821535ns) {Configure:104(31820819ns) {ProcessConfigured:135(31514975ns) {StartInitialHandshake:157(31514507ns) {DropHandshakes:470(553ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:477(986ns) {SwitchToState:179(514ns)}}}}}
2025-12-24T02:10:55.519170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230443:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.519241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241216241230385:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.519618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241216241230387:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20203377ns) {Configure:104(20202169ns) {ProcessConfigured:135(19866337ns) {StartInitialHandshake:157(19865360ns) {DropHandshakes:470(553ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(957ns) {SwitchToState:179(282ns)}}}}}
2025-12-24T02:10:55.523354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268944:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.522808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778293:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.522827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212866778293:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.522880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778291:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.522887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212866778291:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.523155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778299:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.525056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308962:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.525137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241215688308903:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.525419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212866778293:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.525421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268930:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.525436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241216133268930:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.526408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241216133268930:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.526634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268943:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.526678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241216133268866:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.526712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.526723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241216133268866:2049] [node 3] ICP22 created new session: [2:7587241216133268951:2049]
2025-12-24T02:10:55.526728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241216133268951:2049] [node 3] ICS09 handshake done sender: [2:7587241216133268930:2108] self: [2:7587241216133268929:0] peer: [3:7587241215688308982:0] socket: 181 qp: -1
2025-12-24T02:10:55.526738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241216133268951:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.526757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.526986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308973:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.527235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230457:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.527278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241216241230446:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.527390Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241215112486269:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27130910ns) {Configure:104(27130431ns) {ProcessConfigured:135(26904744ns) {StartInitialHandshake:157(26903864ns) {DropHandshakes:470(298ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(1038ns) {SwitchToState:179(468ns)}}}}}
2025-12-24T02:10:55.527359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.527507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724919:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.527524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241212848724919:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.527970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230467:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.528115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241212848724919:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.528259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241212848724916:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.528277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.528285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241212848724916:2108] [node 5] ICP22 created new session: [4:7587241212848724930:2108]
2025-12-24T02:10:55.528291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241212848724930:2108] [node 5] ICS09 handshake done sender: [4:7587241212848724919:2112] self: [4:7587241212848724918:0] peer: [5:7587241216241230466:0] socket: 194 qp: -1
2025-12-24T02:10:55.528299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241212848724930:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:55.528314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.528366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230469:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.528795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241216241230457:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.528906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241216241230446:2107] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.528921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.528927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241216241230446:2107] [node 4] ICP22 created new session: [5:7587241216241230470:2107]
2025-12-24T02:10:55.528932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241216241230470:2107] [node 4] ICS09 handshake done sender: [5:7587241216241230457:2113] self: [5:7587241216241230466:0] peer: [4:7587241212848724918:0] socket: 197 qp: -1
2025-12-24T02:10:55.528938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241216241230470:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:55.528958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.530831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.531961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.532007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486278:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.532324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241215112486278:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.532356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241215112486269:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.532681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212866778280:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.530848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268954:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.534763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308985:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.534733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212866778288:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.534765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.534776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212866778288:2110] [node 6] ICP22 created new session: [1:7587241212866778307:2110]
2025-12-24T02:10:55.534785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212866778307:2110] [node 6] ICS09 handshake done sender: [1:7587241212866778293:2112] self: [1:7587241212866778292:0] peer: [6:7587241215112486276:0] socket: 176 qp: -1
2025-12-24T02:10:55.534796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212866778307:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:55.534837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.535597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486280:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.535662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486279:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.535715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241215112486258:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.535881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241215112486198:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.535900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.535910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241215112486198:2048] [node 1] ICP22 created new session: [6:7587241215112486282:2048]
2025-12-24T02:10:55.535917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241215112486282:2048] [node 1] ICS09 handshake done sender: [6:7587241215112486258:2106] self: [6:7587241215112486276:0] peer: [1:7587241212866778292:0] socket: 178 qp: -1
2025-12-24T02:10:55.535928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241215112486282:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.535944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.538588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778298:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.538621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212866778219:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.538634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.538644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212866778219:2048] [node 2] ICP22 created new session: [1:7587241212866778309:2048]
2025-12-24T02:10:55.538650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212866778309:2048] [node 2] ICS09 handshake done sender: [1:7587241212866778280:2107] self: [1:7587241212866778279:0] peer: [2:7587241216133268947:0] socket: 174 qp: -1
2025-12-24T02:10:55.538656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212866778309:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:55.538664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.540617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230473:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.540909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241216241230443:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.540981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241216241230385:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.541036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.541044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241216241230385:2048] [node 1] ICP22 created new session: [5:7587241216241230474:2048]
2025-12-24T02:10:55.541051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241216241230474:2048] [node 1] ICS09 handshake done sender: [5:7587241216241230443:2105] self: [5:7587241216241230465:0] peer: [1:7587241212866778290:0] socket: 177 qp: -1
2025-12-24T02:10:55.541059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241216241230474:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.541079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.540105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241212866778219:2048] [node 2] ICP18 handshake (actor [1:7587241212866778298:2115]) is too old
2025-12-24T02:10:55.540702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212866778291:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.543035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241216133268865:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241216133268932:2109] held: no
2025-12-24T02:10:55.543033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241215688308966:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36585290ns) {Configure:104(36584355ns) {ProcessConfigured:135(36282398ns) {StartInitialHandshake:157(36281660ns) {DropHandshakes:470(768ns) {DropIncomingHandshake:442(100ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(1231ns) {SwitchToState:179(588ns)}}}}}
2025-12-24T02:10:55.543104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241215688308962:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.543563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724921:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.543587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241212848724921:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.544222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212866778287:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.544245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.544253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212866778287:2109] [node 5] ICP22 created new session: [1:7587241212866778313:2109]
2025-12-24T02:10:55.544258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212866778313:2109] [node 5] ICS09 handshake done sender: [1:7587241212866778291:2111] self: [1:7587241212866778290:0] peer: [5:7587241216241230465:0] socket: 175 qp: -1
2025-12-24T02:10:55.544265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212866778313:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:55.544273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.550996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268955:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.551387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241215688308903:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.551412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.551418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241215688308903:2049] [node 2] ICP22 created new session: [3:7587241215688308989:2049]
2025-12-24T02:10:55.551424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241215688308989:2049] [node 2] ICS09 handshake done sender: [3:7587241215688308962:2107] self: [3:7587241215688308982:0] peer: [2:7587241216133268929:0] socket: 182 qp: -1
2025-12-24T02:10:55.551430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241215688308989:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.551447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.551117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241216133268866:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24379088ns) {HandleHandshakeStatus:329(24378340ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(202ns) SwitchToState:179(25ns) ProcessPendingSessionEvents:495(16278388ns) {ForwardSessionEventToSession:612(16262939ns) ForwardSessionEventToSession:612(2332ns) ForwardSessionEventToSession:612(965ns) ForwardSessionEventToSession:612(746ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(2034ns) ForwardSessionEventToSession:612(478ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(1575ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.551782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308987:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.551800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308988:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.551943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241216133268932:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.552021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241216133268865:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.552032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.552045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241216133268865:2048] [node 1] ICP22 created new session: [2:7587241216133268956:2048]
2025-12-24T02:10:55.552051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241216133268956:2048] [node 1] ICS09 handshake done sender: [2:7587241216133268932:2109] self: [2:7587241216133268947:0] peer: [1:7587241212866778279:0] socket: 179 qp: -1
2025-12-24T02:10:55.552058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241216133268956:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.552065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.552106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230477:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.554792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241212848724851:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(52247959ns) {Configure:104(52247014ns) {ProcessConfigured:135(51964287ns) {StartInitialHandshake:157(51963669ns) {DropHandshakes:470(402ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1680ns) {SwitchToState:179(749ns)}}}}}
2025-12-24T02:10:55.554832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241212848724851:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.554854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241212848724916:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:55.555626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308987:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.555765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486284:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.555839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486285:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.556022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241215112486284:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.556055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241215112486285:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.557664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230464:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.557677Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241216241230464:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.557446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241215112486199:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.557475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241215112486263:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.555545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241212866778288:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29971434ns) {HandleHandshakeStatus:329(29970750ns) {DropIncomingHandshake:442(677ns) DropOutgoingHandshake:457(509ns) SwitchToState:179(149ns) ProcessPendingSessionEvents:495(20612140ns) {ForwardSessionEventToSession:612(20599454ns) ForwardSessionEventToSession:612(1964ns) ForwardSessionEventToSession:612(1013ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(1023ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(516ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(673ns)}}}
2025-12-24T02:10:55.558407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241216241230464:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.558873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268933:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.558481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241216241230462:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.558897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241216133268933:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.558507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.558513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241216241230462:2114] [node 6] ICP22 created new session: [5:7587241216241230479:2114]
2025-12-24T02:10:55.558520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241216241230479:2114] [node 6] ICS09 handshake done sender: [5:7587241216241230464:2116] self: [5:7587241216241230463:0] peer: [6:7587241215112486288:0] socket: 192 qp: -1
2025-12-24T02:10:55.558528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241216241230479:2114] [node 6] ICS10 traffic start
2025-12-24T02:10:55.558537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.557713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.558773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724935:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.558980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308988:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.559013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241215688308902:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.562739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268941:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.562763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241216133268941:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.565878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724936:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.566193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241215112486263:2109] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241215112486285:2120] held: no
2025-12-24T02:10:55.566240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724936:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.566338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241215112486198:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30429869ns) {HandleHandshakeStatus:329(30429122ns) {DropIncomingHandshake:442(165ns) DropOutgoingHandshake:457(2619ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(22967479ns) {ForwardSessionEventToSession:612(22957906ns) ForwardSessionEventToSession:612(1386ns) ForwardSessionEventToSession:612(747ns) ForwardSessionEventToSession:612(395ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(692ns) ForwardSessionEventToSession:612(421ns) ForwardSessionEventToSession:612(985ns)}}}
2025-12-24T02:10:55.566805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778296:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.566818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212866778296:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.567591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241212866778287:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(23344671ns) {HandleHandshakeStatus:329(23343746ns) {DropIncomingHandshake:442(4971ns) DropOutgoingHandshake:457(126ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(23236988ns) {ForwardSessionEventToSession:612(23226684ns) ForwardSessionEventToSession:612(1465ns) ForwardSessionEventToSession:612(665ns) ForwardSessionEventToSession:612(1764ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(867ns)}}}
2025-12-24T02:10:55.568124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241212848724921:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.568245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241212848724917:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.568264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.568273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241212848724917:2109] [node 6] ICP22 created new session: [4:7587241212848724940:2109]
2025-12-24T02:10:55.568279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241212848724940:2109] [node 6] ICS09 handshake done sender: [4:7587241212848724921:2113] self: [4:7587241212848724920:0] peer: [6:7587241215112486281:0] socket: 195 qp: -1
2025-12-24T02:10:55.568286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241212848724940:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:55.568295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.570837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241215112486269:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27472) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241215112486278:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.571137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212866778297:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.571157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212866778297:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.572076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212866778296:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.576051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212866778220:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.576071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.576080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212866778220:2049] [node 3] ICP22 created new session: [1:7587241212866778316:2049]
2025-12-24T02:10:55.576086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212866778316:2049] [node 3] ICS09 handshake done sender: [1:7587241212866778296:2113] self: [1:7587241212866778284:0] peer: [3:7587241215688308994:0] socket: 228 qp: -1
2025-12-24T02:10:55.576094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212866778316:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.576103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.576502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230481:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.582903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241216133268933:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.582913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724943:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.583367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241212848724915:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.583498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241212848724851:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.583515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.583520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241212848724851:2049] [node 2] ICP22 created new session: [4:7587241212848724944:2049]
2025-12-24T02:10:55.583524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241212848724944:2049] [node 2] ICS09 handshake done sender: [4:7587241212848724915:2111] self: [4:7587241212848724934:0] peer: [2:7587241216133268927:0] socket: 193 qp: -1
2025-12-24T02:10:55.583530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241212848724944:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.583548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.582902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486289:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.582957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486290:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.583259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241215112486278:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.583475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241215112486269:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.583493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.583501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241215112486269:2111] [node 4] ICP22 created new session: [6:7587241215112486291:2111]
2025-12-24T02:10:55.583508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241215112486291:2111] [node 4] ICS09 handshake done sender: [6:7587241215112486278:2116] self: [6:7587241215112486281:0] peer: [4:7587241212848724920:0] socket: 202 qp: -1
2025-12-24T02:10:55.583517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241215112486291:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:55.583526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.583803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241215112486285:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.586322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230477:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.586378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241216241230387:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.585094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241216133268926:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.585111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.585119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241216133268926:2107] [node 4] ICP22 created new session: [2:7587241216133268960:2107]
2025-12-24T02:10:55.585125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241216133268960:2107] [node 4] ICS09 handshake done sender: [2:7587241216133268933:2110] self: [2:7587241216133268927:0] peer: [4:7587241212848724934:0] socket: 187 qp: -1
2025-12-24T02:10:55.585132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241216133268960:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:55.585139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.586531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308981:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.586542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241215688308981:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.587202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230483:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.587354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241215688308981:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.587538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241216241230477:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.587649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241216241230387:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.587668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.587675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241216241230387:2050] [node 3] ICP22 created new session: [5:7587241216241230484:2050]
2025-12-24T02:10:55.587680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241216241230484:2050] [node 3] ICS09 handshake done sender: [5:7587241216241230477:2120] self: [5:7587241216241230482:0] peer: [3:7587241215688308970:0] socket: 234 qp: -1
2025-12-24T02:10:55.587686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241216241230484:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.587696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.594207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241216241230481:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.594262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241216241230386:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.594759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241215112486263:2109] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.594783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.594794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241215112486263:2109] [node 5] ICP22 created new session: [6:7587241215112486293:2109]
2025-12-24T02:10:55.594803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241215112486293:2109] [node 5] ICS09 handshake done sender: [6:7587241215112486285:2120] self: [6:7587241215112486288:0] peer: [5:7587241216241230463:0] socket: 186 qp: -1
2025-12-24T02:10:55.594813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241215112486293:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:55.594827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.595113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241212848724850:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.595162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241212848724850:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36319889ns) {IncomingHandshake:261(36319463ns) {DropIncomingHandshake:442(66ns) PrepareNewSessionHandshake:477(439ns) {SwitchToState:179(93ns)} GenerateSessionVirtualId:988(141ns) IssueIncomingHandshakeReply:193(36314596ns) {SwitchToState:179(926ns)}}}
2025-12-24T02:10:55.598735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241215688308995:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20991
2025-12-24T02:10:55.602871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212866778297:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.603033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212866778286:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.603062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.603073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212866778286:2108] [node 4] ICP22 created new session: [1:7587241212866778319:2108]
2025-12-24T02:10:55.603082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212866778319:2108] [node 4] ICS09 handshake done sender: [1:7587241212866778297:2114] self: [1:7587241212866778289:0] peer: [4:7587241212848724937:0] socket: 232 qp: -1
2025-12-24T02:10:55.603105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212866778319:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:55.603115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.604674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241215688308968:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.604701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.604709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241215688308968:2109] [node 5] ICP22 created new session: [3:7587241215688308997:2109]
2025-12-24T02:10:55.604717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241215688308997:2109] [node 5] ICS09 handshake done sender: [3:7587241215688308981:2115] self: [3:7587241215688308970:0] peer: [5:7587241216241230482:0] socket: 208 qp: -1
2025-12-24T02:10:55.604725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241215688308997:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:55.604747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.605431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241215688308988:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.605777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241215688308902:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.605790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.605796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241215688308902:2048] [node 1] ICP22 created new session: [3:7587241215688309000:2048]
2025-12-24T02:10:55.605801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241215688309000:2048] [node 1] ICS09 handshake done sender: [3:7587241215688308988:2120] self: [3:7587241215688308994:0] peer: [1:7587241212866778284:0] socket: 183 qp: -1
2025-12-24T02:10:55.605808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241215688309000:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.605813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.603493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241216133268962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:609
2025-12-24T02:10:55.606173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241216133268946:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.606186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241216133268946:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.606826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241212848724850:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21776) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241212848724935:2117] held: no
2025-12-24T02:10:55.606917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724946:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.607130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241216133268941:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.607239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486295:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.607219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241216133268939:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.607234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.607241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241216133268939:2112] [node 6] ICP22 created new session: [2:7587241216133268964:2112]
2025-12-24T02:10:55.607247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241216133268964:2112] [node 6] ICS09 handshake done sender: [2:7587241216133268941:2114] self: [2:7587241216133268940:0] peer: [6:7587241215112486287:0] socket: 218 qp: -1
2025-12-24T02:10:55.607252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241216133268964:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:55.607259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.607416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241212848724935:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.607471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241215112486279:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.607515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241215112486200:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.607591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241212848724850:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.607604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.607610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241212848724850:2048] [node 1] ICP22 created new session: [4:7587241212848724948:2048]
2025-12-24T02:10:55.607616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241212848724948:2048] [node 1] ICS09 handshake done sender: [4:7587241212848724935:2117] self: [4:7587241212848724937:0] peer: [1:7587241212866778289:0] socket: 198 qp: -1
2025-12-24T02:10:55.607621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241212848724948:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.607627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.608241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241215112486199:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241215112486284:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.608481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241215112486284:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.608569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241215112486199:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.608586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.608592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241215112486199:2049] [node 2] ICP22 created new session: [6:7587241215112486297:2049]
2025-12-24T02:10:55.608598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241215112486297:2049] [node 2] ICS09 handshake done sender: [6:7587241215112486284:2119] self: [6:7587241215112486287:0] peer: [2:7587241216133268940:0] socket: 222 qp: -1
2025-12-24T02:10:55.608604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241215112486297:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.608612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.608330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724947:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.609503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308979:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.609518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241215688308979:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.610994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241215688308979:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.611097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241215688308969:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.611116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.611124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241215688308969:2110] [node 6] ICP22 created new session: [3:7587241215688309003:2110]
2025-12-24T02:10:55.611130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241215688309003:2110] [node 6] ICS09 handshake done sender: [3:7587241215688308979:2114] self: [3:7587241215688308971:0] peer: [6:7587241215112486296:0] socket: 213 qp: -1
2025-12-24T02:10:55.611135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241215688309003:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:55.611144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.611718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241212866778220:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:55.615157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241216133268926:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:55.615455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241215112486200:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241215112486279:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.620346Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241216241230386:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24834) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241216241230481:2121] held: no
2025-12-24T02:10:55.620408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241216241230487:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60747
2025-12-24T02:10:55.622485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241215112486299:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32080
2025-12-24T02:10:55.619258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241216133268946:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.622982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241212848724947:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.619360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241216133268937:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.619384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.623038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241212848724852:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.619395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241216133268937:2111] [node 5] ICP22 created new session: [2:7587241216133268967:2111]
2025-12-24T02:10:55.619404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241216133268967:2111] [node 5] ICS09 handshake done sender: [2:7587241216133268946:2117] self: [2:7587241216133268945:0] peer: [5:7587241216241230486:0] socket: 240 qp: -1
2025-12-24T02:10:55.619412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241216133268967:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:55.619423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.623351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241215112486279:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.623415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241216241230481:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.623606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241215112486200:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.623625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.623638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241215112486200:2050] [node 3] ICP22 created new session: [6:7587241215112486300:2050]
2025-12-24T02:10:55.623646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241215112486300:2050] [node 3] ICS09 handshake done sender: [6:7587241215112486279:2117] self: [6:7587241215112486296:0] peer: [3:7587241215688308971:0] socket: 220 qp: -1
2025-12-24T02:10:55.623653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241215112486300:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.623666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.623225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241215688308986:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.623236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241215688308986:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.623963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241215688308986:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.623895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241216241230386:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.623909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.623914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241216241230386:2049] [node 2] ICP22 created new session: [5:7587241216241230488:2049]
2025-12-24T02:10:55.623918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241216241230488:2049] [node 2] ICS09 handshake done sender: [5:7587241216241230481:2121] self: [5:7587241216241230486:0] peer: [2:7587241216133268945:0] socket: 235 qp: -1
2025-12-24T02:10:55.623924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241216241230488:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.623934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.624289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241215688308966:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.624304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.624309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241215688308966:2108] [node 4] ICP22 created new session: [3:7587241215688309006:2108]
2025-12-24T02:10:55.624314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241215688309006:2108] [node 4] ICS09 handshake done sender: [3:7587241215688308986:2118] self: [3:7587241215688308978:0] peer: [4:7587241212848724950:0] socket: 214 qp: -1
2025-12-24T02:10:55.624319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241215688309006:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:55.624324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.624670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241212848724951:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20587
2025-12-24T02:10:55.624868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241212848724947:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.625741Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241212848724852:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241212848724947:2121] held: no
2025-12-24T02:10:55.625862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241212848724852:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.625875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.625882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241212848724852:2050] [node 3] ICP22 created new session: [4:7587241212848724952:2050]
2025-12-24T02:10:55.625887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241212848724952:2050] [node 3] ICS09 handshake done sender: [4:7587241212848724947:2121] self: [4:7587241212848724950:0] peer: [3:7587241215688308978:0] socket: 230 qp: -1
2025-12-24T02:10:55.625893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241212848724952:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.625900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.639312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:55.639395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241216133268956:2048] [node 1] ICS01 socket: 179 reason# EndOfStream
2025-12-24T02:10:55.639403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241216133268865:2048] [node 1] ICP30 unregister session Session# [2:7587241216133268956:2048] VirtualId# [2:7587241216133268947:0]
2025-12-24T02:10:55.639410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241216133268956:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.639443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.639462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.654809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:55.654913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241215688309000:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-12-24T02:10:55.654926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241215688308902:2048] [node 1] ICP30 unregister session Session# [3:7587241215688309000:2048] VirtualId# [3:7587241215688308994:0]
2025-12-24T02:10:55.654933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241215688309000:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.655004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.655041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.655313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:55.655401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241212848724948:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:10:55.655407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241212848724850:2048] [node 1] ICP30 unregister session Session# [4:7587241212848724948:2048] VirtualId# [4:7587241212848724937:0]
2025-12-24T02:10:55.655412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241212848724948:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.655456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.655478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.657973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:55.658061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241216241230474:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:10:55.658068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241216241230385:2048] [node 1] ICP30 unregister session Session# [5:7587241216241230474:2048] VirtualId# [5:7587241216241230465:0]
2025-12-24T02:10:55.658073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241216241230474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.658133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.658156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.662791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:55.662896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241215112486282:2048] [node 1] ICS07 socket disconnect 178 reason# EndOfStream
2025-12-24T02:10:55.662903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241215112486282:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.662993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.663021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241215112486282:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:55.663535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241215112486198:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21776) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:55.663547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241215112486282:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:55.663552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241215112486198:2048] [node 1] ICP30 unregister session Session# [6:7587241215112486282:2048] VirtualId# [6:7587241215112486276:0]
2025-12-24T02:10:55.663568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.663574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241215112486198:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.663576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21776) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:55.666454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:55.666543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241215688308989:2049] [node 2] ICS01 socket: 182 reason# EndOfStream
2025-12-24T02:10:55.666552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241215688308903:2049] [node 2] ICP30 unregister session Session# [3:7587241215688308989:2049] VirtualId# [3:7587241215688308982:0]
2025-12-24T02:10:55.666559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241215688308989:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.666625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.666659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.668482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:55.668559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241212848724944:2049] [node 2] ICS01 socket: 193 reason# EndOfStream
2025-12-24T02:10:55.668566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241212848724851:2049] [node 2] ICP30 unregister session Session# [4:7587241212848724944:2049] VirtualId# [4:7587241212848724934:0]
2025-12-24T02:10:55.668572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241212848724944:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.668626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.668650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.668686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:55.668749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241215112486297:2049] [node 2] ICS01 socket: 222 reason# EndOfStream
2025-12-24T02:10:55.668754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241215112486199:2049] [node 2] ICP30 unregister session Session# [6:7587241215112486297:2049] VirtualId# [6:7587241215112486287:0]
2025-12-24T02:10:55.668759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241215112486297:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.668787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.668806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.670764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:55.670841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241216241230488:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-12-24T02:10:55.670847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241216241230386:2049] [node 2] ICP30 unregister session Session# [5:7587241216241230488:2049] VirtualId# [5:7587241216241230486:0]
2025-12-24T02:10:55.670852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241216241230488:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.670909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:55.670932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:55.674740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241215112486198:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:55.685576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:55.685676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241212848724952:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-12-24T02:10:55.685685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241212848724852:2050] [node 3] ICP30 unregister session Session# [4:7587241212848724952:2050] VirtualId# [4:7587241212848724950:0]
2025-12-24T02:10:55.685692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241212848724952:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.685753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.685778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.685816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:55.685884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241215112486300:2050] [node 3] ICS01 socket: 220 reason# EndOfStream
2025-12-24T02:10:55.685888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241215112486200:2050] [node 3] ICP30 unregister session Session# [6:7587241215112486300:2050] VirtualId# [6:7587241215112486296:0]
2025-12-24T02:10:55.685893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241215112486300:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.685924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.685945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.686768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:55.686840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241216241230484:2050] [node 3] ICS01 socket: 234 reason# EndOfStream
2025-12-24T02:10:55.686845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241216241230387:2050] [node 3] ICP30 unregister session Session# [5:7587241216241230484:2050] VirtualId# [5:7587241216241230482:0]
2025-12-24T02:10:55.686850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241216241230484:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.686896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:55.686916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:55.706785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:55.706885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241215112486291:2111] [node 4] ICS01 socket: 202 reason# EndOfStream
2025-12-24T02:10:55.706894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241215112486269:2111] [node 4] ICP30 unregister session Session# [6:7587241215112486291:2111] VirtualId# [6:7587241215112486281:0]
2025-12-24T02:10:55.706901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241215112486291:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.706970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:55.706994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:55.740324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212637174867:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:55.740332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212637174868:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.746937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213043716839:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.746952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213043716840:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.738401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241214326038647:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.738413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241214326038648:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:55.746112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241214883885497:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.746121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241214883885498:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.746125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241214883885499:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.754855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241212325519772:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.754864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241212325519773:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:55.774748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241213426842862:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:55.774776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241213426842864:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.781844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241213426842875:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.786438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885509:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.783103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213043716870:2074] [node 4] ICP01 ready to work
2025-12-24T02:10:55.783160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213043716871:2076] [node 6] ICP01 ready to work
2025-12-24T02:10:55.783609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716873:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
2025-12-24T02:10:55.784301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716874:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
2025-12-24T02:10:55.783797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241213426842884:2064] [node 4] ICP01 ready to work
2025-12-24T02:10:55.789216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241214883885526:2063] [node 6] ICP01 ready to work
2025-12-24T02:10:55.754869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241212325519774:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:55.783145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212637174894:2073] [node 4] ICP01 ready to work
2025-12-24T02:10:55.790491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241213426842863:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.817460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241213426842864:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29052638ns) {Configure:104(29051916ns) {ProcessConfigured:135(28827544ns) {StartInitialHandshake:157(28827126ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(161ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(807ns)}}}}}
2025-12-24T02:10:55.818211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842938:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.817547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241213043716871:2076] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34321451ns) {Configure:104(34320447ns) {ProcessConfigured:135(28785577ns) {StartInitialHandshake:157(28784479ns) {DropHandshakes:470(244ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(536ns) {SwitchToState:179(294ns)}}}}}
2025-12-24T02:10:55.817860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241213043716840:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34705028ns) {Configure:104(34704395ns) {ProcessConfigured:135(28951082ns) {StartInitialHandshake:157(28950420ns) {DropHandshakes:470(322ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(540ns) {SwitchToState:179(41ns)}}}}}
2025-12-24T02:10:55.818039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241213043716870:2074] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34836634ns) {Configure:104(34836034ns) {ProcessConfigured:135(29072767ns) {StartInitialHandshake:157(29072167ns) {DropHandshakes:470(572ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(176ns) PrepareNewSessionHandshake:477(759ns) {SwitchToState:179(514ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.825766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212637174939:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.825845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212637174938:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33118
2025-12-24T02:10:55.822949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241213426842938:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.823121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716890:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.822989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241213426842864:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.823918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842940:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.823132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213043716890:2092] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.824110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241213426842938:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.823679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213043716890:2092] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.824159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241213426842864:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.823763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213043716871:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.824169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.823774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.824174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241213426842864:2050] [node 3] ICP22 created new session: [6:7587241213426842941:2050]
2025-12-24T02:10:55.823783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213043716871:2076] [node 6] ICP22 created new session: [3:7587241213043716902:2076]
2025-12-24T02:10:55.824180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241213426842941:2050] [node 3] ICS09 handshake done sender: [6:7587241213426842938:2113] self: [6:7587241213426842939:0] peer: [3:7587241213043716876:0] socket: 202 qp: -1
2025-12-24T02:10:55.824186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241213426842941:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.824200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.823789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213043716902:2076] [node 6] ICS09 handshake done sender: [3:7587241213043716890:2092] self: [3:7587241213043716876:0] peer: [6:7587241213426842939:0] socket: 200 qp: -1
2025-12-24T02:10:55.823796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213043716902:2076] [node 6] ICS10 traffic start
2025-12-24T02:10:55.823813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.823862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716873:2079] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.826795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241214883885526:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37495017ns) {Configure:104(37493999ns) {ProcessConfigured:135(37220118ns) {StartInitialHandshake:157(37219262ns) {DropHandshakes:470(786ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(522ns) PrepareNewSessionHandshake:477(1630ns) {SwitchToState:179(829ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.828737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885509:2059] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.828762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241214883885497:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.827915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519840:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.828436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174939:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.828479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212637174940:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33118
2025-12-24T02:10:55.828876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174899:2075] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.828880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212637174899:2075] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.829532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212637174899:2075] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.831232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174938:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.830241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241213426842863:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39683393ns) {Configure:104(39682650ns) {ProcessConfigured:135(39259815ns) {StartInitialHandshake:157(39259359ns) {DropHandshakes:470(554ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(1305ns) {SwitchToState:179(812ns)}}}}}
2025-12-24T02:10:55.834766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212637174908:2079] [node 5] ICP01 ready to work
2025-12-24T02:10:55.835479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519840:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.839130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241214883885499:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49450376ns) {Configure:104(45255247ns) {ProcessConfigured:135(44952493ns) {StartInitialHandshake:157(44952019ns) {DropHandshakes:470(656ns) {DropIncomingHandshake:442(110ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(1202ns) {SwitchToState:179(648ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.855766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212637174948:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33118
2025-12-24T02:10:55.858064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174936:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.858073Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212637174936:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.858537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212637174936:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.855324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038714:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.855347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038715:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.855357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038716:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.855434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038717:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.855483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038718:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.855871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038717:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.856950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241214326038647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.858615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212637174867:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.858630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.858636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212637174867:2048] [node 2] ICP22 created new session: [1:7587241212637174950:2048]
2025-12-24T02:10:55.858642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212637174950:2048] [node 2] ICS09 handshake done sender: [1:7587241212637174936:2108] self: [1:7587241212637174935:0] peer: [2:7587241214326038722:0] socket: 205 qp: -1
2025-12-24T02:10:55.858648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212637174950:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:55.858655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.859646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212637174894:2073] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.859667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.859677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212637174894:2073] [node 4] ICP22 created new session: [1:7587241212637174952:2073]
2025-12-24T02:10:55.859685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212637174952:2073] [node 4] ICS09 handshake done sender: [1:7587241212637174899:2075] self: [1:7587241212637174898:0] peer: [4:7587241214883885561:0] socket: 185 qp: -1
2025-12-24T02:10:55.859693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212637174952:2073] [node 4] ICS10 traffic start
2025-12-24T02:10:55.859707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.862722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241213043716839:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33981802ns) {Configure:104(33980926ns) {ProcessConfigured:135(28730511ns) {StartInitialHandshake:157(28729762ns) {DropHandshakes:470(604ns) {DropIncomingHandshake:442(193ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(1504ns) {SwitchToState:179(780ns)}}}}}
2025-12-24T02:10:55.862797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241213043716839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.862970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241213043716897:2089] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.864255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174896:2074] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.864269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212637174896:2074] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.863793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.874833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241212637174908:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39957875ns) {Configure:104(39956593ns) {ProcessConfigured:135(39519140ns) {StartInitialHandshake:157(39518429ns) {DropHandshakes:470(677ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(1418ns) {SwitchToState:179(821ns)}}}}}
2025-12-24T02:10:55.874908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174948:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.875555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241213043716871:2076] [node 6] ICP35 event processing took too much time PendingConnection:279(51757152ns) {HandleHandshakeStatus:329(51756378ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(129ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(39726645ns) {ForwardSessionEventToSession:612(39719765ns) ForwardSessionEventToSession:612(1756ns) ForwardSessionEventToSession:612(920ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(563ns)}}}
2025-12-24T02:10:55.875604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241212325519772:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88671540ns) {Configure:104(88670907ns) {ProcessConfigured:135(88279439ns) {StartInitialHandshake:157(88278810ns) {DropHandshakes:470(392ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:477(1234ns) {SwitchToState:179(672ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.875272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885575:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.875355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885576:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.875406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885577:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.876011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241214883885509:2059] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.876091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241214883885497:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.876109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.876116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241214883885497:2048] [node 1] ICP22 created new session: [4:7587241214883885578:2048]
2025-12-24T02:10:55.876122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241214883885578:2048] [node 1] ICS09 handshake done sender: [4:7587241214883885509:2059] self: [4:7587241214883885561:0] peer: [1:7587241212637174898:0] socket: 188 qp: -1
2025-12-24T02:10:55.876130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241214883885578:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.876147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.875756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519846:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.879080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885575:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.879124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885576:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.879166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241214883885499:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.879338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716895:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.879351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213043716895:2094] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.883178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241212637174956:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33118
2025-12-24T02:10:55.884480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241213426842946:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.884538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241213426842884:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.886762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241212637174906:2077] [node 6] ICP01 ready to work
2025-12-24T02:10:55.890382Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241213043716897:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27333215ns) {Configure:104(27332297ns) {ProcessConfigured:135(26953867ns) {StartInitialHandshake:157(26953434ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(872ns) {SwitchToState:179(547ns)}}}}}
2025-12-24T02:10:55.886722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241212325519805:2071] [node 6] ICP01 ready to work
2025-12-24T02:10:55.889929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241214326038648:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(95932727ns) {Configure:104(95931439ns) {ProcessConfigured:135(95541980ns) {StartInitialHandshake:157(95541602ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(1366ns) {SwitchToState:179(757ns)}}}}}
2025-12-24T02:10:55.886915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241214883885528:2068] [node 5] ICP01 ready to work
2025-12-24T02:10:55.890000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038716:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.890466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038718:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.891632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038715:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.895029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174956:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.899112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241212637174867:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(40452289ns) {HandleHandshakeStatus:329(40451383ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(40365752ns) {ForwardSessionEventToSession:612(40353279ns) ForwardSessionEventToSession:612(1921ns) ForwardSessionEventToSession:612(1065ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(882ns) ForwardSessionEventToSession:612(740ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(475ns) ForwardSessionEventToSession:612(881ns)}}}
2025-12-24T02:10:55.899192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212637174896:2074] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.899542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241212637174867:2048] [node 2] ICP18 handshake (actor [1:7587241212637174948:2116]) is too old
2025-12-24T02:10:55.898978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716924:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
2025-12-24T02:10:55.899320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885556:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.899108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716874:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.899337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241214883885556:2100] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.900066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038714:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.902794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241214326038694:2078] [node 5] ICP01 ready to work
2025-12-24T02:10:55.902777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241212325519839:2085] [node 4] ICP01 ready to work
2025-12-24T02:10:55.903213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241214326038693:2071] [node 6] ICP01 ready to work
2025-12-24T02:10:55.904538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241212637174894:2073] [node 4] ICP35 event processing took too much time PendingConnection:279(74879786ns) {HandleHandshakeStatus:329(74878877ns) {DropIncomingHandshake:442(406ns) DropOutgoingHandshake:457(148ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(17489784ns) {ForwardSessionEventToSession:612(17478165ns) ForwardSessionEventToSession:612(2163ns) ForwardSessionEventToSession:612(1568ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(1459ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(595ns) ForwardSessionEventToSession:612(534ns)}}}
2025-12-24T02:10:55.904629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241212637174894:2073] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:55.904714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212637174868:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.904730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.904735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212637174868:2049] [node 3] ICP22 created new session: [1:7587241212637174963:2049]
2025-12-24T02:10:55.904740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212637174963:2049] [node 3] ICS09 handshake done sender: [1:7587241212637174896:2074] self: [1:7587241212637174895:0] peer: [3:7587241213043716922:0] socket: 182 qp: -1
2025-12-24T02:10:55.904746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212637174963:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.904752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.906336Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241212325519805:2071] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19507243ns) {Configure:104(19506824ns) {ProcessConfigured:135(19140256ns) {StartInitialHandshake:157(19124183ns) {DropHandshakes:470(1100ns) {DropIncomingHandshake:442(505ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(420ns) PrepareNewSessionHandshake:477(814ns) {SwitchToState:179(417ns)}} IncomingHandshake:261(6794ns) {DropIncomingHandshake:442(163ns) PrepareNewSessionHandshake:477(321ns) {SwitchToState:179(111ns)} GenerateSessionVirtualId:988(166ns) IssueIncomingHandshakeReply:193(376ns)}}}}
2025-12-24T02:10:55.906742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716924:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.906935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241213043716839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241213043716873:2079] held: no
2025-12-24T02:10:55.906762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241214326038647:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241214326038717:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.911166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842948:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.911233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842949:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.912599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519846:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.912684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241212325519772:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.910912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716928:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
2025-12-24T02:10:55.911005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.915606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241213426842948:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.915653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241213426842949:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.915687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241213426842862:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.915697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241213426842863:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.914873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241214326038731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16880
2025-12-24T02:10:55.915790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241214326038708:2076] [node 4] ICP01 ready to work
2025-12-24T02:10:55.918917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174954:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.918933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212637174954:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.915263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716928:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.915475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241213043716840:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.916319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213043716895:2094] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.918788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213043716870:2074] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.918810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.918819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213043716870:2074] [node 4] ICP22 created new session: [3:7587241213043716932:2074]
2025-12-24T02:10:55.918827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213043716932:2074] [node 4] ICS09 handshake done sender: [3:7587241213043716895:2094] self: [3:7587241213043716878:0] peer: [4:7587241214883885581:0] socket: 207 qp: -1
2025-12-24T02:10:55.918835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213043716932:2074] [node 4] ICS10 traffic start
2025-12-24T02:10:55.918846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.925486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241214883885556:2100] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.925687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241214883885526:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.925716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.925726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241214883885526:2063] [node 6] ICP22 created new session: [4:7587241214883885586:2063]
2025-12-24T02:10:55.925733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241214883885586:2063] [node 6] ICS09 handshake done sender: [4:7587241214883885556:2100] self: [4:7587241214883885527:0] peer: [6:7587241213426842947:0] socket: 209 qp: -1
2025-12-24T02:10:55.925741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241214883885586:2063] [node 6] ICS10 traffic start
2025-12-24T02:10:55.925752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.926894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.927048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241213426842946:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.926838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241214326038694:2078] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23947464ns) {Configure:104(23946533ns) {ProcessConfigured:135(23598687ns) {StartInitialHandshake:157(23575324ns) {DropHandshakes:470(1068ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(206ns)} GenerateSessionVirtualId:988(956ns) PrepareNewSessionHandshake:477(1741ns) {SwitchToState:179(791ns)}} IncomingHandshake:261(9939ns) {DropIncomingHandshake:442(134ns) PrepareNewSessionHandshake:477(137ns) {SwitchToState:179(104ns)} GenerateSessionVirtualId:988(342ns) IssueIncomingHandshakeReply:193(541ns)}}}}
2025-12-24T02:10:55.926908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038728:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.926913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241214326038728:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.927912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519854:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.928779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519855:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.927668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241214326038717:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.928243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241214326038647:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.928258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.928266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241214326038647:2048] [node 1] ICP22 created new session: [2:7587241214326038736:2048]
2025-12-24T02:10:55.928272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241214326038736:2048] [node 1] ICS09 handshake done sender: [2:7587241214326038717:2113] self: [2:7587241214326038722:0] peer: [1:7587241212637174935:0] socket: 217 qp: -1
2025-12-24T02:10:55.928281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241214326038736:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.928303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.928880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241214326038728:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.929013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241214326038693:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.929025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.929032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241214326038693:2071] [node 6] ICP22 created new session: [2:7587241214326038739:2071]
2025-12-24T02:10:55.929037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241214326038739:2071] [node 6] ICS09 handshake done sender: [2:7587241214326038728:2117] self: [2:7587241214326038727:0] peer: [6:7587241213426842951:0] socket: 211 qp: -1
2025-12-24T02:10:55.929043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241214326038739:2071] [node 6] ICS10 traffic start
2025-12-24T02:10:55.929048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.934276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241213426842863:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241213426842949:2119] held: no
2025-12-24T02:10:55.934358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842953:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.934590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241213426842949:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.931138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213043716873:2079] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.934667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241213426842863:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.934681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.934687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241213426842863:2049] [node 2] ICP22 created new session: [6:7587241213426842954:2049]
2025-12-24T02:10:55.934709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241213426842954:2049] [node 2] ICS09 handshake done sender: [6:7587241213426842949:2119] self: [6:7587241213426842951:0] peer: [2:7587241214326038727:0] socket: 219 qp: -1
2025-12-24T02:10:55.934717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241213426842954:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.934725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.934908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885589:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.935190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241214883885576:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.934854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241213043716839:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.934876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.934884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213043716839:2048] [node 1] ICP22 created new session: [3:7587241213043716935:2048]
2025-12-24T02:10:55.934890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213043716935:2048] [node 1] ICS09 handshake done sender: [3:7587241213043716873:2079] self: [3:7587241213043716922:0] peer: [1:7587241212637174895:0] socket: 184 qp: -1
2025-12-24T02:10:55.934898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213043716935:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.934908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.935322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241214883885499:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.935341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.935348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241214883885499:2050] [node 3] ICP22 created new session: [4:7587241214883885590:2050]
2025-12-24T02:10:55.935354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241214883885590:2050] [node 3] ICS09 handshake done sender: [4:7587241214883885576:2116] self: [4:7587241214883885581:0] peer: [3:7587241213043716878:0] socket: 225 qp: -1
2025-12-24T02:10:55.935361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241214883885590:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.935369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.938890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519855:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.938963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241212325519773:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.938962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038723:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.938979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241214326038723:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.938879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241212637174960:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.938903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241212637174960:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.944386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212637174954:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.944490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212637174908:2079] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.944509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.944518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212637174908:2079] [node 5] ICP22 created new session: [1:7587241212637174966:2079]
2025-12-24T02:10:55.944525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212637174966:2079] [node 5] ICS09 handshake done sender: [1:7587241212637174954:2117] self: [1:7587241212637174943:0] peer: [5:7587241212325519853:0] socket: 224 qp: -1
2025-12-24T02:10:55.944532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212637174966:2079] [node 5] ICS10 traffic start
2025-12-24T02:10:55.944543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.946776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241213043716870:2074] [node 4] ICP35 event processing took too much time PendingConnection:279(27950772ns) {HandleHandshakeStatus:329(27949761ns) {DropIncomingHandshake:442(5785ns) DropOutgoingHandshake:457(167ns) SwitchToState:179(184ns) ProcessPendingSessionEvents:495(27831964ns) {ForwardSessionEventToSession:612(27817366ns) ForwardSessionEventToSession:612(2877ns) ForwardSessionEventToSession:612(836ns) ForwardSessionEventToSession:612(2385ns) ForwardSessionEventToSession:612(593ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(710ns) ForwardSessionEventToSession:612(763ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(1722ns)}}}
2025-12-24T02:10:55.947085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241213426842884:2064] [node 4] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-12-24T02:10:55.947131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
2025-12-24T02:10:55.947142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241213426842884:2064] [node 4] ICP22 created new session: [6:7587241213426842956:2064]
2025-12-24T02:10:55.947150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241213426842956:2064] [node 4] ICS09 handshake done sender: [6:7587241213426842946:2117] self: [6:7587241213426842947:0] peer: [4:7587241214883885527:0] socket: 214 qp: -1
2025-12-24T02:10:55.947158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241213426842956:2064] [node 4] ICS10 traffic start
2025-12-24T02:10:55.947187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:55.947284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241213426842884:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(20094717ns) {HandleHandshakeStatus:329(20093958ns) {DropIncomingHandshake:442(1034ns) DropOutgoingHandshake:457(4445ns) SwitchToState:179(310ns) ProcessPendingSessionEvents:495(28119ns) {ForwardSessionEventToSession:612(17973ns) ForwardSessionEventToSession:612(692ns) ForwardSessionEventToSession:612(1056ns) ForwardSessionEventToSession:612(1338ns) ForwardSessionEventToSession:612(688ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(685ns) ForwardSessionEventToSession:612(513ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(1125ns)}}}
2025-12-24T02:10:55.947303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241213426842884:2064] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.947998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885584:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.948010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241214883885584:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.948896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241214883885584:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.947157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519854:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.947221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241212325519839:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.947601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.947653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.948980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519858:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.949064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519860:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.949110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241212325519774:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.949009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241214883885528:2068] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.949029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.949037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241214883885528:2068] [node 5] ICP22 created new session: [4:7587241214883885593:2068]
2025-12-24T02:10:55.949045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241214883885593:2068] [node 5] ICS09 handshake done sender: [4:7587241214883885584:2118] self: [4:7587241214883885583:0] peer: [5:7587241212325519857:0] socket: 238 qp: -1
2025-12-24T02:10:55.949053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241214883885593:2068] [node 5] ICS10 traffic start
2025-12-24T02:10:55.949063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.949341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241213043716925:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.949358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241213043716925:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.950195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213043716925:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.950310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241213043716897:2089] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.950330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.950336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213043716897:2089] [node 5] ICP22 created new session: [3:7587241213043716938:2089]
2025-12-24T02:10:55.950342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213043716938:2089] [node 5] ICS09 handshake done sender: [3:7587241213043716925:2117] self: [3:7587241213043716923:0] peer: [5:7587241212325519861:0] socket: 213 qp: -1
2025-12-24T02:10:55.950347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213043716938:2089] [node 5] ICS10 traffic start
2025-12-24T02:10:55.950354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.946981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038733:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.949392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241212325519846:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.947000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241214326038733:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.949513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241212325519854:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.949559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241212325519839:2085] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.949569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.949576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241212325519839:2085] [node 4] ICP22 created new session: [5:7587241212325519862:2085]
2025-12-24T02:10:55.949583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241212325519862:2085] [node 4] ICS09 handshake done sender: [5:7587241212325519854:2117] self: [5:7587241212325519857:0] peer: [4:7587241214883885583:0] socket: 193 qp: -1
2025-12-24T02:10:55.949590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241212325519862:2085] [node 4] ICS10 traffic start
2025-12-24T02:10:55.949608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.953129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842958:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.953938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241213426842862:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241213426842948:2118] held: no
2025-12-24T02:10:55.954149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241213426842948:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:55.954236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241213426842862:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.954248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.954254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241213426842862:2048] [node 1] ICP22 created new session: [6:7587241213426842959:2048]
2025-12-24T02:10:55.954260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241213426842959:2048] [node 1] ICS09 handshake done sender: [6:7587241213426842948:2118] self: [6:7587241213426842950:0] peer: [1:7587241212637174959:0] socket: 235 qp: -1
2025-12-24T02:10:55.954266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241213426842959:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.954273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.952093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241212637174960:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.952720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241212637174906:2077] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.952735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.952742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241212637174906:2077] [node 6] ICP22 created new session: [1:7587241212637174969:2077]
2025-12-24T02:10:55.952749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241212637174969:2077] [node 6] ICS09 handshake done sender: [1:7587241212637174960:2119] self: [1:7587241212637174959:0] peer: [6:7587241213426842950:0] socket: 234 qp: -1
2025-12-24T02:10:55.952755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241212637174969:2077] [node 6] ICS10 traffic start
2025-12-24T02:10:55.952762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.952172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241214326038723:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.952254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241214326038648:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.952269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.952278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241214326038648:2049] [node 3] ICP22 created new session: [2:7587241214326038745:2049]
2025-12-24T02:10:55.952285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241214326038745:2049] [node 3] ICS09 handshake done sender: [2:7587241214326038723:2116] self: [2:7587241214326038685:0] peer: [3:7587241213043716931:0] socket: 210 qp: -1
2025-12-24T02:10:55.952291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241214326038745:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:55.952298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:55.955006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241213043716840:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241213043716928:2118] held: no
2025-12-24T02:10:55.963585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241212325519772:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.963612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:55.963621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241212325519772:2048] [node 1] ICP22 created new session: [5:7587241212325519864:2048]
2025-12-24T02:10:55.963629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241212325519864:2048] [node 1] ICS09 handshake done sender: [5:7587241212325519846:2114] self: [5:7587241212325519853:0] peer: [1:7587241212637174943:0] socket: 227 qp: -1
2025-12-24T02:10:55.963636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241212325519864:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:55.963658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:55.963761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241212325519772:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.964979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241212325519774:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241212325519858:2119] held: no
2025-12-24T02:10:55.965067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519866:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.965791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241212325519867:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55072
2025-12-24T02:10:55.965951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241212325519858:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:55.966000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241212325519774:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.966009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:55.966015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241212325519774:2050] [node 3] ICP22 created new session: [5:7587241212325519869:2050]
2025-12-24T02:10:55.966020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241212325519869:2050] [node 3] ICS09 handshake done sender: [5:7587241212325519858:2119] self: [5:7587241212325519861:0] peer: [3:7587241213043716923:0] socket: 230 qp: -1
2025-12-24T02:10:55.966025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241212325519869:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:55.966030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.967006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885595:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.967411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241213043716940:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30473
2025-12-24T02:10:55.967488Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241212325519773:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241212325519855:2118] held: no
2025-12-24T02:10:55.968869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241212325519855:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.970283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241214883885595:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.970408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241214883885498:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.964838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241214326038733:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.965148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241214326038694:2078] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.965164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.965170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241214326038694:2078] [node 5] ICP22 created new session: [2:7587241214326038748:2078]
2025-12-24T02:10:55.965176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241214326038748:2078] [node 5] ICS09 handshake done sender: [2:7587241214326038733:2119] self: [2:7587241214326038726:0] peer: [5:7587241212325519856:0] socket: 241 qp: -1
2025-12-24T02:10:55.965183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241214326038748:2078] [node 5] ICS10 traffic start
2025-12-24T02:10:55.965201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.969906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241212325519839:2085] [node 4] ICP35 event processing took too much time PendingConnection:279(20306897ns) {HandleHandshakeStatus:329(20306478ns) {DropIncomingHandshake:442(156ns) DropOutgoingHandshake:457(1151ns) SwitchToState:179(169ns) ProcessPendingSessionEvents:495(14831618ns) {ForwardSessionEventToSession:612(14819629ns) ForwardSessionEventToSession:612(1851ns) ForwardSessionEventToSession:612(894ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(910ns) ForwardSessionEventToSession:612(672ns) ForwardSessionEventToSession:612(846ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(917ns)}}}
2025-12-24T02:10:55.970065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241212325519773:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.970075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.970083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241212325519773:2049] [node 2] ICP22 created new session: [5:7587241212325519871:2049]
2025-12-24T02:10:55.970089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241212325519871:2049] [node 2] ICS09 handshake done sender: [5:7587241212325519855:2118] self: [5:7587241212325519856:0] peer: [2:7587241214326038726:0] socket: 242 qp: -1
2025-12-24T02:10:55.970096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241212325519871:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.970102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.973146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842961:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.975158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241214326038741:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.975169Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241214326038741:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.975005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241213043716928:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.976078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241214326038741:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:55.976167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241214326038708:2076] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.976187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:55.976194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241214326038708:2076] [node 4] ICP22 created new session: [2:7587241214326038751:2076]
2025-12-24T02:10:55.976202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241214326038751:2076] [node 4] ICS09 handshake done sender: [2:7587241214326038741:2120] self: [2:7587241214326038732:0] peer: [4:7587241214883885596:0] socket: 218 qp: -1
2025-12-24T02:10:55.976208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241214326038751:2076] [node 4] ICS10 traffic start
2025-12-24T02:10:55.976215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.978542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241214883885498:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61505) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241214883885595:2120] held: no
2025-12-24T02:10:55.978590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241214883885597:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22329
2025-12-24T02:10:55.978815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241214883885595:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:55.978790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241213426842961:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.978845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241213426842875:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:55.979069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241212325519851:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:55.979081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241212325519851:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:55.978903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241214883885498:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.978915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.978922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241214883885498:2049] [node 2] ICP22 created new session: [4:7587241214883885598:2049]
2025-12-24T02:10:55.978930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241214883885598:2049] [node 2] ICS09 handshake done sender: [4:7587241214883885595:2120] self: [4:7587241214883885596:0] peer: [2:7587241214326038732:0] socket: 192 qp: -1
2025-12-24T02:10:55.978937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241214883885598:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.978955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:55.979922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241213426842963:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34659
2025-12-24T02:10:55.980242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241213426842961:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:55.980340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241213426842875:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.980358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:55.980363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241213426842875:2059] [node 5] ICP22 created new session: [6:7587241213426842964:2059]
2025-12-24T02:10:55.980369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241213426842964:2059] [node 5] ICS09 handshake done sender: [6:7587241213426842961:2123] self: [6:7587241213426842962:0] peer: [5:7587241212325519848:0] socket: 195 qp: -1
2025-12-24T02:10:55.980373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241213426842964:2059] [node 5] ICS10 traffic start
2025-12-24T02:10:55.980381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:55.980085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241212325519851:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:55.981253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241212325519805:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:55.981266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:55.981270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241212325519805:2071] [node 6] ICP22 created new session: [5:7587241212325519874:2071]
2025-12-24T02:10:55.981275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241212325519874:2071] [node 6] ICS09 handshake done sender: [5:7587241212325519851:2116] self: [5:7587241212325519848:0] peer: [6:7587241213426842962:0] socket: 186 qp: -1
2025-12-24T02:10:55.981279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241212325519874:2071] [node 6] ICS10 traffic start
2025-12-24T02:10:55.981285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:55.982780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241213043716840:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:55.982808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:55.982815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241213043716840:2049] [node 2] ICP22 created new session: [3:7587241213043716941:2049]
2025-12-24T02:10:55.982820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241213043716941:2049] [node 2] ICS09 handshake done sender: [3:7587241213043716928:2118] self: [3:7587241213043716931:0] peer: [2:7587241214326038685:0] socket: 232 qp: -1
2025-12-24T02:10:55.982827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241213043716941:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:55.982854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:55.991260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:55.991343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241213043716935:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-12-24T02:10:55.991351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241213043716839:2048] [node 1] ICP30 unregister session Session# [3:7587241213043716935:2048] VirtualId# [3:7587241213043716922:0]
2025-12-24T02:10:55.991357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241213043716935:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.991394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.991414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.994453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:55.994546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241214326038736:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-12-24T02:10:55.994556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241214326038647:2048] [node 1] ICP30 unregister session Session# [2:7587241214326038736:2048] VirtualId# [2:7587241214326038722:0]
2025-12-24T02:10:55.994563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241214326038736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:55.994622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:55.994648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:55.999948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:56.000031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241213426842959:2048] [node 1] ICS01 socket: 235 reason# EndOfStream
2025-12-24T02:10:56.000038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241213426842862:2048] [node 1] ICP30 unregister session Session# [6:7587241213426842959:2048] VirtualId# [6:7587241213426842950:0]
2025-12-24T02:10:56.000044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241213426842959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.000088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.000109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.000141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:56.000201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241212325519864:2048] [node 1] ICS01 socket: 227 reason# EndOfStream
2025-12-24T02:10:56.000205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241212325519772:2048] [node 1] ICP30 unregister session Session# [5:7587241212325519864:2048] VirtualId# [5:7587241212325519853:0]
2025-12-24T02:10:56.000209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241212325519864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.000235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.000253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.002791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:56.002892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241214883885578:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-12-24T02:10:56.002901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241214883885497:2048] [node 1] ICP30 unregister session Session# [4:7587241214883885578:2048] VirtualId# [4:7587241214883885561:0]
2025-12-24T02:10:56.002908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241214883885578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.002980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.003010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.011286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:56.011383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241213043716941:2049] [node 2] ICS01 socket: 232 reason# EndOfStream
2025-12-24T02:10:56.011392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241213043716840:2049] [node 2] ICP30 unregister session Session# [3:7587241213043716941:2049] VirtualId# [3:7587241213043716931:0]
2025-12-24T02:10:56.011398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241213043716941:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.011443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.011465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.022579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:56.022675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241213426842954:2049] [node 2] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:10:56.022685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241213426842863:2049] [node 2] ICP30 unregister session Session# [6:7587241213426842954:2049] VirtualId# [6:7587241213426842951:0]
2025-12-24T02:10:56.022704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241213426842954:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.022749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.022779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.022748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:56.022851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241214883885598:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-12-24T02:10:56.022860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241214883885498:2049] [node 2] ICP30 unregister session Session# [4:7587241214883885598:2049] VirtualId# [4:7587241214883885596:0]
2025-12-24T02:10:56.022866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241214883885598:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.022929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.022961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.022812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:56.022871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241212325519871:2049] [node 2] ICS01 socket: 242 reason# EndOfStream
2025-12-24T02:10:56.022884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241212325519773:2049] [node 2] ICP30 unregister session Session# [5:7587241212325519871:2049] VirtualId# [5:7587241212325519856:0]
2025-12-24T02:10:56.022889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241212325519871:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.022910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.022927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.049093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:56.049191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241213426842941:2050] [node 3] ICS01 socket: 202 reason# EndOfStream
2025-12-24T02:10:56.049200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241213426842864:2050] [node 3] ICP30 unregister session Session# [6:7587241213426842941:2050] VirtualId# [6:7587241213426842939:0]
2025-12-24T02:10:56.049207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241213426842941:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.049263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.049288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.049338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:56.049409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241212325519869:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-12-24T02:10:56.049414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241212325519774:2050] [node 3] ICP30 unregister session Session# [5:7587241212325519869:2050] VirtualId# [5:7587241212325519861:0]
2025-12-24T02:10:56.049419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241212325519869:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.049447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.049470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.064025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:56.064117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241213426842956:2064] [node 4] ICS01 socket: 214 reason# EndOfStream
2025-12-24T02:10:56.064126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241213426842884:2064] [node 4] ICP30 unregister session Session# [6:7587241213426842956:2064] VirtualId# [6:7587241213426842947:0]
2025-12-24T02:10:56.064132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241213426842956:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.064187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:56.064209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:56.099961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219290925735:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.099969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219290925736:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.110530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241217900202760:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.110547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241217900202761:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.110552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241217900202762:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.110860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241216997716084:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:56.110875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241216997716085:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.110787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241218036692655:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.110800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241218036692656:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.139181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220714821078:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.139201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220714821079:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.139207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220714821080:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.139066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241217087518355:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.139124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241217087518356:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.139132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241217087518357:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.145815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241216997716145:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:56.147646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241218036692719:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:56.148317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241217087518397:2088] [node 4] ICP01 ready to work
2025-12-24T02:10:56.148883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241217087518402:2090] [node 6] ICP01 ready to work
2025-12-24T02:10:56.151446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241218036692725:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:56.159028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241216997716150:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.164355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219290925798:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:56.164409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925795:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.164469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925797:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
2025-12-24T02:10:56.165372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202821:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.165393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202822:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.166688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925795:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.166754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241219290925736:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.171062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220714821148:2073] [node 4] ICP01 ready to work
2025-12-24T02:10:56.171158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220714821149:2082] [node 5] ICP01 ready to work
2025-12-24T02:10:56.171922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821146:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.172071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202823:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.171991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821147:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.177914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241217087518357:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23071614ns) {Configure:104(23070671ns) {ProcessConfigured:135(22737532ns) {StartInitialHandshake:157(22736999ns) {DropHandshakes:470(1078ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(142ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(1041ns) {SwitchToState:179(554ns)}}}}}
2025-12-24T02:10:56.187299Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241216997716150:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28194956ns) {Configure:104(28193780ns) {ProcessConfigured:135(27858572ns) {StartInitialHandshake:157(27857930ns) {DropHandshakes:470(1210ns) {DropIncomingHandshake:442(186ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(1367ns) {SwitchToState:179(669ns)}}}}}
2025-12-24T02:10:56.192403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202823:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.192754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241217900202762:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.192874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925804:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.192886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241219290925804:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.193635Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219290925804:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.193730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241219290925798:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.193747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.193754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219290925798:2108] [node 4] ICP22 created new session: [3:7587241219290925810:2108]
2025-12-24T02:10:56.193761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219290925810:2108] [node 4] ICS09 handshake done sender: [3:7587241219290925804:2112] self: [3:7587241219290925803:0] peer: [4:7587241217900202830:0] socket: 202 qp: -1
2025-12-24T02:10:56.193814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219290925810:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:56.193825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.194208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925797:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.194364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219290925813:2113] [node 6] ICP01 ready to work
2025-12-24T02:10:56.194435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716153:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.194490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716154:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.194528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716155:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.194565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716156:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.194740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241219290925813:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.196342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925815:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.197960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925816:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.197930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241218036692734:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:10:56.197992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692716:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.197996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241218036692716:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.198664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241218036692731:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56630
2025-12-24T02:10:56.195050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716154:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.198819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716153:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.198802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220714821146:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.198776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241218036692732:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56630
2025-12-24T02:10:56.198839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241218036692733:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56630
2025-12-24T02:10:56.199318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925815:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.199345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241219290925735:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.198914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220714821100:2068] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.198923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241220714821100:2068] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.199517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220714821078:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.199716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925818:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
2025-12-24T02:10:56.199168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241216997716084:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.202933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202822:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.203010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241217900202761:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.203082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202831:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.203369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220714821100:2068] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.205614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219290925797:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.205700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241219290925813:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.205716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.205722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219290925813:2113] [node 6] ICP22 created new session: [3:7587241219290925820:2113]
2025-12-24T02:10:56.205729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219290925820:2113] [node 6] ICS09 handshake done sender: [3:7587241219290925797:2109] self: [3:7587241219290925814:0] peer: [6:7587241220714821099:0] socket: 194 qp: -1
2025-12-24T02:10:56.205736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219290925820:2113] [node 6] ICS10 traffic start
2025-12-24T02:10:56.205746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.205834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241220714821080:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.205852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.205859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220714821080:2050] [node 3] ICP22 created new session: [6:7587241220714821160:2050]
2025-12-24T02:10:56.205865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220714821160:2050] [node 3] ICS09 handshake done sender: [6:7587241220714821100:2068] self: [6:7587241220714821099:0] peer: [3:7587241219290925814:0] socket: 189 qp: -1
2025-12-24T02:10:56.205872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220714821160:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.205880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.205346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518431:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.206524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692731:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.206568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692724:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.206576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241218036692724:2111] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.206181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220714821147:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.206312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220714821149:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.206381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.203171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716156:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.203218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716158:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.203222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241216997716158:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.203795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716147:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.203799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241216997716147:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.204190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716155:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.204502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241216997716167:2116] [node 4] ICP01 ready to work
2025-12-24T02:10:56.204836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241216997716167:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.206890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
2025-12-24T02:10:56.206908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219290925822:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17962
2025-12-24T02:10:56.207462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692732:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.207499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692733:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.207539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692722:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.207542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241218036692722:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.208354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241217087518404:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.208365Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241217087518404:2092] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.209118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219290925815:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.209195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241219290925735:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.209210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.209217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219290925735:2048] [node 1] ICP22 created new session: [3:7587241219290925823:2048]
2025-12-24T02:10:56.209222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219290925823:2048] [node 1] ICS09 handshake done sender: [3:7587241219290925815:2114] self: [3:7587241219290925817:0] peer: [1:7587241216997716157:0] socket: 221 qp: -1
2025-12-24T02:10:56.209228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219290925823:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.209236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.207806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241216997716158:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.209967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241216997716147:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.210123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241216997716085:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.210135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.210140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241216997716085:2049] [node 3] ICP22 created new session: [1:7587241216997716169:2049]
2025-12-24T02:10:56.210144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241216997716169:2049] [node 3] ICS09 handshake done sender: [1:7587241216997716158:2115] self: [1:7587241216997716157:0] peer: [3:7587241219290925817:0] socket: 220 qp: -1
2025-12-24T02:10:56.210150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241216997716169:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.210156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.210320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241216997716145:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.210330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.210336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241216997716145:2107] [node 6] ICP22 created new session: [1:7587241216997716172:2107]
2025-12-24T02:10:56.210341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241216997716172:2107] [node 6] ICS09 handshake done sender: [1:7587241216997716147:2108] self: [1:7587241216997716146:0] peer: [6:7587241220714821159:0] socket: 186 qp: -1
2025-12-24T02:10:56.210346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241216997716172:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:56.210352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.210823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241218036692716:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.210889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202825:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.210900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241217900202825:2111] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.210989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241218036692724:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.211137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241217087518404:2092] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.211284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202821:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.211380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241217900202823:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.211340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241217087518431:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.211353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241217087518402:2090] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.211360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.211366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241217087518402:2090] [node 6] ICP22 created new session: [5:7587241217087518433:2090]
2025-12-24T02:10:56.211371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241217087518433:2090] [node 6] ICS09 handshake done sender: [5:7587241217087518404:2092] self: [5:7587241217087518403:0] peer: [6:7587241220714821164:0] socket: 191 qp: -1
2025-12-24T02:10:56.211377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241217087518433:2090] [node 6] ICS10 traffic start
2025-12-24T02:10:56.211383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.211768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241217900202762:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.211781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.211787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241217900202762:2050] [node 3] ICP22 created new session: [4:7587241217900202837:2050]
2025-12-24T02:10:56.211792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241217900202837:2050] [node 3] ICS09 handshake done sender: [4:7587241217900202823:2110] self: [4:7587241217900202830:0] peer: [3:7587241219290925803:0] socket: 203 qp: -1
2025-12-24T02:10:56.211798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241217900202837:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.211803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.213115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716171:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.213251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241216997716174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22291
2025-12-24T02:10:56.213262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241216997716153:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.213302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241216997716084:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.213309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.213314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241216997716084:2048] [node 2] ICP22 created new session: [1:7587241216997716175:2048]
2025-12-24T02:10:56.213319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241216997716175:2048] [node 2] ICS09 handshake done sender: [1:7587241216997716153:2111] self: [1:7587241216997716162:0] peer: [2:7587241218036692723:0] socket: 213 qp: -1
2025-12-24T02:10:56.213325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241216997716175:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:56.213330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.214861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241218036692722:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.215521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219290925826:2119] [node 5] ICP01 ready to work
2025-12-24T02:10:56.215603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925816:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.215704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219290925795:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.215583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241217900202839:2116] [node 5] ICP01 ready to work
2025-12-24T02:10:56.216004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202833:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.216053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241217900202834:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46655
2025-12-24T02:10:56.216283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241217900202825:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.215861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241218036692656:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.215879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.215885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241218036692656:2049] [node 3] ICP22 created new session: [2:7587241218036692743:2049]
2025-12-24T02:10:56.215891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241218036692743:2049] [node 3] ICS09 handshake done sender: [2:7587241218036692716:2107] self: [2:7587241218036692715:0] peer: [3:7587241219290925808:0] socket: 188 qp: -1
2025-12-24T02:10:56.215896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241218036692743:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.215903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.215984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241218036692655:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.215989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.215992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241218036692655:2048] [node 1] ICP22 created new session: [2:7587241218036692745:2048]
2025-12-24T02:10:56.215997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241218036692745:2048] [node 1] ICS09 handshake done sender: [2:7587241218036692724:2111] self: [2:7587241218036692723:0] peer: [1:7587241216997716162:0] socket: 198 qp: -1
2025-12-24T02:10:56.216001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241218036692745:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.216006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.216092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241218036692719:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.216099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.216103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241218036692719:2108] [node 4] ICP22 created new session: [2:7587241218036692747:2108]
2025-12-24T02:10:56.216107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241218036692747:2108] [node 4] ICS09 handshake done sender: [2:7587241218036692722:2110] self: [2:7587241218036692721:0] peer: [4:7587241217900202832:0] socket: 197 qp: -1
2025-12-24T02:10:56.216111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241218036692747:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:56.216116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.219309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241216997716155:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.220634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241220714821078:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241220714821146:2112] held: no
2025-12-24T02:10:56.220816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241216997716167:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.220825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.220831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241216997716167:2116] [node 4] ICP22 created new session: [1:7587241216997716177:2116]
2025-12-24T02:10:56.220768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220714821146:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.220836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241216997716177:2116] [node 4] ICS09 handshake done sender: [1:7587241216997716155:2113] self: [1:7587241216997716168:0] peer: [4:7587241217900202824:0] socket: 217 qp: -1
2025-12-24T02:10:56.221014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220714821078:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.220843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241216997716177:2116] [node 4] ICS10 traffic start
2025-12-24T02:10:56.220849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.219746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241219290925736:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.219765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.219772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219290925736:2049] [node 2] ICP22 created new session: [3:7587241219290925827:2049]
2025-12-24T02:10:56.219777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219290925827:2049] [node 2] ICS09 handshake done sender: [3:7587241219290925795:2107] self: [3:7587241219290925808:0] peer: [2:7587241218036692715:0] socket: 193 qp: -1
2025-12-24T02:10:56.219784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219290925827:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.219791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.221644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241219290925736:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:56.222845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241217900202760:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.222862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.222870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241217900202760:2048] [node 1] ICP22 created new session: [4:7587241217900202844:2048]
2025-12-24T02:10:56.222876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241217900202844:2048] [node 1] ICS09 handshake done sender: [4:7587241217900202825:2111] self: [4:7587241217900202824:0] peer: [1:7587241216997716168:0] socket: 207 qp: -1
2025-12-24T02:10:56.222882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241217900202844:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.222888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.221022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.221027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220714821078:2048] [node 1] ICP22 created new session: [6:7587241220714821166:2048]
2025-12-24T02:10:56.221032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220714821166:2048] [node 1] ICS09 handshake done sender: [6:7587241220714821146:2112] self: [6:7587241220714821159:0] peer: [1:7587241216997716146:0] socket: 195 qp: -1
2025-12-24T02:10:56.221037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220714821166:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.221040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.221165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821165:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.221454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220714821147:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.221841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220714821149:2082] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.221849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.221854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220714821149:2082] [node 5] ICP22 created new session: [6:7587241220714821169:2082]
2025-12-24T02:10:56.221859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220714821169:2082] [node 5] ICS09 handshake done sender: [6:7587241220714821147:2113] self: [6:7587241220714821164:0] peer: [5:7587241217087518403:0] socket: 205 qp: -1
2025-12-24T02:10:56.221865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220714821169:2082] [node 5] ICS10 traffic start
2025-12-24T02:10:56.221871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.227465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241217900202846:2118] [node 6] ICP01 ready to work
2025-12-24T02:10:56.227515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202833:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.229318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241217900202822:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.229440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241217900202761:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241217900202822:2109] held: no
2025-12-24T02:10:56.229455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241217900202761:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.229463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.229467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241217900202761:2049] [node 2] ICP22 created new session: [4:7587241217900202847:2049]
2025-12-24T02:10:56.229472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241217900202847:2049] [node 2] ICS09 handshake done sender: [4:7587241217900202822:2109] self: [4:7587241217900202832:0] peer: [2:7587241218036692721:0] socket: 200 qp: -1
2025-12-24T02:10:56.229478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241217900202847:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.229485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.228073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241217087518355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.229190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518439:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.227410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821171:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.229454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241216997716152:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.229471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241216997716152:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.230002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241216997716152:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.230091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241216997716150:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.230105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.230113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241216997716150:2109] [node 5] ICP22 created new session: [1:7587241216997716180:2109]
2025-12-24T02:10:56.230119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241216997716180:2109] [node 5] ICS09 handshake done sender: [1:7587241216997716152:2110] self: [1:7587241216997716151:0] peer: [5:7587241217087518438:0] socket: 219 qp: -1
2025-12-24T02:10:56.230126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241216997716180:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:56.230134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.230160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518440:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.230204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.231347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518442:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.233694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241218036692751:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56630
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.234880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821172:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.239493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202850:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.239508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241217900202850:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.240041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241217900202841:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.240050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241217900202841:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.239380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220714821172:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.239413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220714821148:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.239671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220714821171:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.239686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220714821079:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.240948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821175:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.239008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518443:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.239072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241217087518440:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.239081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241217087518431:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.239147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241217087518355:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.239161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.239166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241217087518355:2048] [node 1] ICP22 created new session: [5:7587241217087518444:2048]
2025-12-24T02:10:56.239172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241217087518444:2048] [node 1] ICS09 handshake done sender: [5:7587241217087518431:2114] self: [5:7587241217087518438:0] peer: [1:7587241216997716151:0] socket: 231 qp: -1
2025-12-24T02:10:56.239177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241217087518444:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.239184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.242849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241217900202850:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.239387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241217087518357:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.239503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241217087518443:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.239517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241217087518397:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.242807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219290925830:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.242821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241219290925830:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.242838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692751:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.242878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692727:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.242881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241218036692727:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.250031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241217900202841:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.250790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220714821172:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.250851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220714821148:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.250864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.250871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220714821148:2073] [node 4] ICP22 created new session: [6:7587241220714821176:2073]
2025-12-24T02:10:56.250876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220714821176:2073] [node 4] ICS09 handshake done sender: [6:7587241220714821172:2120] self: [6:7587241220714821173:0] peer: [4:7587241217900202849:0] socket: 251 qp: -1
2025-12-24T02:10:56.250883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220714821176:2073] [node 4] ICS10 traffic start
2025-12-24T02:10:56.250890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.254762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241217900202846:2118] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.254787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.254794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241217900202846:2118] [node 6] ICP22 created new session: [4:7587241217900202856:2118]
2025-12-24T02:10:56.254801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241217900202856:2118] [node 6] ICS09 handshake done sender: [4:7587241217900202850:2119] self: [4:7587241217900202849:0] peer: [6:7587241220714821173:0] socket: 244 qp: -1
2025-12-24T02:10:56.254809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241217900202856:2118] [node 6] ICS10 traffic start
2025-12-24T02:10:56.254816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.258504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241217900202839:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.258526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.258532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241217900202839:2116] [node 5] ICP22 created new session: [4:7587241217900202858:2116]
2025-12-24T02:10:56.258538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241217900202858:2116] [node 5] ICS09 handshake done sender: [4:7587241217900202841:2117] self: [4:7587241217900202840:0] peer: [5:7587241217087518447:0] socket: 247 qp: -1
2025-12-24T02:10:56.258544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241217900202858:2116] [node 5] ICS10 traffic start
2025-12-24T02:10:56.258553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.315446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518448:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.315611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241217087518441:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.315706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241217087518397:2088] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16310) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241217087518443:2120] held: no
2025-12-24T02:10:56.315736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241217087518443:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.315826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518449:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.315844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241217087518397:2088] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.315852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.315858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241217087518397:2088] [node 4] ICP22 created new session: [5:7587241217087518450:2088]
2025-12-24T02:10:56.315863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241217087518450:2088] [node 4] ICS09 handshake done sender: [5:7587241217087518443:2120] self: [5:7587241217087518447:0] peer: [4:7587241217900202840:0] socket: 249 qp: -1
2025-12-24T02:10:56.315869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241217087518450:2088] [node 4] ICS10 traffic start
2025-12-24T02:10:56.315876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.315973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241217087518356:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.318853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219290925830:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.319177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241219290925826:2119] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.319198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.319206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219290925826:2119] [node 5] ICP22 created new session: [3:7587241219290925834:2119]
2025-12-24T02:10:56.319218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219290925834:2119] [node 5] ICS09 handshake done sender: [3:7587241219290925830:2120] self: [3:7587241219290925829:0] peer: [5:7587241217087518446:0] socket: 208 qp: -1
2025-12-24T02:10:56.319233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219290925834:2119] [node 5] ICS10 traffic start
2025-12-24T02:10:56.319242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.319040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241218036692737:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.319063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241218036692737:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.320257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241218036692727:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.322226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241217087518440:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.322371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241217087518357:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241217087518440:2117] held: no
2025-12-24T02:10:56.322396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241217087518357:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.322406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.322412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241217087518357:2050] [node 3] ICP22 created new session: [5:7587241217087518453:2050]
2025-12-24T02:10:56.322419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241217087518453:2050] [node 3] ICS09 handshake done sender: [5:7587241217087518440:2117] self: [5:7587241217087518446:0] peer: [3:7587241219290925829:0] socket: 214 qp: -1
2025-12-24T02:10:56.322427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241217087518453:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.322446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.322436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220714821178:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15604
2025-12-24T02:10:56.322615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220714821171:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.322663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220714821079:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.322671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.322676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220714821079:2049] [node 2] ICP22 created new session: [6:7587241220714821179:2049]
2025-12-24T02:10:56.322682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220714821179:2049] [node 2] ICS09 handshake done sender: [6:7587241220714821171:2119] self: [6:7587241220714821174:0] peer: [2:7587241218036692726:0] socket: 224 qp: -1
2025-12-24T02:10:56.322687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220714821179:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.322707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.334905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241218036692725:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.334942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.334956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241218036692725:2109] [node 6] ICP22 created new session: [2:7587241218036692755:2109]
2025-12-24T02:10:56.334967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241218036692755:2109] [node 6] ICS09 handshake done sender: [2:7587241218036692727:2112] self: [2:7587241218036692726:0] peer: [6:7587241220714821174:0] socket: 201 qp: -1
2025-12-24T02:10:56.334978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241218036692755:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:56.334986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.340454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241218036692737:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.343819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241218036692734:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.343843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.343852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241218036692734:2113] [node 5] ICP22 created new session: [2:7587241218036692757:2113]
2025-12-24T02:10:56.343858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241218036692757:2113] [node 5] ICS09 handshake done sender: [2:7587241218036692737:2117] self: [2:7587241218036692736:0] peer: [5:7587241217087518452:0] socket: 212 qp: -1
2025-12-24T02:10:56.343865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241218036692757:2113] [node 5] ICS10 traffic start
2025-12-24T02:10:56.343874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.344390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241217087518455:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:10:56.351083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241217087518356:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241217087518441:2118] held: no
2025-12-24T02:10:56.351150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241217087518441:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.358327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241217087518356:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.358354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.358365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241217087518356:2049] [node 2] ICP22 created new session: [5:7587241217087518456:2049]
2025-12-24T02:10:56.358373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241217087518456:2049] [node 2] ICS09 handshake done sender: [5:7587241217087518441:2118] self: [5:7587241217087518452:0] peer: [2:7587241218036692736:0] socket: 245 qp: -1
2025-12-24T02:10:56.358385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241217087518456:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.358396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.362426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:56.362527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241218036692745:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:10:56.362541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241218036692655:2048] [node 1] ICP30 unregister session Session# [2:7587241218036692745:2048] VirtualId# [2:7587241218036692723:0]
2025-12-24T02:10:56.362551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241218036692745:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.362608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.362641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.362674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:56.362775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241219290925823:2048] [node 1] ICS01 socket: 221 reason# EndOfStream
2025-12-24T02:10:56.362782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241219290925735:2048] [node 1] ICP30 unregister session Session# [3:7587241219290925823:2048] VirtualId# [3:7587241219290925817:0]
2025-12-24T02:10:56.362790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241219290925823:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.362839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.362864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.365489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:56.365584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241217087518444:2048] [node 1] ICS01 socket: 231 reason# EndOfStream
2025-12-24T02:10:56.365595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241217087518355:2048] [node 1] ICP30 unregister session Session# [5:7587241217087518444:2048] VirtualId# [5:7587241217087518438:0]
2025-12-24T02:10:56.365601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241217087518444:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.365656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.365685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.366765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:56.366846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241217900202844:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:10:56.366854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241217900202760:2048] [node 1] ICP30 unregister session Session# [4:7587241217900202844:2048] VirtualId# [4:7587241217900202824:0]
2025-12-24T02:10:56.366860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241217900202844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.366918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.366953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.368782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:56.368870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241219290925827:2049] [node 2] ICS07 socket disconnect 193 reason# EndOfStream
2025-12-24T02:10:56.368879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241219290925827:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.368914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.368936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241219290925827:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:56.366888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:56.366971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241220714821166:2048] [node 1] ICS07 socket disconnect 195 reason# EndOfStream
2025-12-24T02:10:56.366976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220714821166:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.367022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.367043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241220714821166:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:56.367743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241220714821078:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.367750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220714821166:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.367754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220714821078:2048] [node 1] ICP30 unregister session Session# [6:7587241220714821166:2048] VirtualId# [6:7587241220714821159:0]
2025-12-24T02:10:56.367765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.367771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241220714821078:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.367773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.374234Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241219290925736:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:56.374251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241219290925827:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:56.374573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241219290925736:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.374582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241219290925827:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.374588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241219290925736:2049] [node 2] ICP30 unregister session Session# [3:7587241219290925827:2049] VirtualId# [3:7587241219290925808:0]
2025-12-24T02:10:56.374605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.374618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241219290925736:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.374621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.378763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241220714821078:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:56.382784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:56.382784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:56.382918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241217900202847:2049] [node 2] ICS07 socket disconnect 200 reason# EndOfStream
2025-12-24T02:10:56.382925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241217900202847:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.382993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.383018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241217900202847:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:56.383698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241217900202761:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.383712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241217900202847:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.383716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241217900202761:2049] [node 2] ICP30 unregister session Session# [4:7587241217900202847:2049] VirtualId# [4:7587241217900202832:0]
2025-12-24T02:10:56.383728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.383734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241217900202761:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.383736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14045) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.383360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241217087518456:2049] [node 2] ICS01 socket: 245 reason# EndOfStream
2025-12-24T02:10:56.383374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241217087518356:2049] [node 2] ICP30 unregister session Session# [5:7587241217087518456:2049] VirtualId# [5:7587241217087518452:0]
2025-12-24T02:10:56.383380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241217087518456:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.383431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.383463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.384568Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:56.384637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220714821179:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-12-24T02:10:56.384643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220714821079:2049] [node 2] ICP30 unregister session Session# [6:7587241220714821179:2049] VirtualId# [6:7587241220714821174:0]
2025-12-24T02:10:56.384648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220714821179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.384694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.384714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.390017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:56.390112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241220714821160:2050] [node 3] ICS07 socket disconnect 189 reason# EndOfStream
2025-12-24T02:10:56.390123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220714821160:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.390167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.390188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241220714821160:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:56.390496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241220714821080:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.390511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220714821160:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.390517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220714821080:2050] [node 3] ICP30 unregister session Session# [6:7587241220714821160:2050] VirtualId# [6:7587241220714821099:0]
2025-12-24T02:10:56.390531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.390536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241220714821080:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.390539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.390776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:56.390861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241217900202837:2050] [node 3] ICS01 socket: 203 reason# EndOfStream
2025-12-24T02:10:56.390867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241217900202762:2050] [node 3] ICP30 unregister session Session# [4:7587241217900202837:2050] VirtualId# [4:7587241217900202830:0]
2025-12-24T02:10:56.390872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241217900202837:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.390925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.390954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.390926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:56.391031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241217087518453:2050] [node 3] ICS07 socket disconnect 214 reason# EndOfStream
2025-12-24T02:10:56.391036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241217087518453:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.391117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.391145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241217087518453:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:56.395088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241217087518357:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.395130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241217087518453:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.395136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241217087518357:2050] [node 3] ICP30 unregister session Session# [5:7587241217087518453:2050] VirtualId# [5:7587241217087518446:0]
2025-12-24T02:10:56.395150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.395156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241217087518357:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.395159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.403693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241220714821080:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:56.407652Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:56.407744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220714821176:2073] [node 4] ICS01 socket: 251 reason# EndOfStream
2025-12-24T02:10:56.407753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220714821148:2073] [node 4] ICP30 unregister session Session# [6:7587241220714821176:2073] VirtualId# [6:7587241220714821173:0]
2025-12-24T02:10:56.407760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220714821176:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.407818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:56.407842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:56.418491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:56.418585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220714821169:2082] [node 5] ICS01 socket: 205 reason# EndOfStream
2025-12-24T02:10:56.418594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220714821149:2082] [node 5] ICP30 unregister session Session# [6:7587241220714821169:2082] VirtualId# [6:7587241220714821164:0]
2025-12-24T02:10:56.418599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220714821169:2082] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.418640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:56.418662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:56.438324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220506219338:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.438332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220506219339:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.437032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241219347182454:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:56.437038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241219347182455:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.450786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219086717411:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.450802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219086717412:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.450807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219086717413:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.453040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241219087840069:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.453062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241219087840070:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.453068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241219087840071:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.454839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241217849124224:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.457443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241217849124225:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.457452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241217849124226:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.439768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219679059110:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.439774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219679059111:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.493013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220506219403:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:56.493101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220506219404:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.494231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241219347182523:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:56.490828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059170:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
2025-12-24T02:10:56.494079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059171:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
2025-12-24T02:10:56.494315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241219347182524:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:56.494466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241219347182525:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:56.499175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241219087840095:2071] [node 6] ICP01 ready to work
2025-12-24T02:10:56.502379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241219087840123:2085] [node 4] ICP01 ready to work
2025-12-24T02:10:56.511707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241217849124278:2066] [node 4] ICP01 ready to work
2025-12-24T02:10:56.511811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241217849124279:2074] [node 5] ICP01 ready to work
2025-12-24T02:10:56.521247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241219087840123:2085] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18781663ns) {Configure:104(18780899ns) {ProcessConfigured:135(18552924ns) {StartInitialHandshake:157(18552358ns) {DropHandshakes:470(390ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1399ns) {SwitchToState:179(898ns)}}}}}
2025-12-24T02:10:56.521439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219679059179:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:56.521528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219679059180:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:56.521596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241219679059181:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.522982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220506219418:2115] [node 6] ICP01 ready to work
2025-12-24T02:10:56.523054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220506219413:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35061
2025-12-24T02:10:56.523072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220506219414:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35061
2025-12-24T02:10:56.523443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220506219417:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35061
2025-12-24T02:10:56.523520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241219679059110:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20331844ns) {Configure:104(20331324ns) {ProcessConfigured:135(19979419ns) {StartInitialHandshake:157(19979219ns) {DropHandshakes:470(455ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(531ns) {SwitchToState:179(208ns)}}}}}
2025-12-24T02:10:56.523720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059176:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.524181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840147:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.525279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220506219421:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.526809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241219086717411:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19959319ns) {Configure:104(19958562ns) {ProcessConfigured:135(17951521ns) {StartInitialHandshake:157(17950524ns) {DropHandshakes:470(646ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(121ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(1987ns) {SwitchToState:179(778ns)}}}}}
2025-12-24T02:10:56.527023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219086717478:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:56.527123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219086717479:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:56.527162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717472:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.528342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717480:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.528518Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241219086717412:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21651684ns) {Configure:104(21650544ns) {ProcessConfigured:135(19635209ns) {StartInitialHandshake:157(19634643ns) {DropHandshakes:470(210ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:477(709ns) {SwitchToState:179(220ns)}}}}}
2025-12-24T02:10:56.529558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717472:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.530477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219086717412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.535170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241219347182525:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40338384ns) {Configure:104(40337743ns) {ProcessConfigured:135(40134212ns) {StartInitialHandshake:157(40133490ns) {DropHandshakes:470(318ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1401ns) {SwitchToState:179(903ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.539196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124296:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.541688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219414:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.539897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241219347182532:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50548
2025-12-24T02:10:56.539941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241219347182533:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50548
2025-12-24T02:10:56.539967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241219347182534:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50548
2025-12-24T02:10:56.545176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182532:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.545221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182533:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.546642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219408:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.546666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220506219408:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.549013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717492:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.553463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241217849124226:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(57238542ns) {Configure:104(57237615ns) {ProcessConfigured:135(57010279ns) {StartInitialHandshake:157(57009637ns) {DropHandshakes:470(346ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1347ns) {SwitchToState:179(816ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.554278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840148:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.554629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219406:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.554642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220506219406:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.555456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219413:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.555514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220506219406:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.556095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220506219339:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.556111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.556117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220506219339:2049] [node 3] ICP22 created new session: [2:7587241220506219424:2049]
2025-12-24T02:10:56.556123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220506219424:2049] [node 3] ICS09 handshake done sender: [2:7587241220506219406:2110] self: [2:7587241220506219405:0] peer: [3:7587241219679059193:0] socket: 196 qp: -1
2025-12-24T02:10:56.556130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220506219424:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.556136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.556598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840149:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.556319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241220506219338:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.558468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241219087840149:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.558500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241219087840070:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.553753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059192:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
2025-12-24T02:10:56.554500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059171:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.554535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241219679059111:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.555237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059170:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.555331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059194:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
2025-12-24T02:10:56.555345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241219679059110:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.555699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219679059171:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.555753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241219679059111:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.555767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.555775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219679059111:2049] [node 2] ICP22 created new session: [3:7587241219679059196:2049]
2025-12-24T02:10:56.555780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219679059196:2049] [node 2] ICS09 handshake done sender: [3:7587241219679059171:2108] self: [3:7587241219679059193:0] peer: [2:7587241220506219405:0] socket: 198 qp: -1
2025-12-24T02:10:56.555788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219679059196:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.555806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.558613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219410:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.558622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220506219410:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.559174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220506219410:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.564262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717492:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.564351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.564416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.563132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220506219404:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.563154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.563163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220506219404:2109] [node 5] ICP22 created new session: [2:7587241220506219429:2109]
2025-12-24T02:10:56.563169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220506219429:2109] [node 5] ICS09 handshake done sender: [2:7587241220506219410:2112] self: [2:7587241220506219409:0] peer: [5:7587241219087840150:0] socket: 236 qp: -1
2025-12-24T02:10:56.563181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220506219429:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:56.563191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.566769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182517:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.566792Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241219347182517:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.566882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220506219408:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.568386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124303:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.568405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124304:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.566870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182515:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.566880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241219347182515:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.568905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241219347182515:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.569126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241219347182454:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.569139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.569144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241219347182454:2048] [node 2] ICP22 created new session: [1:7587241219347182539:2048]
2025-12-24T02:10:56.569149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241219347182539:2048] [node 2] ICS09 handshake done sender: [1:7587241219347182515:2107] self: [1:7587241219347182514:0] peer: [2:7587241220506219426:0] socket: 192 qp: -1
2025-12-24T02:10:56.569154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241219347182539:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:56.569160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.569913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840151:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.568124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220506219403:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.568143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.568151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220506219403:2108] [node 4] ICP22 created new session: [2:7587241220506219433:2108]
2025-12-24T02:10:56.568158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220506219433:2108] [node 4] ICS09 handshake done sender: [2:7587241220506219408:2111] self: [2:7587241220506219407:0] peer: [4:7587241219086717491:0] socket: 197 qp: -1
2025-12-24T02:10:56.568164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220506219433:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:56.568173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.571754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241220506219338:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241220506219413:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.572550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220506219435:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35061
2025-12-24T02:10:56.573251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220506219413:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.573320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241220506219338:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.573333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.573341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220506219338:2048] [node 1] ICP22 created new session: [2:7587241220506219436:2048]
2025-12-24T02:10:56.573347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220506219436:2048] [node 1] ICS09 handshake done sender: [2:7587241220506219413:2113] self: [2:7587241220506219426:0] peer: [1:7587241219347182514:0] socket: 203 qp: -1
2025-12-24T02:10:56.573354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220506219436:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.573361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.571213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717493:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.570262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241219347182517:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.574346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241219679059199:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13383
2025-12-24T02:10:56.574790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219679059170:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.574868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241217849124303:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.574922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241217849124226:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.574986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241219087840149:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.575077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241219087840070:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.575095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.575113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241219087840070:2049] [node 2] ICP22 created new session: [5:7587241219087840153:2049]
2025-12-24T02:10:56.575119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241219087840153:2049] [node 2] ICS09 handshake done sender: [5:7587241219087840149:2117] self: [5:7587241219087840150:0] peer: [2:7587241220506219409:0] socket: 237 qp: -1
2025-12-24T02:10:56.575157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241217849124296:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.575183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241217849124279:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.575127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241219087840153:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.575138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.576387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059176:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.576424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059187:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.576427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241219679059187:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.576822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241219679059110:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.576839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.576845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219679059110:2048] [node 1] ICP22 created new session: [3:7587241219679059200:2048]
2025-12-24T02:10:56.576851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219679059200:2048] [node 1] ICS09 handshake done sender: [3:7587241219679059170:2107] self: [3:7587241219679059195:0] peer: [1:7587241219347182516:0] socket: 194 qp: -1
2025-12-24T02:10:56.576857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219679059200:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.576864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.575352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840152:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.575431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241219087840098:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.575434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241219087840098:2074] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.575948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241219087840098:2074] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.576016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241219087840095:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.576024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.576031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241219087840095:2071] [node 6] ICP22 created new session: [5:7587241219087840156:2071]
2025-12-24T02:10:56.576036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241219087840156:2071] [node 6] ICS09 handshake done sender: [5:7587241219087840098:2074] self: [5:7587241219087840097:0] peer: [6:7587241217849124306:0] socket: 207 qp: -1
2025-12-24T02:10:56.576043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241219087840156:2071] [node 6] ICS10 traffic start
2025-12-24T02:10:56.576049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.577235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241219087840147:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.578210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241219087840071:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.578918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.578959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.579045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241217849124304:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.579071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241217849124225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.579066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219679059187:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.579183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059185:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.579186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241219679059185:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.579518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241219679059181:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.579533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.579543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219679059181:2111] [node 6] ICP22 created new session: [3:7587241219679059204:2111]
2025-12-24T02:10:56.579549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219679059204:2111] [node 6] ICS09 handshake done sender: [3:7587241219679059187:2116] self: [3:7587241219679059186:0] peer: [6:7587241217849124305:0] socket: 216 qp: -1
2025-12-24T02:10:56.579555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219679059204:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:56.579574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.582058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241219087840152:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.582086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241219087840069:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.580496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241219347182455:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:10:56.580513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.580520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241219347182455:2049] [node 3] ICP22 created new session: [1:7587241219347182543:2049]
2025-12-24T02:10:56.580526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241219347182543:2049] [node 3] ICS09 handshake done sender: [1:7587241219347182517:2108] self: [1:7587241219347182516:0] peer: [3:7587241219679059195:0] socket: 193 qp: -1
2025-12-24T02:10:56.580531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241219347182543:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.580538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.582209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182529:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.582213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241219347182529:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.582804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840161:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.582800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241219347182529:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.582770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220506219420:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.582785Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220506219420:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.582891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241219347182524:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.582901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.582907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241219347182524:2110] [node 5] ICP22 created new session: [1:7587241219347182547:2110]
2025-12-24T02:10:56.582912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241219347182547:2110] [node 5] ICS09 handshake done sender: [1:7587241219347182529:2113] self: [1:7587241219347182528:0] peer: [5:7587241219087840160:0] socket: 246 qp: -1
2025-12-24T02:10:56.582921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241219347182547:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:56.582928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.586782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220506219420:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.587082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219086717472:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.592535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241217849124296:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.592602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220506219418:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.592626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.592634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220506219418:2115] [node 6] ICP22 created new session: [2:7587241220506219439:2115]
2025-12-24T02:10:56.592640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220506219439:2115] [node 6] ICS09 handshake done sender: [2:7587241220506219420:2118] self: [2:7587241220506219419:0] peer: [6:7587241217849124309:0] socket: 241 qp: -1
2025-12-24T02:10:56.592648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220506219439:2115] [node 6] ICS10 traffic start
2025-12-24T02:10:56.592657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.593048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241217849124303:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.593109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241217849124279:2074] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.593120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.593125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241217849124279:2074] [node 5] ICP22 created new session: [6:7587241217849124310:2074]
2025-12-24T02:10:56.593129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241217849124310:2074] [node 5] ICS09 handshake done sender: [6:7587241217849124296:2112] self: [6:7587241217849124306:0] peer: [5:7587241219087840097:0] socket: 212 qp: -1
2025-12-24T02:10:56.593134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241217849124310:2074] [node 5] ICS10 traffic start
2025-12-24T02:10:56.593152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.593277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241217849124226:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.593284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.593288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241217849124226:2050] [node 3] ICP22 created new session: [6:7587241217849124312:2050]
2025-12-24T02:10:56.593291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241217849124312:2050] [node 3] ICS09 handshake done sender: [6:7587241217849124303:2116] self: [6:7587241217849124305:0] peer: [3:7587241219679059186:0] socket: 219 qp: -1
2025-12-24T02:10:56.593295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241217849124312:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.593299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.594444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241217849124225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241217849124304:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.595208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219086717412:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.595227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.595233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219086717412:2049] [node 2] ICP22 created new session: [4:7587241219086717497:2049]
2025-12-24T02:10:56.595239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219086717497:2049] [node 2] ICS09 handshake done sender: [4:7587241219086717472:2108] self: [4:7587241219086717491:0] peer: [2:7587241220506219407:0] socket: 205 qp: -1
2025-12-24T02:10:56.595246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219086717497:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.595251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.595417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219086717411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.595532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.596405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717480:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.597459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219086717413:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.597649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059192:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.597690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241219679059183:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.597694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241219679059183:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.598499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219679059183:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.598404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717502:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.598676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219086717480:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.599096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241219679059179:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.599460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182534:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.599128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.599135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219679059179:2109] [node 4] ICP22 created new session: [3:7587241219679059209:2109]
2025-12-24T02:10:56.599141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219679059209:2109] [node 4] ICS09 handshake done sender: [3:7587241219679059183:2114] self: [3:7587241219679059182:0] peer: [4:7587241219086717501:0] socket: 214 qp: -1
2025-12-24T02:10:56.599146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219679059209:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:56.599497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182527:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.599153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.599500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241219347182527:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.600195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241219347182524:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(17273226ns) {HandleHandshakeStatus:329(17272731ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(199ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(17211222ns) {ForwardSessionEventToSession:612(17202812ns) ForwardSessionEventToSession:612(994ns) ForwardSessionEventToSession:612(462ns) ForwardSessionEventToSession:612(1156ns) ForwardSessionEventToSession:612(442ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(390ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(853ns)}}}
2025-12-24T02:10:56.600366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219086717413:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.600381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.600385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219086717413:2050] [node 3] ICP22 created new session: [4:7587241219086717503:2050]
2025-12-24T02:10:56.600391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219086717503:2050] [node 3] ICS09 handshake done sender: [4:7587241219086717480:2112] self: [4:7587241219086717501:0] peer: [3:7587241219679059182:0] socket: 220 qp: -1
2025-12-24T02:10:56.600396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219086717503:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.600413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.601525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241219679059185:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.601104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241219347182527:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.601174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241219347182523:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.601183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.601188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241219347182523:2109] [node 4] ICP22 created new session: [1:7587241219347182551:2109]
2025-12-24T02:10:56.601192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241219347182551:2109] [node 4] ICS09 handshake done sender: [1:7587241219347182527:2112] self: [1:7587241219347182526:0] peer: [4:7587241219086717499:0] socket: 232 qp: -1
2025-12-24T02:10:56.601196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241219347182551:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:56.601201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.602115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241219679059180:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.602134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.602140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241219679059180:2110] [node 5] ICP22 created new session: [3:7587241219679059211:2110]
2025-12-24T02:10:56.602145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241219679059211:2110] [node 5] ICS09 handshake done sender: [3:7587241219679059185:2115] self: [3:7587241219679059184:0] peer: [5:7587241219087840159:0] socket: 215 qp: -1
2025-12-24T02:10:56.602149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241219679059211:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:56.602156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.602621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219086717505:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59467
2025-12-24T02:10:56.606825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241219087840152:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.606929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241219087840071:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18228) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241219087840147:2115] held: no
2025-12-24T02:10:56.606954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241219087840069:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.606964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.606969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241219087840069:2048] [node 1] ICP22 created new session: [5:7587241219087840163:2048]
2025-12-24T02:10:56.606974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241219087840163:2048] [node 1] ICS09 handshake done sender: [5:7587241219087840152:2119] self: [5:7587241219087840160:0] peer: [1:7587241219347182528:0] socket: 247 qp: -1
2025-12-24T02:10:56.606980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241219087840163:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.606987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.606918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124314:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.606985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124315:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.607261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.607478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840162:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.607412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241217849124314:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.607437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241217849124224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.607616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241219087840147:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.607555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241219347182531:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.607563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241219347182531:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.608346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124318:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.608498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241217849124304:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.608578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241217849124225:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.608589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.608594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241217849124225:2049] [node 2] ICP22 created new session: [6:7587241217849124319:2049]
2025-12-24T02:10:56.608600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241217849124319:2049] [node 2] ICS09 handshake done sender: [6:7587241217849124304:2117] self: [6:7587241217849124309:0] peer: [2:7587241220506219419:0] socket: 244 qp: -1
2025-12-24T02:10:56.608605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241217849124319:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.608611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.608838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241219347182531:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.608985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241217849124314:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.609034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241217849124224:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.609045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.609050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241217849124224:2048] [node 1] ICP22 created new session: [6:7587241217849124321:2048]
2025-12-24T02:10:56.609055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241217849124321:2048] [node 1] ICS09 handshake done sender: [6:7587241217849124314:2120] self: [6:7587241217849124317:0] peer: [1:7587241219347182530:0] socket: 257 qp: -1
2025-12-24T02:10:56.609060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241217849124321:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.609066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.609784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241219347182525:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.609800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.609805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241219347182525:2111] [node 6] ICP22 created new session: [1:7587241219347182554:2111]
2025-12-24T02:10:56.609810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241219347182554:2111] [node 6] ICS09 handshake done sender: [1:7587241219347182531:2114] self: [1:7587241219347182530:0] peer: [6:7587241217849124317:0] socket: 253 qp: -1
2025-12-24T02:10:56.609814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241219347182554:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:56.609820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.611076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241220506219404:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:56.610763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241219086717411:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29893) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241219086717493:2118] held: no
2025-12-24T02:10:56.610890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717496:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.610920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219086717493:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.614929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241219087840071:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.614945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.614952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241219087840071:2050] [node 3] ICP22 created new session: [5:7587241219087840166:2050]
2025-12-24T02:10:56.614956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241219087840166:2050] [node 3] ICS09 handshake done sender: [5:7587241219087840147:2115] self: [5:7587241219087840159:0] peer: [3:7587241219679059184:0] socket: 218 qp: -1
2025-12-24T02:10:56.614963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241219087840166:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.614969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.615060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241219087840148:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.619176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219086717411:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.619206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.619218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219086717411:2048] [node 1] ICP22 created new session: [4:7587241219086717506:2048]
2025-12-24T02:10:56.619224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219086717506:2048] [node 1] ICS09 handshake done sender: [4:7587241219086717493:2118] self: [4:7587241219086717499:0] peer: [1:7587241219347182526:0] socket: 233 qp: -1
2025-12-24T02:10:56.619235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219086717506:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.619244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.619383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717484:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.619388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241219086717484:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.621247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241219087840169:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57954
2025-12-24T02:10:56.622915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241217849124316:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.622960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241217849124278:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.622891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219086717484:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.626889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219086717486:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.626899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241219086717486:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.622925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241219087840123:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.629471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241217849124324:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52079
2025-12-24T02:10:56.630535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241219087840148:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.634795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241219087840123:2085] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.634820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.634830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241219087840123:2085] [node 4] ICP22 created new session: [5:7587241219087840170:2085]
2025-12-24T02:10:56.634836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241219087840170:2085] [node 4] ICS09 handshake done sender: [5:7587241219087840148:2116] self: [5:7587241219087840168:0] peer: [4:7587241219086717483:0] socket: 224 qp: -1
2025-12-24T02:10:56.634850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241219087840170:2085] [node 4] ICS10 traffic start
2025-12-24T02:10:56.634861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.634835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219086717486:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.634922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241219086717478:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.634939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.634948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219086717478:2109] [node 5] ICP22 created new session: [4:7587241219086717511:2109]
2025-12-24T02:10:56.634956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219086717511:2109] [node 5] ICS09 handshake done sender: [4:7587241219086717484:2114] self: [4:7587241219086717483:0] peer: [5:7587241219087840168:0] socket: 222 qp: -1
2025-12-24T02:10:56.634963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219086717511:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:56.634972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.636777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241217849124316:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.636871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241217849124278:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.636883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.636889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241217849124278:2066] [node 4] ICP22 created new session: [6:7587241217849124325:2066]
2025-12-24T02:10:56.636893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241217849124325:2066] [node 4] ICS09 handshake done sender: [6:7587241217849124316:2122] self: [6:7587241217849124323:0] peer: [4:7587241219086717485:0] socket: 221 qp: -1
2025-12-24T02:10:56.636898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241217849124325:2066] [node 4] ICS10 traffic start
2025-12-24T02:10:56.636904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.636389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241219086717479:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.636408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.636415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219086717479:2110] [node 6] ICP22 created new session: [4:7587241219086717513:2110]
2025-12-24T02:10:56.636421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219086717513:2110] [node 6] ICS09 handshake done sender: [4:7587241219086717486:2115] self: [4:7587241219086717485:0] peer: [6:7587241217849124323:0] socket: 204 qp: -1
2025-12-24T02:10:56.636427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219086717513:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:56.636433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.642873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:56.642969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241220506219436:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-12-24T02:10:56.642978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241220506219338:2048] [node 1] ICP30 unregister session Session# [2:7587241220506219436:2048] VirtualId# [2:7587241220506219426:0]
2025-12-24T02:10:56.642986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241220506219436:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.643032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.643063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.646897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:56.647000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241219679059200:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-12-24T02:10:56.647007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241219679059110:2048] [node 1] ICP30 unregister session Session# [3:7587241219679059200:2048] VirtualId# [3:7587241219679059195:0]
2025-12-24T02:10:56.647014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241219679059200:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.647072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.647109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.649651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:56.649738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241219087840163:2048] [node 1] ICS07 socket disconnect 247 reason# EndOfStream
2025-12-24T02:10:56.649747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241219087840163:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.649799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.649824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241219087840163:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:56.650145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241219087840069:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29893) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:56.650159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241219087840163:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:56.650163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241219087840069:2048] [node 1] ICP30 unregister session Session# [5:7587241219087840163:2048] VirtualId# [5:7587241219087840160:0]
2025-12-24T02:10:56.650179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.650184Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241219087840069:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.650187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29893) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:56.650747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:56.650834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241217849124321:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-12-24T02:10:56.650841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241217849124224:2048] [node 1] ICP30 unregister session Session# [6:7587241217849124321:2048] VirtualId# [6:7587241217849124317:0]
2025-12-24T02:10:56.650847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241217849124321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.650899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.650925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.650747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:56.651084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241219086717506:2048] [node 1] ICS01 socket: 233 reason# EndOfStream
2025-12-24T02:10:56.651091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241219086717411:2048] [node 1] ICP30 unregister session Session# [4:7587241219086717506:2048] VirtualId# [4:7587241219086717499:0]
2025-12-24T02:10:56.651096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241219086717506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.651145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.651169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.660124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241219087840069:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:56.664169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:56.664272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241219086717497:2049] [node 2] ICS01 socket: 205 reason# EndOfStream
2025-12-24T02:10:56.664280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241219086717412:2049] [node 2] ICP30 unregister session Session# [4:7587241219086717497:2049] VirtualId# [4:7587241219086717491:0]
2025-12-24T02:10:56.664287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241219086717497:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.664341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.664366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.666784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:56.666868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241217849124319:2049] [node 2] ICS01 socket: 244 reason# EndOfStream
2025-12-24T02:10:56.666876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241217849124225:2049] [node 2] ICP30 unregister session Session# [6:7587241217849124319:2049] VirtualId# [6:7587241217849124309:0]
2025-12-24T02:10:56.666883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241217849124319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.666942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.666966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.669633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:56.669724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241219087840153:2049] [node 2] ICS01 socket: 237 reason# EndOfStream
2025-12-24T02:10:56.669732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241219087840070:2049] [node 2] ICP30 unregister session Session# [5:7587241219087840153:2049] VirtualId# [5:7587241219087840150:0]
2025-12-24T02:10:56.669738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241219087840153:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.669799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:56.669828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:56.691762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:56.691873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241217849124312:2050] [node 3] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:10:56.691884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241217849124226:2050] [node 3] ICP30 unregister session Session# [6:7587241217849124312:2050] VirtualId# [6:7587241217849124305:0]
2025-12-24T02:10:56.691891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241217849124312:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.691945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.691970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.695437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:56.695545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241219087840166:2050] [node 3] ICS01 socket: 218 reason# EndOfStream
2025-12-24T02:10:56.695554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241219087840071:2050] [node 3] ICP30 unregister session Session# [5:7587241219087840166:2050] VirtualId# [5:7587241219087840159:0]
2025-12-24T02:10:56.695562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241219087840166:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.695631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:56.695658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:56.703911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:56.704021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241217849124325:2066] [node 4] ICS01 socket: 221 reason# EndOfStream
2025-12-24T02:10:56.704030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241217849124278:2066] [node 4] ICP30 unregister session Session# [6:7587241217849124325:2066] VirtualId# [6:7587241217849124323:0]
2025-12-24T02:10:56.704037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241217849124325:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.704097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:56.704123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:56.729682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241217427814588:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:56.729689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241217427814589:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.731407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220754446289:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.731499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220754446290:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:56.740792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219276070170:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.740799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219276070171:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.732795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241217232361544:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.742412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241220080124011:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.732802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241217232361545:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.740803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219276070172:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.742418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241220080124012:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.742423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241220080124013:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:56.754810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220498628313:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:56.754835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220498628314:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:56.754847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220498628315:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.768007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446345:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.768037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446346:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.768569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241220080124024:2059] [node 4] ICP01 ready to work
2025-12-24T02:10:56.768969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220754446348:2105] [node 4] ICP01 ready to work
2025-12-24T02:10:56.767049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241217232361594:2096] [node 4] ICP01 ready to work
2025-12-24T02:10:56.767947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241217232361600:2099] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.769549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241217427814656:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:56.769633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241217427814657:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:56.769703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241217427814658:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:56.775532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220498628333:2063] [node 4] ICP01 ready to work
2025-12-24T02:10:56.777877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241220498628349:2074] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.785538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241217232361612:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.787556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361610:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.794729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241217427814651:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42244
2025-12-24T02:10:56.793604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219276070242:2099] [node 5] ICP01 ready to work
2025-12-24T02:10:56.793732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241219276070243:2101] [node 6] ICP01 ready to work
2025-12-24T02:10:56.798826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241220080124013:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28462657ns) {Configure:104(28461820ns) {ProcessConfigured:135(28174312ns) {StartInitialHandshake:157(28173968ns) {DropHandshakes:470(473ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:477(1139ns) {SwitchToState:179(710ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.801342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241217427814663:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42244
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.801461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241217427814662:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42244
2025-12-24T02:10:56.808219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361611:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.808753Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241217427814589:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40662874ns) {Configure:104(40661546ns) {ProcessConfigured:135(40409778ns) {StartInitialHandshake:157(40409367ns) {DropHandshakes:470(422ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(176ns) PrepareNewSessionHandshake:477(967ns) {SwitchToState:179(513ns)}}}}}
2025-12-24T02:10:56.808849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814663:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.812139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070247:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.812356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241217232361612:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21546485ns) {Configure:104(21545215ns) {ProcessConfigured:135(21202320ns) {StartInitialHandshake:157(21201904ns) {DropHandshakes:470(981ns) {DropIncomingHandshake:442(142ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(698ns) {SwitchToState:179(276ns)}}}}}
2025-12-24T02:10:56.812206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070248:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.812219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070249:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.812834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814651:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.812760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070249:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.815882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070247:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.822824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241220754446348:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52030220ns) {Configure:104(32039340ns) {ProcessConfigured:135(29999203ns) {StartInitialHandshake:157(29998661ns) {DropHandshakes:470(703ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1428ns) {SwitchToState:179(718ns)}}}}}
2025-12-24T02:10:56.823037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220754446359:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:56.823196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241220754446361:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:56.824430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446360:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.824794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241220754446290:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(54280195ns) {Configure:104(34031788ns) {ProcessConfigured:135(31762342ns) {StartInitialHandshake:157(31761524ns) {DropHandshakes:470(286ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(531ns) {SwitchToState:179(89ns)}}}}}
2025-12-24T02:10:56.818816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241217232361545:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27893335ns) {Configure:104(27892700ns) {ProcessConfigured:135(27590807ns) {StartInitialHandshake:157(27590158ns) {DropHandshakes:470(648ns) {DropIncomingHandshake:442(112ns) DropOutgoingHandshake:457(113ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(1297ns) {SwitchToState:179(704ns)}}}}}
2025-12-24T02:10:56.818905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361611:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.826819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241219276070242:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(33024963ns) {Configure:104(33024292ns) {ProcessConfigured:135(32729451ns) {StartInitialHandshake:157(32729206ns) {DropHandshakes:470(511ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(1525ns) {SwitchToState:179(740ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.826978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.827792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361596:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.827814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241217232361596:2097] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.825192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446345:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.825211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446360:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.831044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446346:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.830806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241220080124078:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.835267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628390:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.835305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628391:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.835327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628392:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.838824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070248:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.838940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124088:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.839006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124089:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.839050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124090:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.839153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241220754446289:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70118156ns) {Configure:104(70117188ns) {ProcessConfigured:135(69751996ns) {StartInitialHandshake:157(69751127ns) {DropHandshakes:470(388ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(941ns) {SwitchToState:179(653ns)}}}}}
2025-12-24T02:10:56.839183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241220754446289:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.840021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814649:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.840035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241217427814649:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.845057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241220080124088:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.845096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241220080124089:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.845129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241220080124011:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.845139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241220080124013:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.842811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241217232361544:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34477052ns) {Configure:104(34474641ns) {ProcessConfigured:135(34139910ns) {StartInitialHandshake:157(34139255ns) {DropHandshakes:470(921ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(392ns) PrepareNewSessionHandshake:477(1357ns) {SwitchToState:179(711ns)}}}}}
2025-12-24T02:10:56.840813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241217427814649:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.840896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241217427814588:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.843368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219276070172:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.840909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.840917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241217427814588:2048] [node 2] ICP22 created new session: [1:7587241217427814671:2048]
2025-12-24T02:10:56.843408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241219276070172:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27469701ns) {IncomingHandshake:261(27468921ns) {DropIncomingHandshake:442(71ns) PrepareNewSessionHandshake:477(938ns) {SwitchToState:179(233ns)} GenerateSessionVirtualId:988(277ns) IssueIncomingHandshakeReply:193(27464254ns) {SwitchToState:179(1004ns)}}}
2025-12-24T02:10:56.840924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241217427814671:2048] [node 2] ICS09 handshake done sender: [1:7587241217427814649:2107] self: [1:7587241217427814648:0] peer: [2:7587241220754446373:0] socket: 198 qp: -1
2025-12-24T02:10:56.840934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241217427814671:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:56.840942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.847778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361620:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.847794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241217232361620:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.850002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814665:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.850020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241217427814665:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.849987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241220080124078:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19073916ns) {Configure:104(19072983ns) {ProcessConfigured:135(18767111ns) {StartInitialHandshake:157(18766287ns) {DropHandshakes:470(912ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1122ns) {SwitchToState:179(546ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.846871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220498628390:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.846903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220498628313:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.850325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241220498628314:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(79493141ns) {Configure:104(79492149ns) {ProcessConfigured:135(79112167ns) {StartInitialHandshake:157(79111584ns) {DropHandshakes:470(544ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1573ns) {SwitchToState:179(609ns)}}}}}
2025-12-24T02:10:56.850559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070255:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.851150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361610:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.851255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241217232361596:2097] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.854481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446374:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.854632Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241217427814656:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(59722912ns) {Configure:104(59721670ns) {ProcessConfigured:135(59390788ns) {StartInitialHandshake:157(59389647ns) {DropHandshakes:470(568ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(113ns)} GenerateSessionVirtualId:988(361ns) PrepareNewSessionHandshake:477(1549ns) {SwitchToState:179(1101ns)}}}}}
2025-12-24T02:10:56.854459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241217232361620:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.854797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241217232361612:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.854809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.854816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241217232361612:2106] [node 5] ICP22 created new session: [3:7587241217232361628:2106]
2025-12-24T02:10:56.854822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241217232361628:2106] [node 5] ICS09 handshake done sender: [3:7587241217232361620:2114] self: [3:7587241217232361617:0] peer: [5:7587241220080124092:0] socket: 229 qp: -1
2025-12-24T02:10:56.854829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241217232361628:2106] [node 5] ICS10 traffic start
2025-12-24T02:10:56.854873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814661:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.854887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241217427814661:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.854838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.855030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220498628391:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.855133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220498628315:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.858769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219276070247:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.858780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124094:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.859231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241220080124089:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.859311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241220080124013:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.859326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.859335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241220080124013:2050] [node 3] ICP22 created new session: [5:7587241220080124095:2050]
2025-12-24T02:10:56.859343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241220080124095:2050] [node 3] ICS09 handshake done sender: [5:7587241220080124089:2115] self: [5:7587241220080124092:0] peer: [3:7587241217232361617:0] socket: 238 qp: -1
2025-12-24T02:10:56.859354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241220080124095:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.859363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.858896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814662:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.863858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241217427814665:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.864317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361602:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.864331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241217232361602:2100] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.864833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241217232361594:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.864847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.864853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241217232361594:2096] [node 4] ICP22 created new session: [3:7587241217232361630:2096]
2025-12-24T02:10:56.864858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241217232361630:2096] [node 4] ICS09 handshake done sender: [3:7587241217232361596:2097] self: [3:7587241217232361595:0] peer: [4:7587241219276070250:0] socket: 199 qp: -1
2025-12-24T02:10:56.864866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241217232361630:2096] [node 4] ICS10 traffic start
2025-12-24T02:10:56.864884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.863097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628396:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.864070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220498628390:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.864144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220498628313:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.864159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.864165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220498628313:2048] [node 1] ICP22 created new session: [6:7587241220498628397:2048]
2025-12-24T02:10:56.864172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220498628397:2048] [node 1] ICS09 handshake done sender: [6:7587241220498628390:2114] self: [6:7587241220498628393:0] peer: [1:7587241217427814664:0] socket: 224 qp: -1
2025-12-24T02:10:56.864179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220498628397:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.864202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.866991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241217427814661:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.870265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628399:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.871472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.871591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241217232361612:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(16764512ns) {HandleHandshakeStatus:329(16763215ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(272ns) SwitchToState:179(142ns) ProcessPendingSessionEvents:495(12118153ns) {ForwardSessionEventToSession:612(12104227ns) ForwardSessionEventToSession:612(1627ns) ForwardSessionEventToSession:612(1173ns) ForwardSessionEventToSession:612(957ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(822ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(951ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(850ns)}}}
2025-12-24T02:10:56.872302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241220080124088:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.872392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241220080124011:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.872409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.872417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241220080124011:2048] [node 1] ICP22 created new session: [5:7587241220080124099:2048]
2025-12-24T02:10:56.872423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241220080124099:2048] [node 1] ICS09 handshake done sender: [5:7587241220080124088:2114] self: [5:7587241220080124091:0] peer: [1:7587241217427814660:0] socket: 225 qp: -1
2025-12-24T02:10:56.872430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241220080124099:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.872451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.874889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241217232361602:2100] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.878091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220754446345:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.878445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070253:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.878493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219276070171:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.878323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241220754446289:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.878345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.878354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220754446289:2048] [node 1] ICP22 created new session: [2:7587241220754446376:2048]
2025-12-24T02:10:56.878360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220754446376:2048] [node 1] ICS09 handshake done sender: [2:7587241220754446345:2103] self: [2:7587241220754446373:0] peer: [1:7587241217427814648:0] socket: 202 qp: -1
2025-12-24T02:10:56.878367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220754446376:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.878379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.878775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241217232361600:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.878804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.878813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241217232361600:2099] [node 6] ICP22 created new session: [3:7587241217232361637:2099]
2025-12-24T02:10:56.878820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241217232361637:2099] [node 6] ICS09 handshake done sender: [3:7587241217232361602:2100] self: [3:7587241217232361601:0] peer: [6:7587241220498628395:0] socket: 228 qp: -1
2025-12-24T02:10:56.878825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241217232361637:2099] [node 6] ICS10 traffic start
2025-12-24T02:10:56.878855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.878947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361636:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.879093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219276070172:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.879131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.879140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219276070172:2050] [node 3] ICP22 created new session: [4:7587241219276070257:2050]
2025-12-24T02:10:56.879148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219276070257:2050] [node 3] ICS09 handshake done sender: [4:7587241219276070247:2114] self: [4:7587241219276070250:0] peer: [3:7587241217232361595:0] socket: 207 qp: -1
2025-12-24T02:10:56.879156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219276070257:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.879168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.885410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.886840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241217427814658:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.886869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.886879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241217427814658:2111] [node 6] ICP22 created new session: [1:7587241217427814680:2111]
2025-12-24T02:10:56.886885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241217427814680:2111] [node 6] ICS09 handshake done sender: [1:7587241217427814665:2115] self: [1:7587241217427814664:0] peer: [6:7587241220498628393:0] socket: 223 qp: -1
2025-12-24T02:10:56.886892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241217427814680:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:56.886914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.887122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241217427814657:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.887132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.887137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241217427814657:2110] [node 5] ICP22 created new session: [1:7587241217427814682:2110]
2025-12-24T02:10:56.887141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241217427814682:2110] [node 5] ICS09 handshake done sender: [1:7587241217427814661:2112] self: [1:7587241217427814660:0] peer: [5:7587241220080124091:0] socket: 222 qp: -1
2025-12-24T02:10:56.887147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241217427814682:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:56.887152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.891437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.891045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241217427814679:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42244
2025-12-24T02:10:56.894955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070260:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.894995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241219276070170:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.897460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446358:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.897482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220754446358:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.895028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220498628391:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.895202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220498628315:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.895217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.895224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220498628315:2050] [node 3] ICP22 created new session: [6:7587241220498628402:2050]
2025-12-24T02:10:56.895230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220498628402:2050] [node 3] ICS09 handshake done sender: [6:7587241220498628391:2115] self: [6:7587241220498628395:0] peer: [3:7587241217232361601:0] socket: 236 qp: -1
2025-12-24T02:10:56.895238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220498628402:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:56.895244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.895374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.895414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628401:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.896092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814674:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.896104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241217427814674:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.896775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241217427814674:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.896855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241217427814656:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.896870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.896876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241217427814656:2109] [node 4] ICP22 created new session: [1:7587241217427814686:2109]
2025-12-24T02:10:56.896882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241217427814686:2109] [node 4] ICS09 handshake done sender: [1:7587241217427814674:2117] self: [1:7587241217427814659:0] peer: [4:7587241219276070261:0] socket: 259 qp: -1
2025-12-24T02:10:56.896889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241217427814686:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:56.896895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.898908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241220080124013:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(39560781ns) {HandleHandshakeStatus:329(39560037ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(2063ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(32214902ns) {ForwardSessionEventToSession:612(32203764ns) ForwardSessionEventToSession:612(1810ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(826ns) ForwardSessionEventToSession:612(627ns) ForwardSessionEventToSession:612(865ns) ForwardSessionEventToSession:612(485ns) ForwardSessionEventToSession:612(1115ns)}}}
2025-12-24T02:10:56.904358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241217232361600:2099] [node 6] ICP35 event processing took too much time PendingConnection:279(25548066ns) {HandleHandshakeStatus:329(25546647ns) {DropIncomingHandshake:442(4318ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(114ns) ProcessPendingSessionEvents:495(25401110ns) {ForwardSessionEventToSession:612(25386188ns) ForwardSessionEventToSession:612(3249ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(1913ns) ForwardSessionEventToSession:612(1085ns) ForwardSessionEventToSession:612(745ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(1430ns)}}}
2025-12-24T02:10:56.911026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241220080124090:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.911084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241220080124012:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.911767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241220080124093:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.911775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241220080124093:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.913450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241220080124093:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.913539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241220080124078:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.913552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.913560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241220080124078:2063] [node 6] ICP22 created new session: [5:7587241220080124103:2063]
2025-12-24T02:10:56.913566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241220080124103:2063] [node 6] ICS09 handshake done sender: [5:7587241220080124093:2117] self: [5:7587241220080124087:0] peer: [6:7587241220498628405:0] socket: 218 qp: -1
2025-12-24T02:10:56.913572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241220080124103:2063] [node 6] ICS10 traffic start
2025-12-24T02:10:56.913593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.911368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220498628401:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.911400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220498628400:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.911416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241220498628392:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.911447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220498628333:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.911457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220498628349:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.911468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241220498628314:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.919288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446363:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.919304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220754446363:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.920170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241220754446379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50293
2025-12-24T02:10:56.921726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241219276070172:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(58467780ns) {HandleHandshakeStatus:329(58466508ns) {DropIncomingHandshake:442(508ns) DropOutgoingHandshake:457(1731ns) SwitchToState:179(105ns) ProcessPendingSessionEvents:495(24257271ns) {ForwardSessionEventToSession:612(24243610ns) ForwardSessionEventToSession:612(2358ns) ForwardSessionEventToSession:612(1123ns) ForwardSessionEventToSession:612(577ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(1271ns) ForwardSessionEventToSession:612(677ns) ForwardSessionEventToSession:612(528ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(1415ns)}}}
2025-12-24T02:10:56.921754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241219276070172:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:56.921892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241219276070170:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1189) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241219276070260:2120] held: no
2025-12-24T02:10:56.921945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070246:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.921948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241219276070246:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.922284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220754446363:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.918965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.921207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814668:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.921223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241217427814668:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.921908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241217427814668:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.921991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241217427814589:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.922005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.922017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241217427814589:2049] [node 3] ICP22 created new session: [1:7587241217427814689:2049]
2025-12-24T02:10:56.922022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241217427814689:2049] [node 3] ICS09 handshake done sender: [1:7587241217427814668:2116] self: [1:7587241217427814653:0] peer: [3:7587241217232361640:0] socket: 241 qp: -1
2025-12-24T02:10:56.922028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241217427814689:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.922034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.920597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361636:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.920879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241217232361544:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.920940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361639:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.924547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219276070246:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:56.926876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241219276070243:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.926900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.926908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219276070243:2101] [node 6] ICP22 created new session: [4:7587241219276070264:2101]
2025-12-24T02:10:56.926914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219276070264:2101] [node 6] ICS09 handshake done sender: [4:7587241219276070246:2113] self: [4:7587241219276070245:0] peer: [6:7587241220498628404:0] socket: 212 qp: -1
2025-12-24T02:10:56.926921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219276070264:2101] [node 6] ICS10 traffic start
2025-12-24T02:10:56.926944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.927605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446365:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.927620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220754446365:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.931060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220498628400:2120] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.933110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124105:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.932723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220498628349:2074] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.932745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.932751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220498628349:2074] [node 5] ICP22 created new session: [6:7587241220498628412:2074]
2025-12-24T02:10:56.932757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220498628412:2074] [node 5] ICS09 handshake done sender: [6:7587241220498628400:2120] self: [6:7587241220498628405:0] peer: [5:7587241220080124087:0] socket: 214 qp: -1
2025-12-24T02:10:56.932764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220498628412:2074] [node 5] ICS10 traffic start
2025-12-24T02:10:56.932772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.932908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241220498628333:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13322) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241220498628401:2121] held: no
2025-12-24T02:10:56.932980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628411:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.935021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361641:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.935219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241217232361636:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.935293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241217232361544:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.935307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.935313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241217232361544:2048] [node 1] ICP22 created new session: [3:7587241217232361642:2048]
2025-12-24T02:10:56.935318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241217232361642:2048] [node 1] ICS09 handshake done sender: [3:7587241217232361636:2117] self: [3:7587241217232361640:0] peer: [1:7587241217427814653:0] socket: 243 qp: -1
2025-12-24T02:10:56.935325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241217232361642:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.935332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.939416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220754446359:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.939436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:56.939442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220754446359:2108] [node 6] ICP22 created new session: [2:7587241220754446381:2108]
2025-12-24T02:10:56.939449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220754446381:2108] [node 6] ICS09 handshake done sender: [2:7587241220754446363:2114] self: [2:7587241220754446362:0] peer: [6:7587241220498628406:0] socket: 232 qp: -1
2025-12-24T02:10:56.939457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220754446381:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:56.939466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:56.939806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241217232361639:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.939848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241217232361545:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.941193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446375:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.941297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220754446358:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.941424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220754446365:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.939218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241220498628410:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40515
2025-12-24T02:10:56.942882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241217427814679:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.943047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241217427814589:2049] [node 3] ICP18 handshake (actor [1:7587241217427814679:2118]) is too old
2025-12-24T02:10:56.945666Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241220080124078:2063] [node 6] ICP35 event processing took too much time PendingConnection:279(32090302ns) {HandleHandshakeStatus:329(32088970ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(636ns) SwitchToState:179(147ns) ProcessPendingSessionEvents:495(14803404ns) {ForwardSessionEventToSession:612(14785109ns) ForwardSessionEventToSession:612(2481ns) ForwardSessionEventToSession:612(1118ns) ForwardSessionEventToSession:612(707ns) ForwardSessionEventToSession:612(772ns) ForwardSessionEventToSession:612(1016ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(853ns) ForwardSessionEventToSession:612(1174ns)}}}
2025-12-24T02:10:56.946143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070263:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.948043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446369:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.948065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241220754446369:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:56.948414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220498628401:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.948888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241220498628392:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.949018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220498628333:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.949030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.949036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220498628333:2063] [node 4] ICP22 created new session: [6:7587241220498628414:2063]
2025-12-24T02:10:56.949042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220498628414:2063] [node 4] ICS09 handshake done sender: [6:7587241220498628401:2121] self: [6:7587241220498628404:0] peer: [4:7587241219276070245:0] socket: 221 qp: -1
2025-12-24T02:10:56.949049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220498628414:2063] [node 4] ICS10 traffic start
2025-12-24T02:10:56.949058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.949268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241220498628314:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.949277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.949283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241220498628314:2049] [node 2] ICP22 created new session: [6:7587241220498628416:2049]
2025-12-24T02:10:56.949288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241220498628416:2049] [node 2] ICS09 handshake done sender: [6:7587241220498628392:2116] self: [6:7587241220498628406:0] peer: [2:7587241220754446362:0] socket: 237 qp: -1
2025-12-24T02:10:56.949293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241220498628416:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.949299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.950862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124107:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.951046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241220080124078:2063] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:56.951075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124106:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.951331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241220080124090:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.951435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241220080124012:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.951452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.951459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241220080124012:2049] [node 2] ICP22 created new session: [5:7587241220080124108:2049]
2025-12-24T02:10:56.951465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241220080124108:2049] [node 2] ICS09 handshake done sender: [5:7587241220080124090:2116] self: [5:7587241220080124101:0] peer: [2:7587241220754446364:0] socket: 239 qp: -1
2025-12-24T02:10:56.951471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241220080124108:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.951477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.951990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219276070260:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:56.951970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220754446361:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.951994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.952002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220754446361:2109] [node 5] ICP22 created new session: [2:7587241220754446385:2109]
2025-12-24T02:10:56.952007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220754446385:2109] [node 5] ICS09 handshake done sender: [2:7587241220754446365:2115] self: [2:7587241220754446364:0] peer: [5:7587241220080124101:0] socket: 233 qp: -1
2025-12-24T02:10:56.952014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220754446385:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:56.952021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.952193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241220754446379:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.952382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241220754446359:2108] [node 6] ICP18 handshake (actor [2:7587241220754446379:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.952659Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241220080124106:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.952768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241220080124024:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:56.952850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219276070170:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.952868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:56.952877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219276070170:2048] [node 1] ICP22 created new session: [4:7587241219276070268:2048]
2025-12-24T02:10:56.952883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219276070268:2048] [node 1] ICS09 handshake done sender: [4:7587241219276070260:2120] self: [4:7587241219276070261:0] peer: [1:7587241217427814659:0] socket: 260 qp: -1
2025-12-24T02:10:56.952890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219276070268:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:56.952895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:56.953958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241219276070251:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:56.953964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241219276070251:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.954636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241220080124111:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14901
2025-12-24T02:10:56.954835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219276070251:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:56.955007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241220080124106:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:56.955120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241220080124024:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.955134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.955139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241220080124024:2059] [node 4] ICP22 created new session: [5:7587241220080124112:2059]
2025-12-24T02:10:56.955146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241220080124112:2059] [node 4] ICS09 handshake done sender: [5:7587241220080124106:2121] self: [5:7587241220080124110:0] peer: [4:7587241219276070244:0] socket: 271 qp: -1
2025-12-24T02:10:56.955152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241220080124112:2059] [node 4] ICS10 traffic start
2025-12-24T02:10:56.955161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.956062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241219276070242:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.956081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:56.956086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219276070242:2099] [node 5] ICP22 created new session: [4:7587241219276070271:2099]
2025-12-24T02:10:56.956093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219276070271:2099] [node 5] ICS09 handshake done sender: [4:7587241219276070251:2117] self: [4:7587241219276070244:0] peer: [5:7587241220080124110:0] socket: 226 qp: -1
2025-12-24T02:10:56.956098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219276070271:2099] [node 5] ICS10 traffic start
2025-12-24T02:10:56.956103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:56.962955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220754446348:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.962980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:56.962986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220754446348:2105] [node 4] ICP22 created new session: [2:7587241220754446388:2105]
2025-12-24T02:10:56.962993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220754446388:2105] [node 4] ICS09 handshake done sender: [2:7587241220754446358:2112] self: [2:7587241220754446356:0] peer: [4:7587241219276070256:0] socket: 231 qp: -1
2025-12-24T02:10:56.963001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220754446388:2105] [node 4] ICS10 traffic start
2025-12-24T02:10:56.963012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:56.962748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241219276070267:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2612
2025-12-24T02:10:56.965680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241219276070171:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30148) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241219276070253:2118] held: no
2025-12-24T02:10:56.965759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241219276070253:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.965826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241219276070171:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.965842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.965848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241219276070171:2049] [node 2] ICP22 created new session: [4:7587241219276070273:2049]
2025-12-24T02:10:56.965854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241219276070273:2049] [node 2] ICS09 handshake done sender: [4:7587241219276070253:2118] self: [4:7587241219276070256:0] peer: [2:7587241220754446356:0] socket: 235 qp: -1
2025-12-24T02:10:56.965862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241219276070273:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.965869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:56.970860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241220754446369:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:56.971063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241220754446290:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:56.970919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241217232361645:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62292
2025-12-24T02:10:56.971080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:56.971093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241220754446290:2049] [node 3] ICP22 created new session: [2:7587241220754446391:2049]
2025-12-24T02:10:56.971099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241220754446391:2049] [node 3] ICS09 handshake done sender: [2:7587241220754446369:2116] self: [2:7587241220754446357:0] peer: [3:7587241217232361644:0] socket: 258 qp: -1
2025-12-24T02:10:56.971118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241220754446391:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:56.971127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:56.971188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241217232361639:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:56.971296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241217232361545:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:56.971318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:56.971324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241217232361545:2049] [node 2] ICP22 created new session: [3:7587241217232361646:2049]
2025-12-24T02:10:56.971344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241217232361646:2049] [node 2] ICS09 handshake done sender: [3:7587241217232361639:2118] self: [3:7587241217232361644:0] peer: [2:7587241220754446357:0] socket: 265 qp: -1
2025-12-24T02:10:56.971352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241217232361646:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:56.971361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:56.989045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:56.989135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241220754446376:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-12-24T02:10:56.989144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241220754446289:2048] [node 1] ICP30 unregister session Session# [2:7587241220754446376:2048] VirtualId# [2:7587241220754446373:0]
2025-12-24T02:10:56.989151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241220754446376:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.989204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.989230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.991128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:56.991223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241217232361642:2048] [node 1] ICS01 socket: 243 reason# EndOfStream
2025-12-24T02:10:56.991231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241217232361544:2048] [node 1] ICP30 unregister session Session# [3:7587241217232361642:2048] VirtualId# [3:7587241217232361640:0]
2025-12-24T02:10:56.991239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241217232361642:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.991286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.991311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.997273Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:56.997366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220498628397:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-12-24T02:10:56.997375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220498628313:2048] [node 1] ICP30 unregister session Session# [6:7587241220498628397:2048] VirtualId# [6:7587241220498628393:0]
2025-12-24T02:10:56.997381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220498628397:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.997440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.997465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:56.998968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:56.999062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241219276070268:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-12-24T02:10:56.999072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241219276070170:2048] [node 1] ICP30 unregister session Session# [4:7587241219276070268:2048] VirtualId# [4:7587241219276070261:0]
2025-12-24T02:10:56.999079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241219276070268:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:56.999155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:56.999182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.000882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:57.000961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241220080124099:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-12-24T02:10:57.000969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241220080124011:2048] [node 1] ICP30 unregister session Session# [5:7587241220080124099:2048] VirtualId# [5:7587241220080124091:0]
2025-12-24T02:10:57.000976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241220080124099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.001036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.001061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.004460Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:57.004559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241217232361646:2049] [node 2] ICS01 socket: 265 reason# EndOfStream
2025-12-24T02:10:57.004568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241217232361545:2049] [node 2] ICP30 unregister session Session# [3:7587241217232361646:2049] VirtualId# [3:7587241217232361644:0]
2025-12-24T02:10:57.004575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241217232361646:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.004635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.004668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.008293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:57.008378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241219276070273:2049] [node 2] ICS01 socket: 235 reason# EndOfStream
2025-12-24T02:10:57.008387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241219276070171:2049] [node 2] ICP30 unregister session Session# [4:7587241219276070273:2049] VirtualId# [4:7587241219276070256:0]
2025-12-24T02:10:57.008394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241219276070273:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.008445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.008473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.008514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:57.008577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241220080124108:2049] [node 2] ICS01 socket: 239 reason# EndOfStream
2025-12-24T02:10:57.008582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241220080124012:2049] [node 2] ICP30 unregister session Session# [5:7587241220080124108:2049] VirtualId# [5:7587241220080124101:0]
2025-12-24T02:10:57.008586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241220080124108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.008614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.008633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.009549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:57.009630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220498628416:2049] [node 2] ICS01 socket: 237 reason# EndOfStream
2025-12-24T02:10:57.009638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220498628314:2049] [node 2] ICP30 unregister session Session# [6:7587241220498628416:2049] VirtualId# [6:7587241220498628406:0]
2025-12-24T02:10:57.009644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220498628416:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.009697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.009723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.038550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:57.038654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241219276070257:2050] [node 3] ICS07 socket disconnect 207 reason# EndOfStream
2025-12-24T02:10:57.038662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241219276070257:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.038782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.038775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:57.038863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220498628402:2050] [node 3] ICS01 socket: 236 reason# EndOfStream
2025-12-24T02:10:57.038874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220498628315:2050] [node 3] ICP30 unregister session Session# [6:7587241220498628402:2050] VirtualId# [6:7587241220498628395:0]
2025-12-24T02:10:57.038880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220498628402:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.038935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.038961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.038831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:57.038911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241220080124095:2050] [node 3] ICS07 socket disconnect 238 reason# EndOfStream
2025-12-24T02:10:57.038919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241220080124095:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.038971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.038994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241220080124095:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:57.038813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241219276070257:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:57.039460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241220080124013:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.039474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241220080124095:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.039480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241220080124013:2050] [node 3] ICP30 unregister session Session# [5:7587241220080124095:2050] VirtualId# [5:7587241220080124092:0]
2025-12-24T02:10:57.039494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.039499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241220080124013:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.039502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.039429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241219276070172:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.039450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241219276070257:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.039456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241219276070172:2050] [node 3] ICP30 unregister session Session# [4:7587241219276070257:2050] VirtualId# [4:7587241219276070250:0]
2025-12-24T02:10:57.039475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.039481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241219276070172:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.039483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21747) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.048403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241220080124013:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:57.055712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:10:57.055820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241220080124112:2059] [node 4] ICS01 socket: 271 reason# EndOfStream
2025-12-24T02:10:57.055829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241220080124024:2059] [node 4] ICP30 unregister session Session# [5:7587241220080124112:2059] VirtualId# [5:7587241220080124110:0]
2025-12-24T02:10:57.055837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241220080124112:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.055902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:57.055930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:57.058780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:57.058871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241220498628414:2063] [node 4] ICS01 socket: 221 reason# EndOfStream
2025-12-24T02:10:57.058880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241220498628333:2063] [node 4] ICP30 unregister session Session# [6:7587241220498628414:2063] VirtualId# [6:7587241220498628404:0]
2025-12-24T02:10:57.058887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241220498628414:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.058954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:57.058978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:57.098739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222234602222:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.098753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222234602223:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:57.110723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241222050585045:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.110746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241222050585046:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.111026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241222857096866:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.111035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241222857096867:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.111041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241222857096868:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.099985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241221389982174:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.099993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241221389982175:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.099997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241221389982176:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.113535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241220967538059:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.113544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241220967538060:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.113549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241220967538061:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.106777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223826031157:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:57.106788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223826031158:2049] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.126896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096915:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.124712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241221389982227:2099] [node 5] ICP01 ready to work
2025-12-24T02:10:57.126915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223826031223:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:57.128838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241220967538122:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.128932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241220967538123:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.126930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222234602284:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.135335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241222050585110:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.148922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096915:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.154808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241223826031158:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27835203ns) {Configure:104(27832998ns) {ProcessConfigured:135(27499896ns) {StartInitialHandshake:157(27499236ns) {DropHandshakes:470(972ns) {DropIncomingHandshake:442(265ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:477(1157ns) {SwitchToState:179(703ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.155472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223826031230:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55580
2025-12-24T02:10:57.156087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031230:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.160234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223826031225:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:57.161213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223826031226:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:57.163398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241220967538061:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32954036ns) {Configure:104(30013038ns) {ProcessConfigured:135(29705226ns) {StartInitialHandshake:157(29704173ns) {DropHandshakes:470(746ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(442ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.164196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538129:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.170687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241220967538122:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39915372ns) {Configure:104(35765713ns) {ProcessConfigured:135(35498213ns) {StartInitialHandshake:157(35497670ns) {DropHandshakes:470(408ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(943ns) {SwitchToState:179(527ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.174818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241222050585045:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39998263ns) {Configure:104(39997468ns) {ProcessConfigured:135(39597378ns) {StartInitialHandshake:157(39596899ns) {DropHandshakes:470(480ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(422ns) PrepareNewSessionHandshake:477(1446ns) {SwitchToState:179(775ns)}}}}}
2025-12-24T02:10:57.175371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585117:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.175556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585114:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.175631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585115:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.175671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585116:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.175826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241222050585046:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40433400ns) {Configure:104(40432866ns) {ProcessConfigured:135(40164842ns) {StartInitialHandshake:157(40164478ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(85ns) PrepareNewSessionHandshake:477(731ns) {SwitchToState:179(509ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.178978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096942:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.182540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585114:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.182717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222234602287:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.182846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585115:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.183026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241222050585109:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.182859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096941:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.183343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241222857096927:2104] [node 6] ICP01 ready to work
2025-12-24T02:10:57.183915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241222050585112:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.183663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241222857096928:2100] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.185056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223826031236:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55580
2025-12-24T02:10:57.185079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223826031237:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55580
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.183090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222234602288:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.186246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222234602291:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5923
2025-12-24T02:10:57.186261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222234602292:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5923
2025-12-24T02:10:57.186313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222234602293:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5923
2025-12-24T02:10:57.186353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222234602294:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5923
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.186951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241223826031225:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24183763ns) {Configure:104(24183356ns) {ProcessConfigured:135(23844324ns) {StartInitialHandshake:157(23830705ns) {DropHandshakes:470(975ns) {DropIncomingHandshake:442(122ns) DropOutgoingHandshake:457(123ns)} GenerateSessionVirtualId:988(921ns) PrepareNewSessionHandshake:477(752ns) {SwitchToState:179(431ns)}} IncomingHandshake:261(6423ns) {DropIncomingHandshake:442(504ns) PrepareNewSessionHandshake:477(136ns) {SwitchToState:179(72ns)} GenerateSessionVirtualId:988(270ns) IssueIncomingHandshakeReply:193(434ns)}}}}
2025-12-24T02:10:57.187093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585116:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.187137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241222050585045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.186823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.190556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031227:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.190569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223826031227:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.193107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241220967538129:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.193149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241220967538059:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.190828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602292:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.194831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241221389982174:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31989708ns) {Configure:104(31988616ns) {ProcessConfigured:135(31674337ns) {StartInitialHandshake:157(31673711ns) {DropHandshakes:470(749ns) {DropIncomingHandshake:442(112ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(575ns)}}}}}
2025-12-24T02:10:57.198791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241222234602284:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(65362622ns) {Configure:104(65361728ns) {ProcessConfigured:135(64962482ns) {StartInitialHandshake:157(64961890ns) {DropHandshakes:470(541ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:477(845ns) {SwitchToState:179(112ns)}}}}}
2025-12-24T02:10:57.198841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602293:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.199203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538137:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.206788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602291:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.207022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031229:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.207046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223826031229:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.211297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241220967538060:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(82303187ns) {Configure:104(82302579ns) {ProcessConfigured:135(81954339ns) {StartInitialHandshake:157(81953797ns) {DropHandshakes:470(607ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1538ns) {SwitchToState:179(974ns)}}}}}
2025-12-24T02:10:57.210730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241221389982246:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:57.214445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241222857096927:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31029797ns) {Configure:104(31029677ns) {ProcessConfigured:135(30753768ns) {StartInitialHandshake:157(30743305ns) {DropHandshakes:470(193ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(98ns)}} IncomingHandshake:261(3931ns) {DropIncomingHandshake:442(299ns) PrepareNewSessionHandshake:477(133ns) {SwitchToState:179(105ns)} GenerateSessionVirtualId:988(326ns) IssueIncomingHandshakeReply:193(260ns)}}}}
2025-12-24T02:10:57.214522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241222857096928:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30812917ns) {Configure:104(30812665ns) {ProcessConfigured:135(30599698ns) {StartInitialHandshake:157(30599382ns) {DropHandshakes:470(160ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(421ns) {SwitchToState:179(48ns)}}}}}
2025-12-24T02:10:57.214168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241222050585109:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31076841ns) {Configure:104(31076414ns) {ProcessConfigured:135(30818446ns) {StartInitialHandshake:157(30818308ns) {DropHandshakes:470(364ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1038ns) {SwitchToState:179(409ns)}}}}}
2025-12-24T02:10:57.214067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241222234602223:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31268891ns) {Configure:104(31267815ns) {ProcessConfigured:135(30942636ns) {StartInitialHandshake:157(30941955ns) {DropHandshakes:470(719ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1258ns) {SwitchToState:179(517ns)}}}}}
2025-12-24T02:10:57.214261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241222234602222:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31133906ns) {Configure:104(31133525ns) {ProcessConfigured:135(30904032ns) {StartInitialHandshake:157(30903889ns) {DropHandshakes:470(157ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(37ns)} GenerateSessionVirtualId:988(127ns) PrepareNewSessionHandshake:477(580ns) {SwitchToState:179(124ns)}}}}}
2025-12-24T02:10:57.214282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241222234602222:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.215229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031218:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.215242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223826031218:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.218738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223826031227:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.218869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982251:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.219084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223826031218:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.219128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223826031158:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.219142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.219150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223826031158:2049] [node 3] ICP22 created new session: [1:7587241223826031242:2049]
2025-12-24T02:10:57.219159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223826031242:2049] [node 3] ICS09 handshake done sender: [1:7587241223826031227:2111] self: [1:7587241223826031224:0] peer: [3:7587241222050585129:0] socket: 217 qp: -1
2025-12-24T02:10:57.219167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223826031242:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.219175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.218954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222234602308:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5923
2025-12-24T02:10:57.222815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223826031157:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.222841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.222854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223826031157:2048] [node 2] ICP22 created new session: [1:7587241223826031244:2048]
2025-12-24T02:10:57.222866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223826031244:2048] [node 2] ICS09 handshake done sender: [1:7587241223826031218:2107] self: [1:7587241223826031217:0] peer: [2:7587241222234602307:0] socket: 204 qp: -1
2025-12-24T02:10:57.222879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223826031244:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:57.222911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.225481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241220967538123:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(86673262ns) {Configure:104(86672378ns) {ProcessConfigured:135(86398089ns) {StartInitialHandshake:157(86397642ns) {DropHandshakes:470(533ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(801ns)}}}}}
2025-12-24T02:10:57.225602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538140:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.223345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222234602291:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.224268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241222234602222:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.224281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.224287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222234602222:2048] [node 1] ICP22 created new session: [2:7587241222234602309:2048]
2025-12-24T02:10:57.224294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222234602309:2048] [node 1] ICS09 handshake done sender: [2:7587241222234602291:2110] self: [2:7587241222234602307:0] peer: [1:7587241223826031217:0] socket: 208 qp: -1
2025-12-24T02:10:57.224301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222234602309:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.224322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.224110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982252:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.225762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096949:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.230833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223826031229:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.232845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241220967538129:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.232949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241220967538059:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.232968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.232975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241220967538059:2048] [node 1] ICP22 created new session: [4:7587241220967538141:2048]
2025-12-24T02:10:57.232983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241220967538141:2048] [node 1] ICS09 handshake done sender: [4:7587241220967538129:2111] self: [4:7587241220967538136:0] peer: [1:7587241223826031228:0] socket: 219 qp: -1
2025-12-24T02:10:57.232990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241220967538141:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.233010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.233306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223826031223:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.233321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.233327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223826031223:2108] [node 4] ICP22 created new session: [1:7587241223826031249:2108]
2025-12-24T02:10:57.233333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223826031249:2108] [node 4] ICS09 handshake done sender: [1:7587241223826031229:2112] self: [1:7587241223826031228:0] peer: [4:7587241220967538136:0] socket: 218 qp: -1
2025-12-24T02:10:57.233338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223826031249:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:57.233344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.234541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096949:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.234585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241222857096866:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.235163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031238:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.235172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223826031238:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.236170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096951:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.236028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223826031238:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.236356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223826031225:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.236373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.236381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223826031225:2109] [node 5] ICP22 created new session: [1:7587241223826031252:2109]
2025-12-24T02:10:57.236387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223826031252:2109] [node 5] ICS09 handshake done sender: [1:7587241223826031238:2117] self: [1:7587241223826031233:0] peer: [5:7587241222857096950:0] socket: 252 qp: -1
2025-12-24T02:10:57.236392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223826031252:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:57.236398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.236736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241222857096949:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.236838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241222857096866:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.236854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.236859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241222857096866:2048] [node 1] ICP22 created new session: [5:7587241222857096952:2048]
2025-12-24T02:10:57.236864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241222857096952:2048] [node 1] ICS09 handshake done sender: [5:7587241222857096949:2119] self: [5:7587241222857096950:0] peer: [1:7587241223826031233:0] socket: 256 qp: -1
2025-12-24T02:10:57.236869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241222857096952:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.236886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.238801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241221389982252:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.239012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031237:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.238988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241221389982174:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.239129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096942:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.239174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241222857096868:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.242842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585119:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.242858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241222050585119:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.243811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241223826031158:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24669673ns) {HandleHandshakeStatus:329(24668691ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(330ns) SwitchToState:179(218ns) ProcessPendingSessionEvents:495(14825730ns) {ForwardSessionEventToSession:612(14811032ns) ForwardSessionEventToSession:612(1719ns) ForwardSessionEventToSession:612(1046ns) ForwardSessionEventToSession:612(1353ns) ForwardSessionEventToSession:612(592ns) ForwardSessionEventToSession:612(1029ns) ForwardSessionEventToSession:612(470ns) ForwardSessionEventToSession:612(925ns) ForwardSessionEventToSession:612(907ns) ForwardSessionEventToSession:612(1283ns)}}}
2025-12-24T02:10:57.243855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241223826031158:2049] [node 3] ICP18 handshake (actor [1:7587241223826031237:2116]) is too old
2025-12-24T02:10:57.244488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096956:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.244469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585132:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.244766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585133:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.246797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241220967538137:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.246889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241220967538060:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.247060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602302:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.247071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222234602302:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.247071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585117:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.247157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241222050585119:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.247830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241221389982251:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.247864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241221389982175:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.250777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096945:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.250845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241222857096867:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.248016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222234602302:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.248121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602298:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.248124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222234602298:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.248491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222234602284:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.248501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.248508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222234602284:2107] [node 4] ICP22 created new session: [2:7587241222234602313:2107]
2025-12-24T02:10:57.248514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222234602313:2107] [node 4] ICS09 handshake done sender: [2:7587241222234602302:2116] self: [2:7587241222234602286:0] peer: [4:7587241220967538143:0] socket: 244 qp: -1
2025-12-24T02:10:57.248521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222234602313:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.248531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.248928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222234602298:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.249003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222234602288:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.249017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.249024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222234602288:2109] [node 6] ICP22 created new session: [2:7587241222234602316:2109]
2025-12-24T02:10:57.249030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222234602316:2109] [node 6] ICS09 handshake done sender: [2:7587241222234602298:2115] self: [2:7587241222234602297:0] peer: [6:7587241221389982254:0] socket: 240 qp: -1
2025-12-24T02:10:57.249036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222234602316:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.249041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.250942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602294:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.250988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602296:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.250992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222234602296:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.250770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585133:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.250858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241222050585110:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.251044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241222857096942:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.250869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.250875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241222050585110:2108] [node 5] ICP22 created new session: [3:7587241222050585135:2108]
2025-12-24T02:10:57.250882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241222050585135:2108] [node 5] ICS09 handshake done sender: [3:7587241222050585119:2115] self: [3:7587241222050585118:0] peer: [5:7587241222857096955:0] socket: 230 qp: -1
2025-12-24T02:10:57.250887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241222050585135:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.250893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.250906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538144:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.251255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241220967538137:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.254876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241220967538060:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.254895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.254901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241220967538060:2049] [node 2] ICP22 created new session: [4:7587241220967538145:2049]
2025-12-24T02:10:57.254909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241220967538145:2049] [node 2] ICS09 handshake done sender: [4:7587241220967538137:2114] self: [4:7587241220967538143:0] peer: [2:7587241222234602286:0] socket: 245 qp: -1
2025-12-24T02:10:57.254917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241220967538145:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.254935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.254809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096941:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.254752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031235:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.254774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223826031235:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.254843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241222857096928:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.254803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241221389982246:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43954982ns) {Configure:104(43953715ns) {ProcessConfigured:135(43588159ns) {StartInitialHandshake:157(43587055ns) {DropHandshakes:470(1528ns) {DropIncomingHandshake:442(115ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:477(1056ns) {SwitchToState:179(768ns)}}}}}
2025-12-24T02:10:57.258770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241221389982175:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241221389982251:2114] held: no
2025-12-24T02:10:57.258897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241222050585045:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241222050585116:2113] held: no
2025-12-24T02:10:57.258940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241222050585116:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.259009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241222050585045:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.259020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.259026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241222050585045:2048] [node 1] ICP22 created new session: [3:7587241222050585137:2048]
2025-12-24T02:10:57.259032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241222050585137:2048] [node 1] ICS09 handshake done sender: [3:7587241222050585116:2113] self: [3:7587241222050585129:0] peer: [1:7587241223826031224:0] socket: 227 qp: -1
2025-12-24T02:10:57.259040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241222050585137:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.259048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.259241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241222050585046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.261340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241222050585046:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241222050585133:2120] held: no
2025-12-24T02:10:57.261434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241222050585141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27135
2025-12-24T02:10:57.261666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241222050585133:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.259374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222234602305:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.259385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222234602305:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.260030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222234602305:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.260095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222234602223:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.260107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.260112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222234602223:2049] [node 3] ICP22 created new session: [2:7587241222234602320:2049]
2025-12-24T02:10:57.260118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222234602320:2049] [node 3] ICS09 handshake done sender: [2:7587241222234602305:2117] self: [2:7587241222234602289:0] peer: [3:7587241222050585140:0] socket: 254 qp: -1
2025-12-24T02:10:57.260124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222234602320:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.260130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.261749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241222050585046:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.261763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.261769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241222050585046:2049] [node 2] ICP22 created new session: [3:7587241222050585143:2049]
2025-12-24T02:10:57.261775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241222050585143:2049] [node 2] ICS09 handshake done sender: [3:7587241222050585133:2120] self: [3:7587241222050585140:0] peer: [2:7587241222234602289:0] socket: 261 qp: -1
2025-12-24T02:10:57.261780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241222050585143:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.261786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.267386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241220967538134:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.267402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241220967538134:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.268067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538147:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.270185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222234602296:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.270302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222234602287:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.270330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.270340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222234602287:2108] [node 5] ICP22 created new session: [2:7587241222234602323:2108]
2025-12-24T02:10:57.270347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222234602323:2108] [node 5] ICS09 handshake done sender: [2:7587241222234602296:2114] self: [2:7587241222234602295:0] peer: [5:7587241222857096957:0] socket: 239 qp: -1
2025-12-24T02:10:57.270353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222234602323:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.270363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.270882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223826031254:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55580
2025-12-24T02:10:57.272285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223826031235:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.272390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223826031226:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.272401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.272410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223826031226:2110] [node 6] ICP22 created new session: [1:7587241223826031256:2110]
2025-12-24T02:10:57.272417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223826031256:2110] [node 6] ICS09 handshake done sender: [1:7587241223826031235:2114] self: [1:7587241223826031234:0] peer: [6:7587241221389982253:0] socket: 251 qp: -1
2025-12-24T02:10:57.272423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223826031256:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:57.272432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.272362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241222857096868:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.272392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.272408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241222857096868:2050] [node 3] ICP22 created new session: [5:7587241222857096961:2050]
2025-12-24T02:10:57.272415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241222857096961:2050] [node 3] ICS09 handshake done sender: [5:7587241222857096942:2115] self: [5:7587241222857096955:0] peer: [3:7587241222050585118:0] socket: 233 qp: -1
2025-12-24T02:10:57.272423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241222857096961:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.272433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.270323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241220967538134:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.270457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241220967538122:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.270465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.270470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241220967538122:2108] [node 5] ICP22 created new session: [4:7587241220967538150:2108]
2025-12-24T02:10:57.270475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241220967538150:2108] [node 5] ICS09 handshake done sender: [4:7587241220967538134:2113] self: [4:7587241220967538126:0] peer: [5:7587241222857096958:0] socket: 223 qp: -1
2025-12-24T02:10:57.270479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241220967538150:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.270485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.274919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982257:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.274996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982260:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.275001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982258:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.275013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982261:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.275068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982259:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.275120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241222857096867:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241222857096945:2116] held: no
2025-12-24T02:10:57.275216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096960:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.275316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241221389982259:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.275333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241221389982252:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.275352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241221389982176:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.275390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241221389982174:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.275400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.275408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241221389982174:2048] [node 1] ICP22 created new session: [6:7587241221389982263:2048]
2025-12-24T02:10:57.275415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241221389982263:2048] [node 1] ICS09 handshake done sender: [6:7587241221389982252:2115] self: [6:7587241221389982253:0] peer: [1:7587241223826031234:0] socket: 253 qp: -1
2025-12-24T02:10:57.275422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241221389982263:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.275449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.275836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585127:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.275852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241222050585127:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.276705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223826031254:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.276907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241223826031223:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.278890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241222050585127:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.278966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241222050585112:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.278995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.279002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241222050585112:2109] [node 6] ICP22 created new session: [3:7587241222050585146:2109]
2025-12-24T02:10:57.279008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241222050585146:2109] [node 6] ICS09 handshake done sender: [3:7587241222050585127:2117] self: [3:7587241222050585126:0] peer: [6:7587241221389982262:0] socket: 225 qp: -1
2025-12-24T02:10:57.279014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241222050585146:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.279024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.279301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241220967538147:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.279348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241220967538061:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.280492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241220967538060:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25577518ns) {HandleHandshakeStatus:329(25576750ns) {DropIncomingHandshake:442(161ns) DropOutgoingHandshake:457(1961ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(8110727ns) {ForwardSessionEventToSession:612(8098499ns) ForwardSessionEventToSession:612(1277ns) ForwardSessionEventToSession:612(1102ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(822ns) ForwardSessionEventToSession:612(1390ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(496ns) ForwardSessionEventToSession:612(652ns)}}}
2025-12-24T02:10:57.285376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241222050585130:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.285396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241222050585130:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.287019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241222857096959:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58975
2025-12-24T02:10:57.287185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241223826031226:2110] [node 6] ICP18 handshake (actor [1:7587241223826031254:2118]) is too old
2025-12-24T02:10:57.290820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241222857096941:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.291013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241222857096928:2100] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.291037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.291047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241222857096928:2100] [node 4] ICP22 created new session: [5:7587241222857096964:2100]
2025-12-24T02:10:57.291055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241222857096964:2100] [node 4] ICS09 handshake done sender: [5:7587241222857096941:2114] self: [5:7587241222857096958:0] peer: [4:7587241220967538126:0] socket: 224 qp: -1
2025-12-24T02:10:57.291066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241222857096964:2100] [node 4] ICS10 traffic start
2025-12-24T02:10:57.291077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.291984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.294824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241222857096868:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(43629316ns) {HandleHandshakeStatus:329(43628741ns) {DropIncomingHandshake:442(798ns) DropOutgoingHandshake:457(2009ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(11745878ns) {ForwardSessionEventToSession:612(11731287ns) ForwardSessionEventToSession:612(1918ns) ForwardSessionEventToSession:612(1052ns) ForwardSessionEventToSession:612(638ns) ForwardSessionEventToSession:612(562ns) ForwardSessionEventToSession:612(1212ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(666ns) ForwardSessionEventToSession:612(503ns) ForwardSessionEventToSession:612(1509ns)}}}
2025-12-24T02:10:57.294850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241222857096945:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.295852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241221389982174:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20426141ns) {HandleHandshakeStatus:329(20425652ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(1050ns) SwitchToState:179(38ns) ProcessPendingSessionEvents:495(10547022ns) {ForwardSessionEventToSession:612(10535660ns) ForwardSessionEventToSession:612(1463ns) ForwardSessionEventToSession:612(930ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(813ns) ForwardSessionEventToSession:612(829ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(1348ns)}}}
2025-12-24T02:10:57.295951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241221389982257:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.296093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241221389982251:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.296158Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241221389982176:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241221389982259:2119] held: no
2025-12-24T02:10:57.296243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241221389982227:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.296275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982265:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.296374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241221389982175:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.296388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.296395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241221389982175:2049] [node 2] ICP22 created new session: [6:7587241221389982268:2049]
2025-12-24T02:10:57.296401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241221389982268:2049] [node 2] ICS09 handshake done sender: [6:7587241221389982251:2114] self: [6:7587241221389982254:0] peer: [2:7587241222234602297:0] socket: 243 qp: -1
2025-12-24T02:10:57.296411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241221389982268:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.296419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.297400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241222857096868:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.297622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241222050585130:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.297710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241222050585109:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.297729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.297736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241222050585109:2107] [node 4] ICP22 created new session: [3:7587241222050585149:2107]
2025-12-24T02:10:57.297741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241222050585149:2107] [node 4] ICS09 handshake done sender: [3:7587241222050585130:2118] self: [3:7587241222050585125:0] peer: [4:7587241220967538152:0] socket: 231 qp: -1
2025-12-24T02:10:57.297747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241222050585149:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.297767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.297923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241222857096946:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.297936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241222857096946:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.298893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241221389982260:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.298939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982270:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.298981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241221389982246:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.299208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241221389982259:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.298436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241222857096867:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.298454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.298461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241222857096867:2049] [node 2] ICP22 created new session: [5:7587241222857096966:2049]
2025-12-24T02:10:57.298468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241222857096966:2049] [node 2] ICS09 handshake done sender: [5:7587241222857096945:2116] self: [5:7587241222857096957:0] peer: [2:7587241222234602295:0] socket: 241 qp: -1
2025-12-24T02:10:57.298475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241222857096966:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.298483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.302819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241221389982176:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.302776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241220967538061:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241220967538147:2119] held: no
2025-12-24T02:10:57.302839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.302847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241221389982176:2050] [node 3] ICP22 created new session: [6:7587241221389982272:2050]
2025-12-24T02:10:57.302853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241221389982272:2050] [node 3] ICS09 handshake done sender: [6:7587241221389982259:2119] self: [6:7587241221389982262:0] peer: [3:7587241222050585126:0] socket: 271 qp: -1
2025-12-24T02:10:57.302862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241221389982272:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.302870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.302961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241222857096946:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.303143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241222857096927:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.303161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.303168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241222857096927:2104] [node 6] ICP22 created new session: [5:7587241222857096969:2104]
2025-12-24T02:10:57.303173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241222857096969:2104] [node 6] ICS09 handshake done sender: [5:7587241222857096946:2117] self: [5:7587241222857096943:0] peer: [6:7587241221389982266:0] socket: 259 qp: -1
2025-12-24T02:10:57.303179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241222857096969:2104] [node 6] ICS10 traffic start
2025-12-24T02:10:57.303185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.306730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241220967538139:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.306757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241220967538139:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.308881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241221389982257:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.308956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241221389982227:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24550) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241221389982257:2117] held: no
2025-12-24T02:10:57.308975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241221389982227:2099] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.308985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.308991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241221389982227:2099] [node 5] ICP22 created new session: [6:7587241221389982274:2099]
2025-12-24T02:10:57.308998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241221389982274:2099] [node 5] ICS09 handshake done sender: [6:7587241221389982257:2117] self: [6:7587241221389982266:0] peer: [5:7587241222857096943:0] socket: 260 qp: -1
2025-12-24T02:10:57.309006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241221389982274:2099] [node 5] ICS10 traffic start
2025-12-24T02:10:57.309014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.316500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538154:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.316541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241220967538155:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42527
2025-12-24T02:10:57.317472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241220967538139:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.317548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241220967538123:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.317562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.317572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241220967538123:2109] [node 6] ICP22 created new session: [4:7587241220967538157:2109]
2025-12-24T02:10:57.317579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241220967538157:2109] [node 6] ICS09 handshake done sender: [4:7587241220967538139:2116] self: [4:7587241220967538127:0] peer: [6:7587241221389982271:0] socket: 268 qp: -1
2025-12-24T02:10:57.317586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241220967538157:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.317595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.317939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241220967538147:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.318013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241220967538061:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.318023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.318030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241220967538061:2050] [node 3] ICP22 created new session: [4:7587241220967538159:2050]
2025-12-24T02:10:57.318036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241220967538159:2050] [node 3] ICS09 handshake done sender: [4:7587241220967538147:2119] self: [4:7587241220967538152:0] peer: [3:7587241222050585125:0] socket: 266 qp: -1
2025-12-24T02:10:57.318041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241220967538159:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.318048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.319195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241221389982276:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16912
2025-12-24T02:10:57.319395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241221389982260:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.319504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241221389982246:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.319516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.319521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241221389982246:2108] [node 4] ICP22 created new session: [6:7587241221389982277:2108]
2025-12-24T02:10:57.319527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241221389982277:2108] [node 4] ICS09 handshake done sender: [6:7587241221389982260:2120] self: [6:7587241221389982271:0] peer: [4:7587241220967538127:0] socket: 272 qp: -1
2025-12-24T02:10:57.319532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241221389982277:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:57.319540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.323075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241220967538123:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.330598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:57.330713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241222234602309:2048] [node 1] ICS01 socket: 208 reason# EndOfStream
2025-12-24T02:10:57.330722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241222234602222:2048] [node 1] ICP30 unregister session Session# [2:7587241222234602309:2048] VirtualId# [2:7587241222234602307:0]
2025-12-24T02:10:57.330729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241222234602309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.330774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.330803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.332205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:57.332292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241220967538141:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:10:57.332299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241220967538059:2048] [node 1] ICP30 unregister session Session# [4:7587241220967538141:2048] VirtualId# [4:7587241220967538136:0]
2025-12-24T02:10:57.332305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241220967538141:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.332357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.332380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.332413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:57.332480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241222050585137:2048] [node 1] ICS01 socket: 227 reason# EndOfStream
2025-12-24T02:10:57.332484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241222050585045:2048] [node 1] ICP30 unregister session Session# [3:7587241222050585137:2048] VirtualId# [3:7587241222050585129:0]
2025-12-24T02:10:57.332489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241222050585137:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.332514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.332541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.334732Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:57.334826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241221389982263:2048] [node 1] ICS07 socket disconnect 253 reason# EndOfStream
2025-12-24T02:10:57.334831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241221389982263:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.334884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.334905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241221389982263:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:57.334743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:57.334829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241222857096952:2048] [node 1] ICS01 socket: 256 reason# EndOfStream
2025-12-24T02:10:57.334836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241222857096866:2048] [node 1] ICP30 unregister session Session# [5:7587241222857096952:2048] VirtualId# [5:7587241222857096950:0]
2025-12-24T02:10:57.334842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241222857096952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.334895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.334919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.338869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241221389982174:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.338883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241221389982263:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.338888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241221389982174:2048] [node 1] ICP30 unregister session Session# [6:7587241221389982263:2048] VirtualId# [6:7587241221389982253:0]
2025-12-24T02:10:57.338911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.338918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241221389982174:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.338921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.350765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241221389982174:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:57.350774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:57.350871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241222050585143:2049] [node 2] ICS01 socket: 261 reason# EndOfStream
2025-12-24T02:10:57.350880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241222050585046:2049] [node 2] ICP30 unregister session Session# [3:7587241222050585143:2049] VirtualId# [3:7587241222050585140:0]
2025-12-24T02:10:57.350887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241222050585143:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.350937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.350962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.363026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:57.363128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241221389982268:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-12-24T02:10:57.363135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241221389982175:2049] [node 2] ICP30 unregister session Session# [6:7587241221389982268:2049] VirtualId# [6:7587241221389982254:0]
2025-12-24T02:10:57.362824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:57.363141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241221389982268:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.363188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.363215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.362918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241220967538145:2049] [node 2] ICS07 socket disconnect 245 reason# EndOfStream
2025-12-24T02:10:57.362924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241220967538145:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.362989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.363017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241220967538145:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:57.365086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241220967538060:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.365101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241220967538145:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.365106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241220967538060:2049] [node 2] ICP30 unregister session Session# [4:7587241220967538145:2049] VirtualId# [4:7587241220967538143:0]
2025-12-24T02:10:57.365120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.365126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241220967538060:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.365128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.368995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:57.369090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241222857096966:2049] [node 2] ICS01 socket: 241 reason# EndOfStream
2025-12-24T02:10:57.369100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241222857096867:2049] [node 2] ICP30 unregister session Session# [5:7587241222857096966:2049] VirtualId# [5:7587241222857096957:0]
2025-12-24T02:10:57.369108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241222857096966:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.369177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.369205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.379327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241220967538060:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:57.380874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:57.380955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241221389982272:2050] [node 3] ICS01 socket: 271 reason# EndOfStream
2025-12-24T02:10:57.380962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241221389982176:2050] [node 3] ICP30 unregister session Session# [6:7587241221389982272:2050] VirtualId# [6:7587241221389982262:0]
2025-12-24T02:10:57.380966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241221389982272:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.381006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.381027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.383223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:57.383303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241222857096961:2050] [node 3] ICS07 socket disconnect 233 reason# EndOfStream
2025-12-24T02:10:57.383310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241222857096961:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.383366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.383389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241222857096961:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:57.383754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241222857096868:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65385) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.383777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241222857096961:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.383782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241222857096868:2050] [node 3] ICP30 unregister session Session# [5:7587241222857096961:2050] VirtualId# [5:7587241222857096955:0]
2025-12-24T02:10:57.383802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.383814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241222857096868:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.383816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65385) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.397298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241222857096868:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:10:57.398543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:57.398636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241221389982277:2108] [node 4] ICS01 socket: 272 reason# EndOfStream
2025-12-24T02:10:57.398644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241221389982246:2108] [node 4] ICP30 unregister session Session# [6:7587241221389982277:2108] VirtualId# [6:7587241221389982271:0]
2025-12-24T02:10:57.398650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241221389982277:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.398708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:57.398733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:57.445440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223161413744:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:57.445453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223161413745:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:57.447069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222443262605:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.447091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222443262606:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:57.448337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223825751690:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.448360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223825751691:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.477138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223460390357:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.477148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223460390358:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.477153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223460390359:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.471884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225038894272:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.471893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225038894273:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.471897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225038894274:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.474838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241224766001407:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.474849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241224766001408:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.474854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241224766001409:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.517520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223161413806:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.518233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390418:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.516668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225038894332:2106] [node 5] ICP01 ready to work
2025-12-24T02:10:57.518810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223161413815:2110] [node 5] ICP01 ready to work
2025-12-24T02:10:57.522751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241223161413816:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.523957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001468:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.526092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222443262674:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.536751Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241223161413815:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17885835ns) {Configure:104(17885361ns) {ProcessConfigured:135(17619505ns) {StartInitialHandshake:157(17619332ns) {DropHandshakes:470(787ns) {DropIncomingHandshake:442(270ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(464ns) {SwitchToState:179(211ns)}}}}}
2025-12-24T02:10:57.537104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222443262666:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.540059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223161413823:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49001
2025-12-24T02:10:57.541284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413823:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.548067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001468:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.551440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751754:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
2025-12-24T02:10:57.551536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751755:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
2025-12-24T02:10:57.551580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751756:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.551859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894345:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.554933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222443262682:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23889
2025-12-24T02:10:57.554959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241223460390357:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17069136ns) {Configure:104(17068500ns) {ProcessConfigured:135(16757355ns) {StartInitialHandshake:157(16756937ns) {DropHandshakes:470(445ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:477(1306ns) {SwitchToState:179(970ns)}}}}}
2025-12-24T02:10:57.556138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241223460390358:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17911583ns) {Configure:104(17910648ns) {ProcessConfigured:135(17727945ns) {StartInitialHandshake:157(17727621ns) {DropHandshakes:470(214ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(38ns)} GenerateSessionVirtualId:988(39ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(378ns)}}}}}
2025-12-24T02:10:57.552195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241223825751690:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17175815ns) {Configure:104(17174843ns) {ProcessConfigured:135(16776806ns) {StartInitialHandshake:157(16776494ns) {DropHandshakes:470(1038ns) {DropIncomingHandshake:442(115ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(652ns) {SwitchToState:179(273ns)}}}}}
2025-12-24T02:10:57.552272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751754:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.556372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241225038894272:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16294722ns) {Configure:104(16294194ns) {ProcessConfigured:135(15851322ns) {StartInitialHandshake:157(15851045ns) {DropHandshakes:470(744ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(1122ns) {SwitchToState:179(449ns)}}}}}
2025-12-24T02:10:57.557728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241222443262671:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.557696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751755:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.558188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225038894342:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:57.558420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225038894345:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.558449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225038894272:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.562970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390418:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.563006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223460390357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.563241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241224766001472:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:57.565793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413819:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.565819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223161413819:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.579098Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241222443262671:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21286526ns) {Configure:104(21285472ns) {ProcessConfigured:135(20962861ns) {StartInitialHandshake:157(20962117ns) {DropHandshakes:470(1569ns) {DropIncomingHandshake:442(185ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(2194ns) {SwitchToState:179(1701ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.578986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223161413824:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49001
2025-12-24T02:10:57.579202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223161413826:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49001
2025-12-24T02:10:57.579310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241224766001407:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21432988ns) {Configure:104(21431234ns) {ProcessConfigured:135(21008671ns) {StartInitialHandshake:157(21008329ns) {DropHandshakes:470(722ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(608ns) {SwitchToState:179(400ns)}}}}}
2025-12-24T02:10:57.580835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241224766001408:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22914789ns) {Configure:104(22913776ns) {ProcessConfigured:135(22269701ns) {StartInitialHandshake:157(22269576ns) {DropHandshakes:470(171ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(93ns) PrepareNewSessionHandshake:477(617ns) {SwitchToState:179(80ns)}}}}}
2025-12-24T02:10:57.580708Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241225038894342:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22457174ns) {Configure:104(22456772ns) {ProcessConfigured:135(22163427ns) {StartInitialHandshake:157(22163217ns) {DropHandshakes:470(274ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(1381ns) {SwitchToState:179(1019ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.583148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223161413827:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49001
2025-12-24T02:10:57.583051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241223825751690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.583170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751756:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.583187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241223825751691:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.583741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413808:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.583751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223161413808:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.584455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413826:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.584528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223161413808:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.584744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223161413745:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.584762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.584769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223161413745:2049] [node 3] ICP22 created new session: [1:7587241223161413831:2049]
2025-12-24T02:10:57.584776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223161413831:2049] [node 3] ICS09 handshake done sender: [1:7587241223161413808:2108] self: [1:7587241223161413807:0] peer: [3:7587241223825751765:0] socket: 212 qp: -1
2025-12-24T02:10:57.584782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223161413831:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.584799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.585118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001481:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.586395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001481:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.586433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241224766001407:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.586409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751767:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
2025-12-24T02:10:57.586560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413821:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.586567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223161413821:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.586874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262682:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.591022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262678:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.590934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413811:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.591044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222443262678:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.590958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223161413811:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.590975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241223161413744:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20394402ns) {Configure:104(20392948ns) {ProcessConfigured:135(20012775ns) {StartInitialHandshake:157(20012497ns) {DropHandshakes:470(941ns) {DropIncomingHandshake:442(127ns) DropOutgoingHandshake:457(118ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(387ns) {SwitchToState:179(166ns)}}}}}
2025-12-24T02:10:57.591148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223825751764:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.591200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223825751755:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.591001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894343:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.591307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225038894343:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.594925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241224766001472:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31589492ns) {Configure:104(31589188ns) {ProcessConfigured:135(31301877ns) {StartInitialHandshake:157(31283480ns) {DropHandshakes:470(751ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(473ns) PrepareNewSessionHandshake:477(1229ns) {SwitchToState:179(749ns)}} IncomingHandshake:261(7658ns) {DropIncomingHandshake:442(115ns) PrepareNewSessionHandshake:477(124ns) {SwitchToState:179(76ns)} GenerateSessionVirtualId:988(632ns) IssueIncomingHandshakeReply:193(480ns)}}}}
2025-12-24T02:10:57.594738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241223161413825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49001
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.596214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241223825751690:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.596237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.596246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223825751690:2048] [node 1] ICP22 created new session: [3:7587241223825751769:2048]
2025-12-24T02:10:57.596253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223825751769:2048] [node 1] ICS09 handshake done sender: [3:7587241223825751755:2111] self: [3:7587241223825751765:0] peer: [1:7587241223161413807:0] socket: 227 qp: -1
2025-12-24T02:10:57.596260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223825751769:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.596295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.595369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894352:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.598880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223161413819:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.598993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241224766001409:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40499998ns) {Configure:104(32671574ns) {ProcessConfigured:135(32367278ns) {StartInitialHandshake:157(32366580ns) {DropHandshakes:470(1071ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(544ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(1263ns) {SwitchToState:179(642ns)}}}}}
2025-12-24T02:10:57.602567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262676:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.602580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222443262676:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.603909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223161413821:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.603650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223460390423:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:57.604005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223161413815:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.604019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.604026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223161413815:2110] [node 5] ICP22 created new session: [1:7587241223161413836:2110]
2025-12-24T02:10:57.604033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223161413836:2110] [node 5] ICS09 handshake done sender: [1:7587241223161413821:2113] self: [1:7587241223161413817:0] peer: [5:7587241224766001482:0] socket: 246 qp: -1
2025-12-24T02:10:57.604040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223161413836:2110] [node 5] ICS10 traffic start
2025-12-24T02:10:57.604048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.604186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222443262676:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.602271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241225038894273:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31343983ns) {Configure:104(31343200ns) {ProcessConfigured:135(31022955ns) {StartInitialHandshake:157(31022413ns) {DropHandshakes:470(685ns) {DropIncomingHandshake:442(127ns) DropOutgoingHandshake:457(151ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1099ns) {SwitchToState:179(715ns)}}}}}
2025-12-24T02:10:57.602333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225038894273:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.606738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222443262674:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.606760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.606785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222443262674:2109] [node 6] ICP22 created new session: [2:7587241222443262689:2109]
2025-12-24T02:10:57.606792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222443262689:2109] [node 6] ICS09 handshake done sender: [2:7587241222443262676:2111] self: [2:7587241222443262675:0] peer: [6:7587241225038894354:0] socket: 221 qp: -1
2025-12-24T02:10:57.606800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222443262689:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.606822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.603740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241224766001471:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.613117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222443262678:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.613197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222443262606:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.611169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223161413816:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.609087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894355:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.611191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.611197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223161413816:2111] [node 6] ICP22 created new session: [1:7587241223161413840:2111]
2025-12-24T02:10:57.609170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894356:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.611202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223161413840:2111] [node 6] ICS09 handshake done sender: [1:7587241223161413819:2112] self: [1:7587241223161413818:0] peer: [6:7587241225038894348:0] socket: 225 qp: -1
2025-12-24T02:10:57.609320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225038894343:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.609450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225038894345:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.609513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225038894273:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.609525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.611206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223161413840:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:57.609532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225038894273:2049] [node 2] ICP22 created new session: [6:7587241225038894357:2049]
2025-12-24T02:10:57.611224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.609539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225038894357:2049] [node 2] ICS09 handshake done sender: [6:7587241225038894343:2112] self: [6:7587241225038894354:0] peer: [2:7587241222443262675:0] socket: 223 qp: -1
2025-12-24T02:10:57.609546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225038894357:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.609554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.609988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225038894272:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.609999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.610004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225038894272:2048] [node 1] ICP22 created new session: [6:7587241225038894360:2048]
2025-12-24T02:10:57.610009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225038894360:2048] [node 1] ICS09 handshake done sender: [6:7587241225038894345:2113] self: [6:7587241225038894348:0] peer: [1:7587241223161413818:0] socket: 226 qp: -1
2025-12-24T02:10:57.610014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225038894360:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.610019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.613210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.613216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222443262606:2049] [node 3] ICP22 created new session: [2:7587241222443262692:2049]
2025-12-24T02:10:57.613221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222443262692:2049] [node 3] ICS09 handshake done sender: [2:7587241222443262678:2112] self: [2:7587241222443262670:0] peer: [3:7587241223825751766:0] socket: 222 qp: -1
2025-12-24T02:10:57.613226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222443262692:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.613231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.614950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223161413811:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.615054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223161413806:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.615067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.615075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223161413806:2107] [node 4] ICP22 created new session: [1:7587241223161413842:2107]
2025-12-24T02:10:57.615080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223161413842:2107] [node 4] ICS09 handshake done sender: [1:7587241223161413811:2109] self: [1:7587241223161413809:0] peer: [4:7587241223460390430:0] socket: 213 qp: -1
2025-12-24T02:10:57.615087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223161413842:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.615094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.615700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751768:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
2025-12-24T02:10:57.615727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241223825751691:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20829) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241223825751756:2112] held: no
2025-12-24T02:10:57.615773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751772:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.617569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222443262695:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23889
2025-12-24T02:10:57.617586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222443262696:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.622397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894362:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.622998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223825751756:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.623278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223825751763:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.623696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223825751762:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.623645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223460390429:2110] [node 6] ICP01 ready to work
2025-12-24T02:10:57.626732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222443262687:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23889
2025-12-24T02:10:57.626875Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241222443262674:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20112709ns) {HandleHandshakeStatus:329(20111971ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(300ns) SwitchToState:179(180ns) ProcessPendingSessionEvents:495(52273ns) {ForwardSessionEventToSession:612(39147ns) ForwardSessionEventToSession:612(1706ns) ForwardSessionEventToSession:612(1008ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(1036ns) ForwardSessionEventToSession:612(628ns) ForwardSessionEventToSession:612(624ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(913ns)}}}
2025-12-24T02:10:57.626770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241223460390359:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(52599803ns) {Configure:104(48250926ns) {ProcessConfigured:135(47961594ns) {StartInitialHandshake:157(47961152ns) {DropHandshakes:470(508ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(865ns) {SwitchToState:179(768ns)}}}}}
2025-12-24T02:10:57.630788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225038894362:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.630841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225038894274:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.630807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241223825751691:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.630831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.630841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223825751691:2049] [node 2] ICP22 created new session: [3:7587241223825751777:2049]
2025-12-24T02:10:57.630849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223825751777:2049] [node 2] ICS09 handshake done sender: [3:7587241223825751756:2112] self: [3:7587241223825751766:0] peer: [2:7587241222443262670:0] socket: 228 qp: -1
2025-12-24T02:10:57.630858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223825751777:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.630872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.631054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751773:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.631059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223825751773:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.631726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001488:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.632124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241224766001481:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.632172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241224766001407:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.632181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.632187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241224766001407:2048] [node 1] ICP22 created new session: [5:7587241224766001490:2048]
2025-12-24T02:10:57.632193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241224766001490:2048] [node 1] ICS09 handshake done sender: [5:7587241224766001481:2113] self: [5:7587241224766001482:0] peer: [1:7587241223161413817:0] socket: 247 qp: -1
2025-12-24T02:10:57.632198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241224766001490:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.632215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.632383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001489:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.634888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894364:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.635049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223825751773:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.635193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223825751764:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.635210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.635216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223825751764:2109] [node 6] ICP22 created new session: [3:7587241223825751780:2109]
2025-12-24T02:10:57.635222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223825751780:2109] [node 6] ICS09 handshake done sender: [3:7587241223825751773:2118] self: [3:7587241223825751771:0] peer: [6:7587241225038894363:0] socket: 229 qp: -1
2025-12-24T02:10:57.635228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223825751780:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.635235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.636564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262687:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.635964Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241224766001471:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32141534ns) {Configure:104(32140618ns) {ProcessConfigured:135(31859475ns) {StartInitialHandshake:157(31858927ns) {DropHandshakes:470(642ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(432ns)}}}}}
2025-12-24T02:10:57.636011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001489:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.636042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241224766001408:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.636160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241223161413815:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(32130078ns) {HandleHandshakeStatus:329(32129214ns) {DropIncomingHandshake:442(83ns) DropOutgoingHandshake:457(196ns) SwitchToState:179(110ns) ProcessPendingSessionEvents:495(32043272ns) {ForwardSessionEventToSession:612(32033214ns) ForwardSessionEventToSession:612(1484ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(1128ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(438ns) ForwardSessionEventToSession:612(479ns) ForwardSessionEventToSession:612(852ns) ForwardSessionEventToSession:612(935ns)}}}
2025-12-24T02:10:57.636181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241223161413745:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.642452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225038894274:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241225038894362:2120] held: no
2025-12-24T02:10:57.642768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751768:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.643053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262695:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.646220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225038894352:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.646278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225038894332:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.647074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413825:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.647130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225038894362:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.650831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262696:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.650896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241222443262605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.650873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225038894274:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.650892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.650901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225038894274:2050] [node 3] ICP22 created new session: [6:7587241225038894366:2050]
2025-12-24T02:10:57.650908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225038894366:2050] [node 3] ICS09 handshake done sender: [6:7587241225038894362:2120] self: [6:7587241225038894363:0] peer: [3:7587241223825751771:0] socket: 264 qp: -1
2025-12-24T02:10:57.650917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225038894366:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.650931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.651538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241223161413806:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(36445322ns) {HandleHandshakeStatus:329(36444766ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(130ns) SwitchToState:179(182ns) ProcessPendingSessionEvents:495(36351981ns) {ForwardSessionEventToSession:612(36338923ns) ForwardSessionEventToSession:612(1922ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(1640ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(488ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(1215ns)}}}
2025-12-24T02:10:57.651596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241223161413806:2107] [node 4] ICP18 handshake (actor [1:7587241223161413825:2117]) is too old
2025-12-24T02:10:57.651787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262684:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.651804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222443262684:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.654896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001483:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.654912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241224766001483:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.650824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241223825751763:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27462471ns) {Configure:104(27460860ns) {ProcessConfigured:135(27140238ns) {StartInitialHandshake:157(27139104ns) {DropHandshakes:470(726ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(132ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(1789ns) {SwitchToState:179(854ns)}}}}}
2025-12-24T02:10:57.655517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241223161413822:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.655528Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241223161413822:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.656285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241223161413822:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.656384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241223161413744:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.656407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.656416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241223161413744:2048] [node 2] ICP22 created new session: [1:7587241223161413845:2048]
2025-12-24T02:10:57.656423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241223161413845:2048] [node 2] ICS09 handshake done sender: [1:7587241223161413822:2114] self: [1:7587241223161413804:0] peer: [2:7587241222443262699:0] socket: 260 qp: -1
2025-12-24T02:10:57.656430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241223161413845:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:57.656437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.656841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241222443262700:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23889
2025-12-24T02:10:57.658815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241224766001483:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.662098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241224766001472:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.662132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.662139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241224766001472:2110] [node 6] ICP22 created new session: [5:7587241224766001495:2110]
2025-12-24T02:10:57.662146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241224766001495:2110] [node 6] ICS09 handshake done sender: [5:7587241224766001483:2114] self: [5:7587241224766001475:0] peer: [6:7587241225038894365:0] socket: 250 qp: -1
2025-12-24T02:10:57.662153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241224766001495:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:57.662161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.661919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241223460390357:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27071) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241223460390418:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.662745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241222443262605:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27071) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241222443262696:2118] held: no
2025-12-24T02:10:57.662790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222443262696:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.660832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241223825751762:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37080581ns) {Configure:104(37079963ns) {ProcessConfigured:135(9662661ns) {StartInitialHandshake:157(9661571ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(837ns) {SwitchToState:179(557ns)}}}}}
2025-12-24T02:10:57.662844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223825751788:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5424
2025-12-24T02:10:57.666081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241222443262605:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.666102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.666110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222443262605:2048] [node 1] ICP22 created new session: [2:7587241222443262701:2048]
2025-12-24T02:10:57.666116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222443262701:2048] [node 1] ICS09 handshake done sender: [2:7587241222443262696:2118] self: [2:7587241222443262699:0] peer: [1:7587241223161413804:0] socket: 231 qp: -1
2025-12-24T02:10:57.666123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222443262701:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.666131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.666832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225038894332:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241225038894352:2116] held: no
2025-12-24T02:10:57.674832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751788:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.676263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390439:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.676284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390440:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.676354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390441:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.676396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390442:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.676428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390438:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.678832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390439:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.678861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390438:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.678906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223460390358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.678865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894368:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.682198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390441:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.682238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223460390359:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.683346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241224766001472:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(21219015ns) {HandleHandshakeStatus:329(21218326ns) {DropIncomingHandshake:442(5148ns) DropOutgoingHandshake:457(214ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(39969ns) {ForwardSessionEventToSession:612(30832ns) ForwardSessionEventToSession:612(691ns) ForwardSessionEventToSession:612(1051ns) ForwardSessionEventToSession:612(556ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(861ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(918ns)}}}
2025-12-24T02:10:57.682761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225038894352:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.682589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223460390418:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.686899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241223161413815:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.686952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222443262684:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.687097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222443262671:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.686782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751785:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.687131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.687137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222443262671:2108] [node 5] ICP22 created new session: [2:7587241222443262704:2108]
2025-12-24T02:10:57.686796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223825751785:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.687142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222443262704:2108] [node 5] ICS09 handshake done sender: [2:7587241222443262684:2115] self: [2:7587241222443262683:0] peer: [5:7587241224766001493:0] socket: 232 qp: -1
2025-12-24T02:10:57.687147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222443262704:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.687153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.686576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225038894332:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.686595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.686602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225038894332:2106] [node 5] ICP22 created new session: [6:7587241225038894369:2106]
2025-12-24T02:10:57.686608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225038894369:2106] [node 5] ICS09 handshake done sender: [6:7587241225038894352:2116] self: [6:7587241225038894365:0] peer: [5:7587241224766001475:0] socket: 251 qp: -1
2025-12-24T02:10:57.686615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225038894369:2106] [node 5] ICS10 traffic start
2025-12-24T02:10:57.686629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.690899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223825751785:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.686893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390442:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.690849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223460390357:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.690864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.690872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223460390357:2048] [node 1] ICP22 created new session: [4:7587241223460390447:2048]
2025-12-24T02:10:57.690879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223460390447:2048] [node 1] ICS09 handshake done sender: [4:7587241223460390418:2108] self: [4:7587241223460390430:0] peer: [1:7587241223161413809:0] socket: 215 qp: -1
2025-12-24T02:10:57.690886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223460390447:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.690916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.691108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223825751762:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.691142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.691148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223825751762:2107] [node 4] ICP22 created new session: [3:7587241223825751791:2107]
2025-12-24T02:10:57.691154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223825751791:2107] [node 4] ICS09 handshake done sender: [3:7587241223825751785:2120] self: [3:7587241223825751784:0] peer: [4:7587241223460390445:0] socket: 273 qp: -1
2025-12-24T02:10:57.691159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223825751791:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.691166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.693437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241223460390359:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241223460390441:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.690732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241222443262681:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.690757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241222443262681:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.695177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241224766001408:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20829) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241224766001489:2117] held: no
2025-12-24T02:10:57.697397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.697448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001500:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.697483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001501:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.697703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.694879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894371:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.698791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225038894371:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.698851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225038894342:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.698845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001499:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.698886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241224766001500:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.698915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241224766001409:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.698927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241224766001471:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.697959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223460390441:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.698046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223460390359:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.698058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.698063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223460390359:2050] [node 3] ICP22 created new session: [4:7587241223460390451:2050]
2025-12-24T02:10:57.698069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223460390451:2050] [node 3] ICS09 handshake done sender: [4:7587241223460390441:2119] self: [4:7587241223460390445:0] peer: [3:7587241223825751784:0] socket: 274 qp: -1
2025-12-24T02:10:57.698076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223460390451:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.698086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.702643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390434:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.702649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241223460390434:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.702824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241222443262681:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.702939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241222443262666:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.702958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.702966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241222443262666:2107] [node 4] ICP22 created new session: [2:7587241222443262707:2107]
2025-12-24T02:10:57.702972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241222443262707:2107] [node 4] ICS09 handshake done sender: [2:7587241222443262681:2113] self: [2:7587241222443262680:0] peer: [4:7587241223460390444:0] socket: 259 qp: -1
2025-12-24T02:10:57.702978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241222443262707:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.702991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.703255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223825751782:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.703271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223825751782:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.703410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223460390436:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.703419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241223460390436:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.704377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223460390436:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.705911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241223460390358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20829) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241223460390438:2116] held: no
2025-12-24T02:10:57.706048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223460390454:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59986
2025-12-24T02:10:57.706215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223460390434:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.706274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241223460390423:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.706284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.706289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223460390423:2109] [node 5] ICP22 created new session: [4:7587241223460390456:2109]
2025-12-24T02:10:57.706293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223460390456:2109] [node 5] ICS09 handshake done sender: [4:7587241223460390434:2114] self: [4:7587241223460390433:0] peer: [5:7587241224766001503:0] socket: 224 qp: -1
2025-12-24T02:10:57.706297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223460390456:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:57.706302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.706385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223460390438:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.706569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223460390358:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.706575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.706580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223460390358:2049] [node 2] ICP22 created new session: [4:7587241223460390458:2049]
2025-12-24T02:10:57.706583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223460390458:2049] [node 2] ICS09 handshake done sender: [4:7587241223460390438:2116] self: [4:7587241223460390444:0] peer: [2:7587241222443262680:0] socket: 265 qp: -1
2025-12-24T02:10:57.706587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223460390458:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.706591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.702556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241224766001489:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.706896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001504:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.704238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225038894373:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47659
2025-12-24T02:10:57.704533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225038894371:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.704610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225038894342:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.704623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.704628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225038894342:2111] [node 4] ICP22 created new session: [6:7587241225038894374:2111]
2025-12-24T02:10:57.704634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225038894374:2111] [node 4] ICS09 handshake done sender: [6:7587241225038894371:2123] self: [6:7587241225038894372:0] peer: [4:7587241223460390435:0] socket: 244 qp: -1
2025-12-24T02:10:57.706997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241224766001408:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.707019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.707026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241224766001408:2049] [node 2] ICP22 created new session: [5:7587241224766001505:2049]
2025-12-24T02:10:57.707031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241224766001505:2049] [node 2] ICS09 handshake done sender: [5:7587241224766001489:2117] self: [5:7587241224766001493:0] peer: [2:7587241222443262683:0] socket: 266 qp: -1
2025-12-24T02:10:57.707037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241224766001505:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.707044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.707722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241224766001500:2120] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.707811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241224766001471:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21226) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241224766001500:2120] held: no
2025-12-24T02:10:57.707933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241224766001507:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9051
2025-12-24T02:10:57.708043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241224766001471:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.708056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.708061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241224766001471:2109] [node 4] ICP22 created new session: [5:7587241224766001508:2109]
2025-12-24T02:10:57.708066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241224766001508:2109] [node 4] ICS09 handshake done sender: [5:7587241224766001500:2120] self: [5:7587241224766001503:0] peer: [4:7587241223460390433:0] socket: 249 qp: -1
2025-12-24T02:10:57.708072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241224766001508:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:57.708078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.707861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241223460390429:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.707878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.707884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223460390429:2110] [node 6] ICP22 created new session: [4:7587241223460390460:2110]
2025-12-24T02:10:57.707889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223460390460:2110] [node 6] ICS09 handshake done sender: [4:7587241223460390436:2115] self: [4:7587241223460390435:0] peer: [6:7587241225038894372:0] socket: 210 qp: -1
2025-12-24T02:10:57.707894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223460390460:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:57.707905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.704639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225038894374:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:57.704646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.710802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223825751782:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.711019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241224766001499:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.711101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241224766001409:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.711134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.711141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241224766001409:2050] [node 3] ICP22 created new session: [5:7587241224766001510:2050]
2025-12-24T02:10:57.711147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241224766001510:2050] [node 3] ICS09 handshake done sender: [5:7587241224766001499:2119] self: [5:7587241224766001502:0] peer: [3:7587241223825751776:0] socket: 233 qp: -1
2025-12-24T02:10:57.711153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241224766001510:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.711164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.723690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223825751763:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.723725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.723737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223825751763:2108] [node 5] ICP22 created new session: [3:7587241223825751794:2108]
2025-12-24T02:10:57.723744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223825751794:2108] [node 5] ICS09 handshake done sender: [3:7587241223825751782:2119] self: [3:7587241223825751776:0] peer: [5:7587241224766001502:0] socket: 216 qp: -1
2025-12-24T02:10:57.723754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223825751794:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.723764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.737818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:57.737904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241223460390447:2048] [node 1] ICS01 socket: 215 reason# EndOfStream
2025-12-24T02:10:57.737912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241223460390357:2048] [node 1] ICP30 unregister session Session# [4:7587241223460390447:2048] VirtualId# [4:7587241223460390430:0]
2025-12-24T02:10:57.737917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241223460390447:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.737957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.737975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.737095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:57.737190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241222443262701:2048] [node 1] ICS01 socket: 231 reason# EndOfStream
2025-12-24T02:10:57.737198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241222443262605:2048] [node 1] ICP30 unregister session Session# [2:7587241222443262701:2048] VirtualId# [2:7587241222443262699:0]
2025-12-24T02:10:57.737205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241222443262701:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.737264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.737290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.738745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:57.738830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241223825751769:2048] [node 1] ICS01 socket: 227 reason# EndOfStream
2025-12-24T02:10:57.738837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241223825751690:2048] [node 1] ICP30 unregister session Session# [3:7587241223825751769:2048] VirtualId# [3:7587241223825751765:0]
2025-12-24T02:10:57.738842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241223825751769:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.738897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.738920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.739040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:57.739106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241224766001490:2048] [node 1] ICS01 socket: 247 reason# EndOfStream
2025-12-24T02:10:57.739111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241224766001407:2048] [node 1] ICP30 unregister session Session# [5:7587241224766001490:2048] VirtualId# [5:7587241224766001482:0]
2025-12-24T02:10:57.739128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241224766001490:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.739173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.739193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.742862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:57.742945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225038894360:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-12-24T02:10:57.742953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225038894272:2048] [node 1] ICP30 unregister session Session# [6:7587241225038894360:2048] VirtualId# [6:7587241225038894348:0]
2025-12-24T02:10:57.742958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225038894360:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.743004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:57.743028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:57.749831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:57.749915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241223825751777:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-12-24T02:10:57.749924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241223825751691:2049] [node 2] ICP30 unregister session Session# [3:7587241223825751777:2049] VirtualId# [3:7587241223825751766:0]
2025-12-24T02:10:57.749931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241223825751777:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.749972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.749993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.752765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:57.752850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241223460390458:2049] [node 2] ICS01 socket: 265 reason# EndOfStream
2025-12-24T02:10:57.752864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241223460390358:2049] [node 2] ICP30 unregister session Session# [4:7587241223460390458:2049] VirtualId# [4:7587241223460390444:0]
2025-12-24T02:10:57.752870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241223460390458:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.752918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.752943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.752929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:57.752996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225038894357:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-12-24T02:10:57.753002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225038894273:2049] [node 2] ICP30 unregister session Session# [6:7587241225038894357:2049] VirtualId# [6:7587241225038894354:0]
2025-12-24T02:10:57.753006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225038894357:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.753042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.753062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.754765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:57.754847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241224766001505:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-12-24T02:10:57.754855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241224766001408:2049] [node 2] ICP30 unregister session Session# [5:7587241224766001505:2049] VirtualId# [5:7587241224766001493:0]
2025-12-24T02:10:57.754861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241224766001505:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.754926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:57.754951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:57.766777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:57.766778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:57.766944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225038894366:2050] [node 3] ICS01 socket: 264 reason# EndOfStream
2025-12-24T02:10:57.766952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225038894274:2050] [node 3] ICP30 unregister session Session# [6:7587241225038894366:2050] VirtualId# [6:7587241225038894363:0]
2025-12-24T02:10:57.766959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225038894366:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.767021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.767049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.767150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241224766001510:2050] [node 3] ICS01 socket: 233 reason# EndOfStream
2025-12-24T02:10:57.767160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241224766001409:2050] [node 3] ICP30 unregister session Session# [5:7587241224766001510:2050] VirtualId# [5:7587241224766001502:0]
2025-12-24T02:10:57.767165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241224766001510:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.767195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:57.767216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:57.786993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:57.787089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241225038894374:2111] [node 4] ICS07 socket disconnect 244 reason# EndOfStream
2025-12-24T02:10:57.787095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225038894374:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:57.787171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:57.787195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241225038894374:2111] [node 4] ICS15 start handshake
2025-12-24T02:10:57.788711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225038894342:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21226) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:57.788726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225038894374:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:57.788733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225038894342:2111] [node 4] ICP30 unregister session Session# [6:7587241225038894374:2111] VirtualId# [6:7587241225038894372:0]
2025-12-24T02:10:57.788750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:57.788756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241225038894342:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.788758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21226) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:57.795994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241225038894342:2111] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:57.817364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241222994688352:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:57.817374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241222994688353:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:57.818165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241221029063571:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.818176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241221029063572:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:57.829562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223854589970:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.829581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223854589971:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.829588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223854589972:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.822164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241223450353926:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.822172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241223450353927:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.822177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241223450353928:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.819483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223421248121:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.819490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223421248122:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.854097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241223976759631:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:57.854140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241223976759632:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:57.854149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241223976759633:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:57.862382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241223450353962:2077] [node 6] ICP01 ready to work
2025-12-24T02:10:57.863002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241223450353972:2083] [node 4] ICP01 ready to work
2025-12-24T02:10:57.867575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241223976759674:2084] [node 5] ICP01 ready to work
2025-12-24T02:10:57.866902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223854590012:2076] [node 6] ICP01 ready to work
2025-12-24T02:10:57.866995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241223854590013:2078] [node 5] ICP01 ready to work
2025-12-24T02:10:57.871307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241221029063636:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.871405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241221029063637:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.871485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241221029063638:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.874974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241222994688417:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.875079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241222994688418:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:57.875161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241222994688419:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:57.875211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241222994688415:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12062
2025-12-24T02:10:57.878776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241222994688416:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.890865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241223854589972:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23806826ns) {Configure:104(23806035ns) {ProcessConfigured:135(23396656ns) {StartInitialHandshake:157(23396032ns) {DropHandshakes:470(664ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(1774ns) {SwitchToState:179(945ns)}}}}}
2025-12-24T02:10:57.892502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590043:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
2025-12-24T02:10:57.894825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241223976759632:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27176779ns) {Configure:104(27176153ns) {ProcessConfigured:135(26902448ns) {StartInitialHandshake:157(26902019ns) {DropHandshakes:470(497ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(1405ns) {SwitchToState:179(604ns)}}}}}
2025-12-24T02:10:57.895658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241223976759704:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.907000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354004:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.920312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759709:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.920363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759710:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.919212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241221029063650:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8792
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.922958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223854590043:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.922999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223854589971:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.923334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450354004:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.923374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450353970:2085] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.923378Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241223450353970:2085] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.923335Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241222994688417:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44445168ns) {Configure:104(44444032ns) {ProcessConfigured:135(44034241ns) {StartInitialHandshake:157(44033620ns) {DropHandshakes:470(1424ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(238ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(618ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.924815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063650:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.923135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241223976759709:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.923170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241223976759674:2084] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.922977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223421248188:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:57.923060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223421248189:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:57.923136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241223421248190:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:57.923187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248184:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.923232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248185:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.923722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248186:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.926839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241223976759710:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.926882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241223976759704:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.930739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063642:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.930754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223854590040:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.930774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241223854590040:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.930949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759713:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.930766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241221029063642:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.930809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248187:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.932084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248186:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.932119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248184:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.935275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590050:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
2025-12-24T02:10:57.935760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223854590040:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.933500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241223450353970:2085] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.938861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248187:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.939945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241223976759709:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.939997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241223976759674:2084] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.940010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.940018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241223976759674:2084] [node 5] ICP22 created new session: [6:7587241223976759715:2084]
2025-12-24T02:10:57.940025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241223976759715:2084] [node 5] ICS09 handshake done sender: [6:7587241223976759709:2115] self: [6:7587241223976759711:0] peer: [5:7587241223450353964:0] socket: 236 qp: -1
2025-12-24T02:10:57.940032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241223976759715:2084] [node 5] ICS10 traffic start
2025-12-24T02:10:57.940044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.940770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759714:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.941218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241223976759710:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.941266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241223976759704:2103] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.941275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.941280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241223976759704:2103] [node 4] ICP22 created new session: [6:7587241223976759718:2103]
2025-12-24T02:10:57.941286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241223976759718:2103] [node 4] ICS09 handshake done sender: [6:7587241223976759710:2116] self: [6:7587241223976759712:0] peer: [4:7587241223854590039:0] socket: 238 qp: -1
2025-12-24T02:10:57.941290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241223976759718:2103] [node 4] ICS10 traffic start
2025-12-24T02:10:57.941296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.944115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241223450353962:2077] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.944142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.944149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241223450353962:2077] [node 6] ICP22 created new session: [5:7587241223450354007:2077]
2025-12-24T02:10:57.944157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241223450354007:2077] [node 6] ICS09 handshake done sender: [5:7587241223450353970:2085] self: [5:7587241223450353964:0] peer: [6:7587241223976759711:0] socket: 225 qp: -1
2025-12-24T02:10:57.944164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241223450354007:2077] [node 6] ICS10 traffic start
2025-12-24T02:10:57.943516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688415:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.943545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688416:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.943617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241222994688431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12062
2025-12-24T02:10:57.943869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241222994688430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12062
2025-12-24T02:10:57.943922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241222994688432:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12062
2025-12-24T02:10:57.944172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.947721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241223854590012:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.947739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.947745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223854590012:2076] [node 6] ICP22 created new session: [4:7587241223854590052:2076]
2025-12-24T02:10:57.947752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223854590052:2076] [node 6] ICS09 handshake done sender: [4:7587241223854590040:2112] self: [4:7587241223854590039:0] peer: [6:7587241223976759712:0] socket: 231 qp: -1
2025-12-24T02:10:57.947760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223854590052:2076] [node 6] ICS10 traffic start
2025-12-24T02:10:57.947768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:57.948127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590054:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
2025-12-24T02:10:57.948149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590055:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
2025-12-24T02:10:57.947651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241221029063652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8792
2025-12-24T02:10:57.949367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241221029063642:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.949464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241221029063636:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.949523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688432:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.949473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.949480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241221029063636:2107] [node 4] ICP22 created new session: [2:7587241221029063654:2107]
2025-12-24T02:10:57.949486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241221029063654:2107] [node 4] ICS09 handshake done sender: [2:7587241221029063642:2112] self: [2:7587241221029063641:0] peer: [4:7587241223854590049:0] socket: 223 qp: -1
2025-12-24T02:10:57.949491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241221029063654:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.949499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.954197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354009:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.954220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354010:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.954274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354011:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.954890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223854590043:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.954986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223854589971:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.954999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.955007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223854589971:2049] [node 2] ICP22 created new session: [4:7587241223854590057:2049]
2025-12-24T02:10:57.955012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223854590057:2049] [node 2] ICS09 handshake done sender: [4:7587241223854590043:2114] self: [4:7587241223854590049:0] peer: [2:7587241221029063641:0] socket: 228 qp: -1
2025-12-24T02:10:57.955017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223854590057:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.955023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.955785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223854590042:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.955794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241223854590042:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.956331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223854590042:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.956406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241223854590013:2078] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.956417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.956423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223854590013:2078] [node 5] ICP22 created new session: [4:7587241223854590060:2078]
2025-12-24T02:10:57.956429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223854590060:2078] [node 5] ICS09 handshake done sender: [4:7587241223854590042:2113] self: [4:7587241223854590041:0] peer: [5:7587241223450354013:0] socket: 262 qp: -1
2025-12-24T02:10:57.956435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223854590060:2078] [node 5] ICS10 traffic start
2025-12-24T02:10:57.956442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.955629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450354010:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.955664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241223450353972:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.956674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.956800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241223450354010:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.956840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241223450353972:2083] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.956849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.956855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241223450353972:2083] [node 4] ICP22 created new session: [5:7587241223450354015:2083]
2025-12-24T02:10:57.956861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241223450354015:2083] [node 4] ICS09 handshake done sender: [5:7587241223450354010:2117] self: [5:7587241223450354013:0] peer: [4:7587241223854590041:0] socket: 266 qp: -1
2025-12-24T02:10:57.956867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241223450354015:2083] [node 4] ICS10 traffic start
2025-12-24T02:10:57.956873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.958778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241221029063657:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8792
2025-12-24T02:10:57.959013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450354011:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.959059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241223450353927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.964124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063657:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.965040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063644:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.965055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241221029063644:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.962857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688430:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688421:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241222994688421:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.969947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354018:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.970042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241221029063644:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.970218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063640:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241221029063640:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.966903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248185:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.969809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241223421248122:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.969909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248207:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.970268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223854590054:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223854589970:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.971087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241223854590050:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.971111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241223854589972:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.972026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
2025-12-24T02:10:57.972338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223854590050:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.972403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223854589972:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.972419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.972424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223854589972:2050] [node 3] ICP22 created new session: [4:7587241223854590065:2050]
2025-12-24T02:10:57.972431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223854590065:2050] [node 3] ICS09 handshake done sender: [4:7587241223854590050:2116] self: [4:7587241223854590063:0] peer: [3:7587241223421248195:0] socket: 248 qp: -1
2025-12-24T02:10:57.972437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223854590065:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.972445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.970804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248207:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241223421248121:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.971262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248196:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.971266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223421248196:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.972144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223421248196:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.973332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223421248188:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.973347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.973353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223421248188:2107] [node 4] ICP22 created new session: [3:7587241223421248211:2107]
2025-12-24T02:10:57.973358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223421248211:2107] [node 4] ICS09 handshake done sender: [3:7587241223421248196:2116] self: [3:7587241223421248195:0] peer: [4:7587241223854590063:0] socket: 247 qp: -1
2025-12-24T02:10:57.973365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223421248211:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.973372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.974208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759720:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.970768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241221029063637:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.971361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.971367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241221029063637:2108] [node 5] ICP22 created new session: [2:7587241221029063660:2108]
2025-12-24T02:10:57.971377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241221029063660:2108] [node 5] ICS09 handshake done sender: [2:7587241221029063644:2113] self: [2:7587241221029063643:0] peer: [5:7587241223450354017:0] socket: 264 qp: -1
2025-12-24T02:10:57.971382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241221029063660:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.971389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.970937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450354009:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.970970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241223450353926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.971196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241223450354011:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.971246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241223450353927:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.971259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.971265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241223450353927:2049] [node 2] ICP22 created new session: [5:7587241223450354020:2049]
2025-12-24T02:10:57.971271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241223450354020:2049] [node 2] ICS09 handshake done sender: [5:7587241223450354011:2118] self: [5:7587241223450354017:0] peer: [2:7587241221029063643:0] socket: 267 qp: -1
2025-12-24T02:10:57.971278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241223450354020:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.971289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.975089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688425:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.975101Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241222994688425:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.975471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688423:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.975474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241222994688423:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.976112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688431:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.977134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063652:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.978239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354022:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.977602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241221029063640:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.977802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241221029063572:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.977814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.977819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241221029063572:2049] [node 3] ICP22 created new session: [2:7587241221029063665:2049]
2025-12-24T02:10:57.977823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241221029063665:2049] [node 3] ICS09 handshake done sender: [2:7587241221029063640:2111] self: [2:7587241221029063639:0] peer: [3:7587241223421248208:0] socket: 222 qp: -1
2025-12-24T02:10:57.977828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241221029063665:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.977834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.978864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241223854590067:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.979112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241222994688425:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:57.978997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759721:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.979320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241222994688423:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.979397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241222994688418:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.979423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:57.979429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241222994688418:2108] [node 5] ICP22 created new session: [1:7587241222994688444:2108]
2025-12-24T02:10:57.979436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241222994688444:2108] [node 5] ICS09 handshake done sender: [1:7587241222994688425:2114] self: [1:7587241222994688424:0] peer: [5:7587241223450354019:0] socket: 257 qp: -1
2025-12-24T02:10:57.979442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241222994688444:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:57.979450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:57.979438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241223450354009:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.979875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241223450353926:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.979885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.979891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241223450353926:2048] [node 1] ICP22 created new session: [5:7587241223450354023:2048]
2025-12-24T02:10:57.979897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241223450354023:2048] [node 1] ICS09 handshake done sender: [5:7587241223450354009:2116] self: [5:7587241223450354019:0] peer: [1:7587241222994688424:0] socket: 265 qp: -1
2025-12-24T02:10:57.979903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241223450354023:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.979913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.983053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241222994688353:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.983075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.983083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241222994688353:2049] [node 3] ICP22 created new session: [1:7587241222994688446:2049]
2025-12-24T02:10:57.983089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241222994688446:2049] [node 3] ICS09 handshake done sender: [1:7587241222994688423:2113] self: [1:7587241222994688422:0] peer: [3:7587241223421248209:0] socket: 256 qp: -1
2025-12-24T02:10:57.983097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241222994688446:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:57.983104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.987367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241223976759720:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.987400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241223976759721:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.987435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241223976759633:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:57.987446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241223976759632:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.988238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759724:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.988458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241223976759721:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.988494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241223976759632:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.988504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.988509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241223976759632:2049] [node 2] ICP22 created new session: [6:7587241223976759725:2049]
2025-12-24T02:10:57.988515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241223976759725:2049] [node 2] ICS09 handshake done sender: [6:7587241223976759721:2120] self: [6:7587241223976759723:0] peer: [2:7587241221029063645:0] socket: 280 qp: -1
2025-12-24T02:10:57.988521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241223976759725:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.988530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.986906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241223421248122:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22562) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241223421248185:2111] held: no
2025-12-24T02:10:57.987070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248214:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.987544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063646:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.987554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241221029063646:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.988318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241221029063646:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:57.989225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241221029063638:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.989240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:57.989246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241221029063638:2109] [node 6] ICP22 created new session: [2:7587241221029063668:2109]
2025-12-24T02:10:57.989252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241221029063668:2109] [node 6] ICS09 handshake done sender: [2:7587241221029063646:2114] self: [2:7587241221029063645:0] peer: [6:7587241223976759723:0] socket: 279 qp: -1
2025-12-24T02:10:57.989258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241221029063668:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:57.989266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.992123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241222994688421:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:57.992352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241223854590054:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:57.992501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241223854589970:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.992515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.992522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241223854589970:2048] [node 1] ICP22 created new session: [4:7587241223854590068:2048]
2025-12-24T02:10:57.992528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241223854590068:2048] [node 1] ICS09 handshake done sender: [4:7587241223854590054:2117] self: [4:7587241223854590062:0] peer: [1:7587241222994688420:0] socket: 260 qp: -1
2025-12-24T02:10:57.992533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241223854590068:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.992542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.993409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241222994688417:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:57.993425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:57.993433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241222994688417:2107] [node 4] ICP22 created new session: [1:7587241222994688450:2107]
2025-12-24T02:10:57.993438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241222994688450:2107] [node 4] ICS09 handshake done sender: [1:7587241222994688421:2112] self: [1:7587241222994688420:0] peer: [4:7587241223854590062:0] socket: 255 qp: -1
2025-12-24T02:10:57.993447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241222994688450:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:57.993455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:57.991202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241223421248215:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32375
2025-12-24T02:10:57.991383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248198:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:57.991387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223421248198:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:57.991819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223421248207:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.994912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759727:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.994942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241223976759728:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:57.996471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241223421248121:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.996492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:57.996499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223421248121:2048] [node 1] ICP22 created new session: [3:7587241223421248217:2048]
2025-12-24T02:10:57.996505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223421248217:2048] [node 1] ICS09 handshake done sender: [3:7587241223421248207:2119] self: [3:7587241223421248209:0] peer: [1:7587241222994688422:0] socket: 272 qp: -1
2025-12-24T02:10:57.996511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223421248217:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:57.996517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:57.996577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241223421248121:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:57.996783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223421248185:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:57.996873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241223421248122:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.996881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:57.996888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223421248122:2049] [node 2] ICP22 created new session: [3:7587241223421248220:2049]
2025-12-24T02:10:57.996893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241223421248220:2049] [node 2] ICS09 handshake done sender: [3:7587241223421248185:2111] self: [3:7587241223421248208:0] peer: [2:7587241221029063639:0] socket: 227 qp: -1
2025-12-24T02:10:57.996898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241223421248220:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:57.996904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:57.998379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223421248198:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:57.999453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241223976759720:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:57.999552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241223976759633:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:57.999568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:57.999574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241223976759633:2050] [node 3] ICP22 created new session: [6:7587241223976759729:2050]
2025-12-24T02:10:57.999579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241223976759729:2050] [node 3] ICS09 handshake done sender: [6:7587241223976759720:2119] self: [6:7587241223976759722:0] peer: [3:7587241223421248197:0] socket: 241 qp: -1
2025-12-24T02:10:57.999586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241223976759729:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:57.999592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:57.998935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241223450354025:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:57.999236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241221029063670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8792
2025-12-24T02:10:58.001104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241221029063670:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.001138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241221029063571:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.007542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688429:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.007554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241222994688429:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.008161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241223450354025:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.008200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241223450353928:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.009782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241222994688429:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.009916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241222994688352:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.009934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.009940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241222994688352:2048] [node 2] ICP22 created new session: [1:7587241227289655749:2048]
2025-12-24T02:10:58.009946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227289655749:2048] [node 2] ICS09 handshake done sender: [1:7587241222994688429:2116] self: [1:7587241222994688428:0] peer: [2:7587241225324030967:0] socket: 253 qp: -1
2025-12-24T02:10:58.009953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227289655749:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:58.009960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.006972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223421248189:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.007333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241223976759727:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.007429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241223976759631:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.009689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241227745321323:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32239
2025-12-24T02:10:58.010329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241222994688427:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.010335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241222994688427:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.013676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241222994688427:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.007007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.011894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241223450354025:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.011956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241223450353928:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.011968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.011975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241223450353928:2050] [node 3] ICP22 created new session: [5:7587241227745321324:2050]
2025-12-24T02:10:58.011981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241227745321324:2050] [node 3] ICS09 handshake done sender: [5:7587241223450354025:2122] self: [5:7587241227745321322:0] peer: [3:7587241223421248199:0] socket: 245 qp: -1
2025-12-24T02:10:58.011987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241227745321324:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.011995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.014779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241222994688419:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.014796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.014802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241222994688419:2109] [node 6] ICP22 created new session: [1:7587241227289655752:2109]
2025-12-24T02:10:58.014807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227289655752:2109] [node 6] ICS09 handshake done sender: [1:7587241222994688427:2115] self: [1:7587241222994688426:0] peer: [6:7587241228271727027:0] socket: 218 qp: -1
2025-12-24T02:10:58.014813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227289655752:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:58.014819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.011614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228271727028:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11316
2025-12-24T02:10:58.013875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241223976759727:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.013924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241223976759631:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.013935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.013940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241223976759631:2048] [node 1] ICP22 created new session: [6:7587241228271727029:2048]
2025-12-24T02:10:58.013945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228271727029:2048] [node 1] ICS09 handshake done sender: [6:7587241223976759727:2122] self: [6:7587241228271727027:0] peer: [1:7587241222994688426:0] socket: 240 qp: -1
2025-12-24T02:10:58.013951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228271727029:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.013958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.015499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241221029063571:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241221029063670:2118] held: no
2025-12-24T02:10:58.007015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223421248189:2108] [node 6] ICP22 created new session: [3:7587241227716215518:2108]
2025-12-24T02:10:58.007023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227716215518:2108] [node 6] ICS09 handshake done sender: [3:7587241223421248198:2117] self: [3:7587241223421248197:0] peer: [6:7587241223976759722:0] socket: 229 qp: -1
2025-12-24T02:10:58.007032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227716215518:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:58.007043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.008849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241223421248200:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.008854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241223421248200:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.011713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241223421248200:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.013082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241223421248190:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.013094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.013100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241223421248190:2109] [node 5] ICP22 created new session: [3:7587241227716215521:2109]
2025-12-24T02:10:58.013107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227716215521:2109] [node 5] ICS09 handshake done sender: [3:7587241223421248200:2118] self: [3:7587241223421248199:0] peer: [5:7587241227745321322:0] socket: 230 qp: -1
2025-12-24T02:10:58.013113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227716215521:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:58.013120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.026842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225324030968:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8792
2025-12-24T02:10:58.030892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241221029063670:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.030969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241221029063571:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.030987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.030995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241221029063571:2048] [node 1] ICP22 created new session: [2:7587241225324030969:2048]
2025-12-24T02:10:58.031002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225324030969:2048] [node 1] ICS09 handshake done sender: [2:7587241221029063670:2118] self: [2:7587241225324030967:0] peer: [1:7587241222994688428:0] socket: 283 qp: -1
2025-12-24T02:10:58.031011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225324030969:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.031022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.034237Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:58.034324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241225324030969:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:10:58.034335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241221029063571:2048] [node 1] ICP30 unregister session Session# [2:7587241225324030969:2048] VirtualId# [2:7587241225324030967:0]
2025-12-24T02:10:58.034342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241225324030969:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.034388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.034410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.038819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:58.038929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241223421248217:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-12-24T02:10:58.038939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241223421248121:2048] [node 1] ICP30 unregister session Session# [3:7587241223421248217:2048] VirtualId# [3:7587241223421248209:0]
2025-12-24T02:10:58.038946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241223421248217:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.039002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.039046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.042810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:58.042898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241228271727029:2048] [node 1] ICS01 socket: 240 reason# EndOfStream
2025-12-24T02:10:58.042906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241223976759631:2048] [node 1] ICP30 unregister session Session# [6:7587241228271727029:2048] VirtualId# [6:7587241228271727027:0]
2025-12-24T02:10:58.042802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:58.043038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:58.042899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241223450354023:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-12-24T02:10:58.043108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241223854590068:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-12-24T02:10:58.043114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241223854589970:2048] [node 1] ICP30 unregister session Session# [4:7587241223854590068:2048] VirtualId# [4:7587241223854590062:0]
2025-12-24T02:10:58.042906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241223450353926:2048] [node 1] ICP30 unregister session Session# [5:7587241223450354023:2048] VirtualId# [5:7587241223450354019:0]
2025-12-24T02:10:58.043133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241223854590068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.043165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.043190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.042914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241223450354023:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.042973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.042998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.042913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241228271727029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.042971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.042996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.046752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:58.046752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:58.046864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241223854590057:2049] [node 2] ICS01 socket: 228 reason# EndOfStream
2025-12-24T02:10:58.046872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241223854589971:2049] [node 2] ICP30 unregister session Session# [4:7587241223854590057:2049] VirtualId# [4:7587241223854590049:0]
2025-12-24T02:10:58.046878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241223854590057:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.046939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.046979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.047143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241223421248220:2049] [node 2] ICS01 socket: 227 reason# EndOfStream
2025-12-24T02:10:58.047155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241223421248122:2049] [node 2] ICP30 unregister session Session# [3:7587241223421248220:2049] VirtualId# [3:7587241223421248208:0]
2025-12-24T02:10:58.047160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241223421248220:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.047198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.047223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.048443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:58.048514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241223450354020:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:10:58.048520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241223450353927:2049] [node 2] ICP30 unregister session Session# [5:7587241223450354020:2049] VirtualId# [5:7587241223450354017:0]
2025-12-24T02:10:58.048525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241223450354020:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.048568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.048591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.048623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:58.048680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241223976759725:2049] [node 2] ICS01 socket: 280 reason# EndOfStream
2025-12-24T02:10:58.048686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241223976759632:2049] [node 2] ICP30 unregister session Session# [6:7587241223976759725:2049] VirtualId# [6:7587241223976759723:0]
2025-12-24T02:10:58.048690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241223976759725:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.048713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.048731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.057699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:58.057783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241223976759729:2050] [node 3] ICS01 socket: 241 reason# EndOfStream
2025-12-24T02:10:58.057792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241223976759633:2050] [node 3] ICP30 unregister session Session# [6:7587241223976759729:2050] VirtualId# [6:7587241223976759722:0]
2025-12-24T02:10:58.057798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241223976759729:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.057848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.057869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.057902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:58.057957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241227745321324:2050] [node 3] ICS01 socket: 245 reason# EndOfStream
2025-12-24T02:10:58.057962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241223450353928:2050] [node 3] ICP30 unregister session Session# [5:7587241227745321324:2050] VirtualId# [5:7587241227745321322:0]
2025-12-24T02:10:58.057966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241227745321324:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.057988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.058006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.083973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:58.084057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241223976759718:2103] [node 4] ICS01 socket: 238 reason# EndOfStream
2025-12-24T02:10:58.084066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241223976759704:2103] [node 4] ICP30 unregister session Session# [6:7587241223976759718:2103] VirtualId# [6:7587241223976759712:0]
2025-12-24T02:10:58.084072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241223976759718:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.084134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:58.084156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:58.118822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241227614325981:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:58.118838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241227614325982:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.127477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241225135515015:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.127500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241225135515016:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.131413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241225895329701:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.131422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241225895329702:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.131427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241225895329703:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.130772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228107217604:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.130807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228107217605:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.133861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241228118761428:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.133871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241228118761429:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.133875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241228118761430:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.154797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225839256223:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.154822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225839256224:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.154834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225839256225:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.168078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241225895329750:2091] [node 5] ICP01 ready to work
2025-12-24T02:10:58.168169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241225895329751:2095] [node 6] ICP01 ready to work
2025-12-24T02:10:58.168482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241225135515054:2085] [node 6] ICP01 ready to work
2025-12-24T02:10:58.175701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225839256262:2064] [node 4] ICP01 ready to work
2025-12-24T02:10:58.175811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241225839256263:2072] [node 5] ICP01 ready to work
2025-12-24T02:10:58.178997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241228118761476:2070] [node 6] ICP01 ready to work
2025-12-24T02:10:58.178965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228107217665:2092] [node 5] ICP01 ready to work
2025-12-24T02:10:58.182462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228107217669:2088] [node 6] ICP01 ready to work
2025-12-24T02:10:58.182989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241227614326044:2090] [node 5] ICP01 ready to work
2025-12-24T02:10:58.186818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241225839256224:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18098294ns) {Configure:104(18096645ns) {ProcessConfigured:135(17767992ns) {StartInitialHandshake:157(17767423ns) {DropHandshakes:470(235ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1075ns) {SwitchToState:179(641ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.196909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241225839256262:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21033303ns) {Configure:104(21031867ns) {ProcessConfigured:135(20722583ns) {StartInitialHandshake:157(20721906ns) {DropHandshakes:470(713ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:477(1613ns) {SwitchToState:179(813ns)}}}}}
2025-12-24T02:10:58.198814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241228107217665:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19768368ns) {Configure:104(19767373ns) {ProcessConfigured:135(19382568ns) {StartInitialHandshake:157(19381778ns) {DropHandshakes:470(613ns) {DropIncomingHandshake:442(136ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1747ns) {SwitchToState:179(1080ns)}}}}}
2025-12-24T02:10:58.199802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228107217677:2090] [node 4] ICP01 ready to work
2025-12-24T02:10:58.200178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217674:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
2025-12-24T02:10:58.202825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241228107217605:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20295921ns) {Configure:104(20294988ns) {ProcessConfigured:135(19958322ns) {StartInitialHandshake:157(19957596ns) {DropHandshakes:470(696ns) {DropIncomingHandshake:442(132ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(1137ns) {SwitchToState:179(490ns)}}}}}
2025-12-24T02:10:58.203538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217674:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.204631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241228118761482:2068] [node 4] ICP01 ready to work
2025-12-24T02:10:58.204807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241227614326046:2092] [node 4] ICP01 ready to work
2025-12-24T02:10:58.204981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241227614326047:2098] [node 6] ICP01 ready to work
2025-12-24T02:10:58.204790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241225135515068:2087] [node 5] ICP01 ready to work
2025-12-24T02:10:58.204980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241225135515081:2096] [node 4] ICP01 ready to work
2025-12-24T02:10:58.214803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241225135515054:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(46232251ns) {Configure:104(46231211ns) {ProcessConfigured:135(45856712ns) {StartInitialHandshake:157(45856096ns) {DropHandshakes:470(544ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(469ns) PrepareNewSessionHandshake:477(1786ns) {SwitchToState:179(775ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.217699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217685:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
2025-12-24T02:10:58.216483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241225895329750:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48267863ns) {Configure:104(48266755ns) {ProcessConfigured:135(47912566ns) {StartInitialHandshake:157(47912236ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(896ns) {SwitchToState:179(108ns)}}}}}
2025-12-24T02:10:58.227307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241225895329751:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(55675459ns) {Configure:104(55673343ns) {ProcessConfigured:135(55357993ns) {StartInitialHandshake:157(55357285ns) {DropHandshakes:470(600ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(1494ns) {SwitchToState:179(592ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.230929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241228118761429:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26204970ns) {Configure:104(26204186ns) {ProcessConfigured:135(25912722ns) {StartInitialHandshake:157(25912047ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(839ns)}}}}}
2025-12-24T02:10:58.231841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761502:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.232407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761500:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.232476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761501:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.232918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217686:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
2025-12-24T02:10:58.233054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761500:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.234368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225135515094:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7492
2025-12-24T02:10:58.234457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225135515095:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7492
2025-12-24T02:10:58.234478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225135515096:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7492
2025-12-24T02:10:58.234662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515094:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.235786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241227614326048:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35620
2025-12-24T02:10:58.235855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241227614326049:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35620
2025-12-24T02:10:58.235865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241227614326050:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.236323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225135515098:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7492
2025-12-24T02:10:58.236144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241228118761428:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(57077409ns) {Configure:104(57076211ns) {ProcessConfigured:135(56689040ns) {StartInitialHandshake:157(56688842ns) {DropHandshakes:470(491ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(838ns) {SwitchToState:179(423ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.238926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.238972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256301:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.239004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256302:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.239025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256303:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.239036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256304:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.239107Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241227614326044:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56050931ns) {Configure:104(56050502ns) {ProcessConfigured:135(55663435ns) {StartInitialHandshake:157(55662782ns) {DropHandshakes:470(294ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(155ns)}}}}}
2025-12-24T02:10:58.254947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515098:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.254991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241225135515015:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.255341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217686:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761501:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241228118761430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.256045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515096:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217671:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241228107217671:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.256198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217687:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.256222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217685:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.256247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241228107217604:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.256260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241228107217605:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.255153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326049:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326052:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241227614326052:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.255821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326048:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.255876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241227614326052:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.255950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241227614325981:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.255964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.255972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241227614325981:2048] [node 2] ICP22 created new session: [1:7587241227614326065:2048]
2025-12-24T02:10:58.255979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227614326065:2048] [node 2] ICS09 handshake done sender: [1:7587241227614326052:2113] self: [1:7587241227614326051:0] peer: [2:7587241225135515099:0] socket: 255 qp: -1
2025-12-24T02:10:58.255987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227614326065:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:58.255995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.257814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329779:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.256474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228107217671:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.256524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761511:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.256740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241228118761501:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.256813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241228118761430:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.256823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.256829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241228118761430:2050] [node 3] ICP22 created new session: [5:7587241228118761512:2050]
2025-12-24T02:10:58.256835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241228118761512:2050] [node 3] ICS09 handshake done sender: [5:7587241228118761501:2114] self: [5:7587241228118761510:0] peer: [3:7587241228107217666:0] socket: 244 qp: -1
2025-12-24T02:10:58.256842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241228118761512:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.256849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.257941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225839256304:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.257978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225839256302:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.258000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225839256303:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.258023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225839256301:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.258057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225839256223:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.258069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225839256224:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.258080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225839256263:2072] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.258089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225839256225:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.262729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515082:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.262747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241225135515082:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.263286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515084:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.263291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241225135515084:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.263951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241225135515082:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.264411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761499:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.264438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241228118761499:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.264476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217676:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.264534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241228107217676:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.268767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761502:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.268803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241228118761482:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.267035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326054:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.267048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241227614326054:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.267537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326056:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.267540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241227614326056:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.268904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241227614326054:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.271381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515095:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.273521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241228118761499:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.270861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241228107217665:2092] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.270889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.270899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228107217665:2092] [node 5] ICP22 created new session: [3:7587241228107217694:2092]
2025-12-24T02:10:58.270906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228107217694:2092] [node 5] ICS09 handshake done sender: [3:7587241228107217671:2110] self: [3:7587241228107217666:0] peer: [5:7587241228118761510:0] socket: 227 qp: -1
2025-12-24T02:10:58.270914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228107217694:2092] [node 5] ICS10 traffic start
2025-12-24T02:10:58.270925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.273276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217692:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
2025-12-24T02:10:58.273296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217693:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
2025-12-24T02:10:58.271975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329779:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.274629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241225135515015:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241225135515098:2118] held: no
2025-12-24T02:10:58.274824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241225135515084:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.275166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241227614326056:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.274515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256309:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.275349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241227614325982:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.275369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.275377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241227614325982:2049] [node 3] ICP22 created new session: [1:7587241227614326071:2049]
2025-12-24T02:10:58.275384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227614326071:2049] [node 3] ICS09 handshake done sender: [1:7587241227614326054:2114] self: [1:7587241227614326053:0] peer: [3:7587241228107217689:0] socket: 256 qp: -1
2025-12-24T02:10:58.275390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227614326071:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:58.275397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.273714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241228118761476:2070] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.273735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.273742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241228118761476:2070] [node 6] ICP22 created new session: [5:7587241228118761518:2070]
No RDMA context found for GID: 2025-12-24T02:10:58.278742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329780:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.273750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241228118761518:2070] [node 6] ICS09 handshake done sender: [5:7587241228118761499:2112] self: [5:7587241228118761498:0] peer: [6:7587241225839256307:0] socket: 248 qp: -1
2025-12-24T02:10:58.273756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241228118761518:2070] [node 6] ICS10 traffic start
2025-12-24T02:10:58.273764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.278771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329768:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.278779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241225895329768:2110] [node 5] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-12-24T02:10:58.279527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228107217685:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.279607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241225135515016:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.279624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.279630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241225135515016:2049] [node 3] ICP22 created new session: [2:7587241225135515106:2049]
2025-12-24T02:10:58.279637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225135515106:2049] [node 3] ICS09 handshake done sender: [2:7587241225135515084:2111] self: [2:7587241225135515083:0] peer: [3:7587241228107217690:0] socket: 235 qp: -1
2025-12-24T02:10:58.279644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225135515106:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:58.279653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.279790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241225135515103:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7492
2025-12-24T02:10:58.280027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241227614326047:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.280044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.280049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241227614326047:2098] [node 6] ICP22 created new session: [1:7587241227614326073:2098]
2025-12-24T02:10:58.280054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227614326073:2098] [node 6] ICS09 handshake done sender: [1:7587241227614326056:2115] self: [1:7587241227614326055:0] peer: [6:7587241225839256305:0] socket: 257 qp: -1
2025-12-24T02:10:58.280059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227614326073:2098] [node 6] ICS10 traffic start
2025-12-24T02:10:58.280077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.280490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326050:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.282037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241227614326075:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.281405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.282281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.283148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241225895329768:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.283499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225839256263:2072] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25596) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241225839256303:2117] held: no
2025-12-24T02:10:58.285222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329781:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.285307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241225895329750:2091] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.285324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.285331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241225895329750:2091] [node 5] ICP22 created new session: [4:7587241225895329784:2091]
2025-12-24T02:10:58.285338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241225895329784:2091] [node 5] ICS09 handshake done sender: [4:7587241225895329768:2110] self: [4:7587241225895329752:0] peer: [5:7587241228118761515:0] socket: 237 qp: -1
2025-12-24T02:10:58.285344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241225895329784:2091] [node 5] ICS10 traffic start
2025-12-24T02:10:58.285366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.286473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225839256302:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.285603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241225135515054:2085] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.285625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.285631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241225135515054:2085] [node 6] ICP22 created new session: [2:7587241225135515109:2085]
2025-12-24T02:10:58.285636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225135515109:2085] [node 6] ICS09 handshake done sender: [2:7587241225135515082:2110] self: [2:7587241225135515055:0] peer: [6:7587241225839256306:0] socket: 234 qp: -1
2025-12-24T02:10:58.285641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225135515109:2085] [node 6] ICS10 traffic start
2025-12-24T02:10:58.285647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.285997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241225135515098:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.287168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326075:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.286877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.287419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256311:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.288415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225839256224:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.288430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.288436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225839256224:2049] [node 2] ICP22 created new session: [6:7587241225839256313:2049]
2025-12-24T02:10:58.288442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225839256313:2049] [node 2] ICS09 handshake done sender: [6:7587241225839256302:2116] self: [6:7587241225839256306:0] peer: [2:7587241225135515055:0] socket: 261 qp: -1
2025-12-24T02:10:58.288448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225839256313:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.288455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.286890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329780:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.287001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241225895329701:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.289840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241225135515015:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.289859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.289868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241225135515015:2048] [node 1] ICP22 created new session: [2:7587241225135515111:2048]
2025-12-24T02:10:58.289876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225135515111:2048] [node 1] ICS09 handshake done sender: [2:7587241225135515098:2118] self: [2:7587241225135515099:0] peer: [1:7587241227614326051:0] socket: 258 qp: -1
2025-12-24T02:10:58.286915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761522:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.287266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761521:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.287277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241228118761502:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.288325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241228118761482:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.288336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.288343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241228118761482:2068] [node 4] ICP22 created new session: [5:7587241228118761523:2068]
2025-12-24T02:10:58.288350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241228118761523:2068] [node 4] ICS09 handshake done sender: [5:7587241228118761502:2115] self: [5:7587241228118761515:0] peer: [4:7587241225895329752:0] socket: 245 qp: -1
2025-12-24T02:10:58.288355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241228118761523:2068] [node 4] ICS10 traffic start
2025-12-24T02:10:58.288363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.289133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241228118761428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.288549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228107217687:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.288639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241228107217605:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.288650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.288655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228107217605:2049] [node 2] ICP22 created new session: [3:7587241228107217698:2049]
2025-12-24T02:10:58.288660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228107217698:2049] [node 2] ICS09 handshake done sender: [3:7587241228107217685:2116] self: [3:7587241228107217690:0] peer: [2:7587241225135515083:0] socket: 240 qp: -1
2025-12-24T02:10:58.288665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228107217698:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.288672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.289882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225135515111:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.289889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.294749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326058:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.294774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241227614326058:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.291737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241228118761522:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.291766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241228118761429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.298870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241227614326062:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.298884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241227614326062:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.299656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241227614326078:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35620
2025-12-24T02:10:58.299786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241227614326062:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.299879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241227614326044:2090] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.299898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.299904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241227614326044:2090] [node 5] ICP22 created new session: [1:7587241227614326081:2090]
2025-12-24T02:10:58.299910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227614326081:2090] [node 5] ICS09 handshake done sender: [1:7587241227614326062:2117] self: [1:7587241227614326045:0] peer: [5:7587241228118761525:0] socket: 282 qp: -1
2025-12-24T02:10:58.299916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227614326081:2090] [node 5] ICS10 traffic start
2025-12-24T02:10:58.299923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.305405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329781:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.305445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241225895329702:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.306334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329789:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.306619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241225895329781:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.306793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241225839256224:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(18339967ns) {HandleHandshakeStatus:329(18339344ns) {DropIncomingHandshake:442(161ns) DropOutgoingHandshake:457(801ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(18252346ns) {ForwardSessionEventToSession:612(18239021ns) ForwardSessionEventToSession:612(1522ns) ForwardSessionEventToSession:612(791ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(828ns) ForwardSessionEventToSession:612(1364ns) ForwardSessionEventToSession:612(816ns) ForwardSessionEventToSession:612(520ns) ForwardSessionEventToSession:612(555ns) ForwardSessionEventToSession:612(732ns)}}}
2025-12-24T02:10:58.306832Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241225839256224:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:58.306984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225839256303:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.306685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241225895329702:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.306725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.306733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241225895329702:2049] [node 2] ICP22 created new session: [4:7587241225895329790:2049]
2025-12-24T02:10:58.306739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241225895329790:2049] [node 2] ICS09 handshake done sender: [4:7587241225895329781:2117] self: [4:7587241225895329788:0] peer: [2:7587241225135515087:0] socket: 279 qp: -1
2025-12-24T02:10:58.306745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241225895329790:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.306754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.303854Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241228118761428:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241228118761521:2120] held: no
2025-12-24T02:10:58.305585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515088:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.305600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241225135515088:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.306440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241225135515088:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.310765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225839256304:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:10:58.310858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241228107217604:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241228107217687:2118] held: no
2025-12-24T02:10:58.310890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241228107217604:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.310900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.310907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228107217604:2048] [node 1] ICP22 created new session: [3:7587241228107217700:2048]
2025-12-24T02:10:58.310916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228107217700:2048] [node 1] ICS09 handshake done sender: [3:7587241228107217687:2118] self: [3:7587241228107217689:0] peer: [1:7587241227614326053:0] socket: 259 qp: -1
2025-12-24T02:10:58.310923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228107217700:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.310931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.311141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241228107217605:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(22459361ns) {HandleHandshakeStatus:329(22459013ns) {DropIncomingHandshake:442(103ns) DropOutgoingHandshake:457(1155ns) SwitchToState:179(49ns) ProcessPendingSessionEvents:495(22398062ns) {ForwardSessionEventToSession:612(22385016ns) ForwardSessionEventToSession:612(1915ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(1653ns) ForwardSessionEventToSession:612(501ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(1404ns)}}}
2025-12-24T02:10:58.310988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241225135515090:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.311002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241225135515090:2114] [node 5] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-12-24T02:10:58.314878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761527:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.315095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241228118761521:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.315202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241228118761428:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.315215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.315223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241228118761428:2048] [node 1] ICP22 created new session: [5:7587241228118761528:2048]
2025-12-24T02:10:58.315229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241228118761528:2048] [node 1] ICS09 handshake done sender: [5:7587241228118761521:2120] self: [5:7587241228118761525:0] peer: [1:7587241227614326045:0] socket: 283 qp: -1
2025-12-24T02:10:58.315237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241228118761528:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.315246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.319444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241227614326058:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.319582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241227614326046:2092] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.319603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.319614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241227614326046:2092] [node 4] ICP22 created new session: [1:7587241227614326084:2092]
2025-12-24T02:10:58.319621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241227614326084:2092] [node 4] ICS09 handshake done sender: [1:7587241227614326058:2116] self: [1:7587241227614326057:0] peer: [4:7587241225895329787:0] socket: 275 qp: -1
2025-12-24T02:10:58.319628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241227614326084:2092] [node 4] ICS10 traffic start
2025-12-24T02:10:58.319637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.323676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241225895329701:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9355) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241225895329780:2116] held: no
2025-12-24T02:10:58.323781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.324099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225839256263:2072] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.324118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.324129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225839256263:2072] [node 5] ICP22 created new session: [6:7587241225839256315:2072]
2025-12-24T02:10:58.324135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225839256315:2072] [node 5] ICS09 handshake done sender: [6:7587241225839256303:2117] self: [6:7587241225839256307:0] peer: [5:7587241228118761498:0] socket: 262 qp: -1
2025-12-24T02:10:58.324142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225839256315:2072] [node 5] ICS10 traffic start
2025-12-24T02:10:58.324150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.324246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225839256223:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.324252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.324257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225839256223:2048] [node 1] ICP22 created new session: [6:7587241225839256317:2048]
2025-12-24T02:10:58.324262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225839256317:2048] [node 1] ICS09 handshake done sender: [6:7587241225839256304:2118] self: [6:7587241225839256305:0] peer: [1:7587241227614326055:0] socket: 263 qp: -1
2025-12-24T02:10:58.324266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225839256317:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.324271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.325917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241225895329780:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.326646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241225895329701:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.326659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.325397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241225135515081:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.325423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.325432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241225135515081:2096] [node 4] ICP22 created new session: [2:7587241225135515114:2096]
2025-12-24T02:10:58.325438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225135515114:2096] [node 4] ICS09 handshake done sender: [2:7587241225135515088:2113] self: [2:7587241225135515087:0] peer: [4:7587241225895329788:0] socket: 278 qp: -1
2025-12-24T02:10:58.325444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225135515114:2096] [node 4] ICS10 traffic start
2025-12-24T02:10:58.325451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.326779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241227614326047:2098] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:58.326665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241225895329701:2048] [node 1] ICP22 created new session: [4:7587241225895329794:2048]
2025-12-24T02:10:58.326671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241225895329794:2048] [node 1] ICS09 handshake done sender: [4:7587241225895329780:2116] self: [4:7587241225895329787:0] peer: [1:7587241227614326057:0] socket: 276 qp: -1
2025-12-24T02:10:58.326676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241225895329794:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.326685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.328416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241225135515090:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.328492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241225135515068:2087] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.328509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.328516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241225135515068:2087] [node 5] ICP22 created new session: [2:7587241225135515117:2087]
2025-12-24T02:10:58.328525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241225135515117:2087] [node 5] ICS09 handshake done sender: [2:7587241225135515090:2114] self: [2:7587241225135515089:0] peer: [5:7587241228118761526:0] socket: 224 qp: -1
2025-12-24T02:10:58.328531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241225135515117:2087] [node 5] ICS10 traffic start
2025-12-24T02:10:58.328538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.328832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241228118761530:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64611
2025-12-24T02:10:58.329560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241228118761429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241228118761522:2121] held: no
2025-12-24T02:10:58.329651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241228118761522:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.329701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241228118761429:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.329709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.329714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241228118761429:2049] [node 2] ICP22 created new session: [5:7587241228118761531:2049]
2025-12-24T02:10:58.329720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241228118761531:2049] [node 2] ICS09 handshake done sender: [5:7587241228118761522:2121] self: [5:7587241228118761526:0] peer: [2:7587241225135515089:0] socket: 233 qp: -1
2025-12-24T02:10:58.329726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241228118761531:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: 2025-12-24T02:10:58.329734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
::ffff:127.1.0.0
2025-12-24T02:10:58.331137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228107217702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47665
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.331710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256320:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.331763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256321:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.333794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228107217676:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.333932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241228107217669:2088] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.333953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.333960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228107217669:2088] [node 6] ICP22 created new session: [3:7587241228107217703:2088]
2025-12-24T02:10:58.333967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228107217703:2088] [node 6] ICS09 handshake done sender: [3:7587241228107217676:2113] self: [3:7587241228107217675:0] peer: [6:7587241225839256308:0] socket: 229 qp: -1
2025-12-24T02:10:58.333977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228107217703:2088] [node 6] ICS10 traffic start
2025-12-24T02:10:58.333985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.335028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329796:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.335049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329797:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.335496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217702:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.335698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225839256225:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12730) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241225839256301:2115] held: no
2025-12-24T02:10:58.335739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225839256301:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.336348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329796:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.336378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241225895329703:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.336494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228107217679:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.336510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241228107217679:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.335921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225839256225:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.335934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.335940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225839256225:2050] [node 3] ICP22 created new session: [6:7587241225839256322:2050]
2025-12-24T02:10:58.335946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225839256322:2050] [node 3] ICS09 handshake done sender: [6:7587241225839256301:2115] self: [6:7587241225839256308:0] peer: [3:7587241228107217675:0] socket: 230 qp: -1
2025-12-24T02:10:58.335952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225839256322:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.335958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.336086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241225839256320:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.337702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241225895329799:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45179
2025-12-24T02:10:58.337999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241225895329796:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.337829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228107217679:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.338208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241225839256262:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.338073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241225895329703:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.338086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.338092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241225895329703:2050] [node 3] ICP22 created new session: [4:7587241225895329800:2050]
2025-12-24T02:10:58.338097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241225895329800:2050] [node 3] ICS09 handshake done sender: [4:7587241225895329796:2120] self: [4:7587241225895329798:0] peer: [3:7587241228107217678:0] socket: 226 qp: -1
2025-12-24T02:10:58.338104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241225895329800:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.338111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.339912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329778:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.339920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241225895329778:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.340644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241228107217677:2090] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.340660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.340666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228107217677:2090] [node 4] ICP22 created new session: [3:7587241228107217707:2090]
2025-12-24T02:10:58.340671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228107217707:2090] [node 4] ICS09 handshake done sender: [3:7587241228107217679:2114] self: [3:7587241228107217678:0] peer: [4:7587241225895329798:0] socket: 252 qp: -1
2025-12-24T02:10:58.340678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228107217707:2090] [node 4] ICS10 traffic start
2025-12-24T02:10:58.340687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.343054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241225839256325:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61537
2025-12-24T02:10:58.345569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241225895329778:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.347135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241225839256320:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.347228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241225839256262:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.347243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.347249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241225839256262:2064] [node 4] ICP22 created new session: [6:7587241225839256326:2064]
2025-12-24T02:10:58.347256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241225839256326:2064] [node 4] ICS09 handshake done sender: [6:7587241225839256320:2122] self: [6:7587241225839256324:0] peer: [4:7587241225895329764:0] socket: 287 qp: -1
2025-12-24T02:10:58.347263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241225839256326:2064] [node 4] ICS10 traffic start
2025-12-24T02:10:58.347273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.348758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241225895329797:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.348938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241225895329751:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.348955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.348961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241225895329751:2095] [node 6] ICP22 created new session: [4:7587241225895329804:2095]
2025-12-24T02:10:58.348968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241225895329804:2095] [node 6] ICS09 handshake done sender: [4:7587241225895329778:2114] self: [4:7587241225895329764:0] peer: [6:7587241225839256324:0] socket: 239 qp: -1
2025-12-24T02:10:58.348974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241225895329804:2095] [node 6] ICS10 traffic start
2025-12-24T02:10:58.348982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.358765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:58.358896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241228118761528:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:10:58.358907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241228118761428:2048] [node 1] ICP30 unregister session Session# [5:7587241228118761528:2048] VirtualId# [5:7587241228118761525:0]
2025-12-24T02:10:58.358913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241228118761528:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.358941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.358962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.358771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:58.358873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241225135515111:2048] [node 1] ICS01 socket: 258 reason# EndOfStream
2025-12-24T02:10:58.358883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241225135515015:2048] [node 1] ICP30 unregister session Session# [2:7587241225135515111:2048] VirtualId# [2:7587241225135515099:0]
2025-12-24T02:10:58.358890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241225135515111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.358944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.358968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.359089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:58.359183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241228107217700:2048] [node 1] ICS01 socket: 259 reason# EndOfStream
2025-12-24T02:10:58.359188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241228107217604:2048] [node 1] ICP30 unregister session Session# [3:7587241228107217700:2048] VirtualId# [3:7587241228107217689:0]
2025-12-24T02:10:58.359193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241228107217700:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.359227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.359248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.362756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:58.362842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225839256317:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-12-24T02:10:58.362850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225839256223:2048] [node 1] ICP30 unregister session Session# [6:7587241225839256317:2048] VirtualId# [6:7587241225839256305:0]
2025-12-24T02:10:58.362856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225839256317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.362910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.362934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.364828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:58.364906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241225895329794:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-12-24T02:10:58.364912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241225895329701:2048] [node 1] ICP30 unregister session Session# [4:7587241225895329794:2048] VirtualId# [4:7587241225895329787:0]
2025-12-24T02:10:58.364918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241225895329794:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.364963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.364985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.366493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:58.366573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241228107217698:2049] [node 2] ICS07 socket disconnect 240 reason# EndOfStream
2025-12-24T02:10:58.366579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241228107217698:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.366616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.366633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241228107217698:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:58.368794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241228107217605:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:10:58.368807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241228107217698:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:58.369350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241228107217605:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:58.369363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241228107217698:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:58.369368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241228107217605:2049] [node 2] ICP30 unregister session Session# [3:7587241228107217698:2049] VirtualId# [3:7587241228107217690:0]
2025-12-24T02:10:58.369386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.369391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241228107217605:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:58.369394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:58.371805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:58.371890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241225839256313:2049] [node 2] ICS07 socket disconnect 261 reason# EndOfStream
2025-12-24T02:10:58.371897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225839256313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.371944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.371964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241225839256313:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:58.372322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:58.372381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241225895329790:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-12-24T02:10:58.372386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241225895329702:2049] [node 2] ICP30 unregister session Session# [4:7587241225895329790:2049] VirtualId# [4:7587241225895329788:0]
2025-12-24T02:10:58.372390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241225895329790:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.372420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.372442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.372159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:58.372226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241228118761531:2049] [node 2] ICS01 socket: 233 reason# EndOfStream
2025-12-24T02:10:58.372231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241228118761429:2049] [node 2] ICP30 unregister session Session# [5:7587241228118761531:2049] VirtualId# [5:7587241228118761526:0]
2025-12-24T02:10:58.372237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241228118761531:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.372262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.372286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.372907Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241225839256224:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:58.372919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225839256313:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:58.372924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225839256224:2049] [node 2] ICP30 unregister session Session# [6:7587241225839256313:2049] VirtualId# [6:7587241225839256306:0]
2025-12-24T02:10:58.372936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.372942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241225839256224:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:58.372944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17437) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:58.380951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241225839256224:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:58.388970Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:58.389066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225839256322:2050] [node 3] ICS01 socket: 230 reason# EndOfStream
2025-12-24T02:10:58.389076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225839256225:2050] [node 3] ICP30 unregister session Session# [6:7587241225839256322:2050] VirtualId# [6:7587241225839256308:0]
2025-12-24T02:10:58.389083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225839256322:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.389142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.389189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.391352Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:58.391445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241228118761512:2050] [node 3] ICS01 socket: 244 reason# EndOfStream
2025-12-24T02:10:58.391453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241228118761430:2050] [node 3] ICP30 unregister session Session# [5:7587241228118761512:2050] VirtualId# [5:7587241228118761510:0]
2025-12-24T02:10:58.391461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241228118761512:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.391519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.391544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.419164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:58.419279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225839256326:2064] [node 4] ICS01 socket: 287 reason# EndOfStream
2025-12-24T02:10:58.419409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225839256262:2064] [node 4] ICP30 unregister session Session# [6:7587241225839256326:2064] VirtualId# [6:7587241225839256324:0]
2025-12-24T02:10:58.419417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225839256326:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.419469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:58.419495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:58.439344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:58.439442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241225839256315:2072] [node 5] ICS01 socket: 262 reason# EndOfStream
2025-12-24T02:10:58.439451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241225839256263:2072] [node 5] ICP30 unregister session Session# [6:7587241225839256315:2072] VirtualId# [6:7587241225839256307:0]
2025-12-24T02:10:58.439459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241225839256315:2072] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.439517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:58.439542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:58.482763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226048359384:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.473715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241227527172229:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.473730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241227527172230:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.488680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226048359383:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:58.478736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241226715674439:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.478774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241226715674440:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.488574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225202009365:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.488584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225202009366:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.488591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225202009367:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.488477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241226508152156:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.488523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241226508152158:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.488606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241226508152157:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.494255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241228312477714:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.494341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241228312477715:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.494348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241228312477716:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.527050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241227527172270:2079] [node 4] ICP01 ready to work
2025-12-24T02:10:58.527095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241227527172271:2080] [node 6] ICP01 ready to work
2025-12-24T02:10:58.527166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241227527172272:2086] [node 5] ICP01 ready to work
2025-12-24T02:10:58.528034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226048359449:2108] [node 6] ICP01 ready to work
2025-12-24T02:10:58.528116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226048359450:2109] [node 5] ICP01 ready to work
2025-12-24T02:10:58.528180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226048359451:2110] [node 4] ICP01 ready to work
2025-12-24T02:10:58.529738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225202009381:2061] [node 6] ICP01 ready to work
2025-12-24T02:10:58.526626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674487:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.534892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172289:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.546177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241225202009365:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16931910ns) {Configure:104(16930880ns) {ProcessConfigured:135(16537904ns) {StartInitialHandshake:157(16537313ns) {DropHandshakes:470(503ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:477(1701ns) {SwitchToState:179(909ns)}}}}}
2025-12-24T02:10:58.546395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225202009426:2074] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.547340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477775:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.547377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477776:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.549139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241225202009381:2061] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19359674ns) {Configure:104(19359329ns) {ProcessConfigured:135(19111488ns) {StartInitialHandshake:157(19111370ns) {DropHandshakes:470(174ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(105ns) PrepareNewSessionHandshake:477(990ns) {SwitchToState:179(142ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.548233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241226508152158:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18625596ns) {Configure:104(18624837ns) {ProcessConfigured:135(18366212ns) {StartInitialHandshake:157(18365807ns) {DropHandshakes:470(351ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(851ns) {SwitchToState:179(510ns)}}}}}
2025-12-24T02:10:58.548382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241226508152225:2067] [node 5] ICP01 ready to work
2025-12-24T02:10:58.548470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241226508152226:2073] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.551937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009437:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.550030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241226715674508:2101] [node 6] ICP01 ready to work
2025-12-24T02:10:58.550122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674487:2095] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.550857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674506:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
2025-12-24T02:10:58.550873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674507:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
2025-12-24T02:10:58.550887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241226715674439:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.552346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241228312477776:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.554536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226048359463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6001
2025-12-24T02:10:58.554551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226048359464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6001
2025-12-24T02:10:58.554562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226048359465:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6001
2025-12-24T02:10:58.558722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152232:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.558871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226048359462:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6001
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.571728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241228312477760:2061] [node 4] ICP01 ready to work
2025-12-24T02:10:58.571793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241228312477714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.572103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241228312477774:2063] [node 5] ICP01 ready to work
2025-12-24T02:10:58.571058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226048359444:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.571079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226048359444:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.571882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241226715674505:2098] [node 5] ICP01 ready to work
2025-12-24T02:10:58.574731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241226715674503:2096] [node 4] ICP01 ready to work
2025-12-24T02:10:58.570890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241227527172289:2097] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.579172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009437:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.579222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225202009367:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.579182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241227527172229:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40045294ns) {Configure:104(40042597ns) {ProcessConfigured:135(39666419ns) {StartInitialHandshake:157(39665157ns) {DropHandshakes:470(1392ns) {DropIncomingHandshake:442(132ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:477(1098ns) {SwitchToState:179(667ns)}}}}}
2025-12-24T02:10:58.579222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241227527172229:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.579344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241227527172277:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.579347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241227527172277:2089] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.581783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241225202009366:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35337256ns) {Configure:104(35336290ns) {ProcessConfigured:135(34975455ns) {StartInitialHandshake:157(34974919ns) {DropHandshakes:470(956ns) {DropIncomingHandshake:442(147ns) DropOutgoingHandshake:457(175ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:477(1140ns) {SwitchToState:179(611ns)}}}}}
2025-12-24T02:10:58.580209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226048359453:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.580221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226048359453:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.580825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226048359453:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.580901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226048359384:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.580916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.580925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226048359384:2049] [node 3] ICP22 created new session: [1:7587241226048359467:2049]
2025-12-24T02:10:58.580931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226048359467:2049] [node 3] ICS09 handshake done sender: [1:7587241226048359453:2111] self: [1:7587241226048359452:0] peer: [3:7587241227527172310:0] socket: 236 qp: -1
2025-12-24T02:10:58.580937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226048359467:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:58.580946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.582858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241227527172277:2089] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.582804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152232:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.582858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241226508152158:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.583041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241227527172272:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.583054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.583061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241227527172272:2086] [node 5] ICP22 created new session: [3:7587241227527172312:2086]
2025-12-24T02:10:58.583068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227527172312:2086] [node 5] ICS09 handshake done sender: [3:7587241227527172277:2089] self: [3:7587241227527172276:0] peer: [5:7587241225202009443:0] socket: 233 qp: -1
2025-12-24T02:10:58.583074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227527172312:2086] [node 5] ICS10 traffic start
2025-12-24T02:10:58.583095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.585246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477786:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.585613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477787:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.587025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172315:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.587052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172318:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.587826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241227527172289:2097] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.587120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226048359455:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.587146Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226048359455:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.587910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226048359455:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.587979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226048359449:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.587990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.587999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226048359449:2108] [node 6] ICP22 created new session: [1:7587241226048359471:2108]
2025-12-24T02:10:58.588006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226048359471:2108] [node 6] ICS09 handshake done sender: [1:7587241226048359455:2112] self: [1:7587241226048359454:0] peer: [6:7587241228312477792:0] socket: 237 qp: -1
2025-12-24T02:10:58.588013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226048359471:2108] [node 6] ICS10 traffic start
2025-12-24T02:10:58.588022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.590778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241227527172275:2088] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.590790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241227527172275:2088] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.594836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241227527172229:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.594819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241226715674503:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19997216ns) {Configure:104(19996275ns) {ProcessConfigured:135(7915716ns) {StartInitialHandshake:157(7914813ns) {DropHandshakes:470(1039ns) {DropIncomingHandshake:442(182ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(402ns) PrepareNewSessionHandshake:477(1539ns) {SwitchToState:179(771ns)}}}}}
2025-12-24T02:10:58.594863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.597419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241226715674505:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25454795ns) {Configure:104(25454168ns) {ProcessConfigured:135(11778890ns) {StartInitialHandshake:157(11778123ns) {DropHandshakes:470(791ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(128ns) PrepareNewSessionHandshake:477(673ns) {SwitchToState:179(308ns)}}}}}
2025-12-24T02:10:58.594871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241227527172229:2048] [node 1] ICP22 created new session: [3:7587241227527172319:2048]
2025-12-24T02:10:58.594877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227527172319:2048] [node 1] ICS09 handshake done sender: [3:7587241227527172289:2097] self: [3:7587241227527172310:0] peer: [1:7587241226048359452:0] socket: 245 qp: -1
2025-12-24T02:10:58.597561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674522:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
2025-12-24T02:10:58.594886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227527172319:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.597574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674523:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
2025-12-24T02:10:58.594909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.595327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172314:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.595341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172316:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.595354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172317:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.598882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009445:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.598949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009446:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.599223Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225202009437:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.599293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009446:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.599322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225202009365:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.599406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225202009367:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.599419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.599423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225202009367:2050] [node 3] ICP22 created new session: [5:7587241225202009448:2050]
2025-12-24T02:10:58.599429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241225202009448:2050] [node 3] ICS09 handshake done sender: [5:7587241225202009437:2109] self: [5:7587241225202009443:0] peer: [3:7587241227527172276:0] socket: 242 qp: -1
2025-12-24T02:10:58.599435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241225202009448:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.599452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.599068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226048359444:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.599274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226048359383:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.599288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.599296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226048359383:2048] [node 2] ICP22 created new session: [1:7587241226048359475:2048]
2025-12-24T02:10:58.599302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226048359475:2048] [node 2] ICS09 handshake done sender: [1:7587241226048359444:2107] self: [1:7587241226048359443:0] peer: [2:7587241226715674514:0] socket: 228 qp: -1
2025-12-24T02:10:58.599309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226048359475:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:58.599318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.607641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241227527172275:2088] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.608473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241227527172270:2079] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.608485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.608490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241227527172270:2079] [node 4] ICP22 created new session: [3:7587241227527172323:2079]
2025-12-24T02:10:58.608494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227527172323:2079] [node 4] ICS09 handshake done sender: [3:7587241227527172275:2088] self: [3:7587241227527172274:0] peer: [4:7587241226508152235:0] socket: 232 qp: -1
2025-12-24T02:10:58.608498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227527172323:2079] [node 4] ICS10 traffic start
2025-12-24T02:10:58.608504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.607323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152237:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.607342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152236:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.607794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241226508152232:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.607881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241226508152158:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.606833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241226715674487:2095] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.607894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.607899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241226508152158:2050] [node 3] ICP22 created new session: [4:7587241226508152238:2050]
2025-12-24T02:10:58.607905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241226508152238:2050] [node 3] ICS09 handshake done sender: [4:7587241226508152232:2113] self: [4:7587241226508152235:0] peer: [3:7587241227527172274:0] socket: 241 qp: -1
2025-12-24T02:10:58.607911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241226508152238:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.607931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.610799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241227527172272:2086] [node 5] ICP35 event processing took too much time PendingConnection:279(27719192ns) {HandleHandshakeStatus:329(27718431ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(393ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(19919314ns) {ForwardSessionEventToSession:612(19907945ns) ForwardSessionEventToSession:612(1509ns) ForwardSessionEventToSession:612(833ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(626ns) ForwardSessionEventToSession:612(902ns) ForwardSessionEventToSession:612(557ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(543ns) ForwardSessionEventToSession:612(1178ns)}}}
2025-12-24T02:10:58.619619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152236:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.619674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241226508152156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.622273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.619101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241226048359383:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19778698ns) {HandleHandshakeStatus:329(19777660ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(390ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(19693071ns) {ForwardSessionEventToSession:612(19677306ns) ForwardSessionEventToSession:612(2148ns) ForwardSessionEventToSession:612(1293ns) ForwardSessionEventToSession:612(873ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(1566ns) ForwardSessionEventToSession:612(1068ns) ForwardSessionEventToSession:612(730ns) ForwardSessionEventToSession:612(708ns) ForwardSessionEventToSession:612(714ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.627341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.633616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226048359457:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.633653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226048359457:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.634737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226048359459:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.634761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226048359459:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.638767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.638839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226048359459:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.638676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241228312477760:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(53292514ns) {Configure:104(53291188ns) {ProcessConfigured:135(35732202ns) {StartInitialHandshake:157(35731173ns) {DropHandshakes:470(768ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:477(1306ns) {SwitchToState:179(865ns)}}}}}
2025-12-24T02:10:58.638763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241228312477775:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.638885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241228312477716:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.638909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477796:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.639019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241226508152236:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.638985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226048359451:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.639009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.639018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226048359451:2110] [node 4] ICP22 created new session: [1:7587241226048359478:2110]
2025-12-24T02:10:58.639024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226048359478:2110] [node 4] ICS09 handshake done sender: [1:7587241226048359459:2114] self: [1:7587241226048359458:0] peer: [4:7587241226508152241:0] socket: 276 qp: -1
2025-12-24T02:10:58.639031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226048359478:2110] [node 4] ICS10 traffic start
2025-12-24T02:10:58.639041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.639101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241226508152156:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.639117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.639123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241226508152156:2048] [node 1] ICP22 created new session: [4:7587241226508152244:2048]
2025-12-24T02:10:58.639139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241226508152244:2048] [node 1] ICS09 handshake done sender: [4:7587241226508152236:2116] self: [4:7587241226508152241:0] peer: [1:7587241226048359458:0] socket: 277 qp: -1
2025-12-24T02:10:58.639146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241226508152244:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.639163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.639251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241227527172279:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.639262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241227527172279:2090] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.639181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241228312477776:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.639933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241228312477714:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.639947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.639953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241228312477714:2048] [node 1] ICP22 created new session: [6:7587241228312477799:2048]
2025-12-24T02:10:58.639958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228312477799:2048] [node 1] ICS09 handshake done sender: [6:7587241228312477776:2107] self: [6:7587241228312477792:0] peer: [1:7587241226048359454:0] socket: 246 qp: -1
2025-12-24T02:10:58.639963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228312477799:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.639980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.640273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241227527172279:2090] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.640351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241227527172271:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.640363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.640369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241227527172271:2080] [node 6] ICP22 created new session: [3:7587241227527172326:2080]
2025-12-24T02:10:58.640373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227527172326:2080] [node 6] ICS09 handshake done sender: [3:7587241227527172279:2090] self: [3:7587241227527172278:0] peer: [6:7587241228312477797:0] socket: 234 qp: -1
2025-12-24T02:10:58.640378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227527172326:2080] [node 6] ICS10 traffic start
2025-12-24T02:10:58.640383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.640707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477798:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.640844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241228312477775:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.640895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241228312477716:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.640906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.640911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241228312477716:2050] [node 3] ICP22 created new session: [6:7587241228312477802:2050]
2025-12-24T02:10:58.640915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228312477802:2050] [node 3] ICS09 handshake done sender: [6:7587241228312477775:2106] self: [6:7587241228312477797:0] peer: [3:7587241227527172278:0] socket: 239 qp: -1
2025-12-24T02:10:58.640919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228312477802:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:58.640924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.642850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241226715674439:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.642878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.642889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241226715674439:2048] [node 1] ICP22 created new session: [2:7587241226715674524:2048]
2025-12-24T02:10:58.642896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241226715674524:2048] [node 1] ICS09 handshake done sender: [2:7587241226715674487:2095] self: [2:7587241226715674514:0] peer: [1:7587241226048359443:0] socket: 229 qp: -1
2025-12-24T02:10:58.642906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241226715674524:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.642926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:58.643948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674522:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.646156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009452:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.643343Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241228312477774:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57673959ns) {Configure:104(57673490ns) {ProcessConfigured:135(52351250ns) {StartInitialHandshake:157(52350348ns) {DropHandshakes:470(599ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(929ns) {SwitchToState:179(479ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.653881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152246:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.653902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152247:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.654168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477806:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.654877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009453:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.654169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226048359457:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.654295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226048359450:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.654306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.654315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226048359450:2109] [node 5] ICP22 created new session: [1:7587241226048359481:2109]
2025-12-24T02:10:58.654321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226048359481:2109] [node 5] ICS09 handshake done sender: [1:7587241226048359457:2113] self: [1:7587241226048359456:0] peer: [5:7587241225202009447:0] socket: 269 qp: -1
2025-12-24T02:10:58.654328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226048359481:2109] [node 5] ICS10 traffic start
2025-12-24T02:10:58.654338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.655093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225202009446:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.655179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225202009365:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.655194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.655199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225202009365:2048] [node 1] ICP22 created new session: [5:7587241225202009454:2048]
2025-12-24T02:10:58.655206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241225202009454:2048] [node 1] ICS09 handshake done sender: [5:7587241225202009446:2117] self: [5:7587241225202009447:0] peer: [1:7587241226048359456:0] socket: 282 qp: -1
2025-12-24T02:10:58.655214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241225202009454:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.655221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.661101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241226715674528:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23322
2025-12-24T02:10:58.668952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241228312477786:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.669009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241228312477774:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.669224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152246:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.668793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009452:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.669434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152242:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.669535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009450:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.669565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225202009426:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.668720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674507:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.673240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241228312477806:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.673269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241228312477760:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.673909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241228312477787:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.673941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241228312477715:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.674730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009428:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.674748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241225202009428:2105] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.675094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241227527172317:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.675150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241227527172230:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.673282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674506:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.674922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674528:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.678719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152228:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.678735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241226508152228:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.678743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152230:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.678756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241226508152230:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.679002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241226508152247:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.679026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241226508152157:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.679618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241226508152252:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10998
2025-12-24T02:10:58.679862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241226508152247:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.681081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241228312477715:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241228312477787:2112] held: no
2025-12-24T02:10:58.678802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674512:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.678813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241226715674512:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.679124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674520:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.679136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241226715674520:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.679586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241226715674512:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.679686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241226715674520:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.679779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241226715674508:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.679794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.679800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241226715674508:2101] [node 6] ICP22 created new session: [2:7587241226715674534:2101]
2025-12-24T02:10:58.679806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241226715674534:2101] [node 6] ICS09 handshake done sender: [2:7587241226715674512:2114] self: [2:7587241226715674511:0] peer: [6:7587241228312477809:0] socket: 258 qp: -1
2025-12-24T02:10:58.679811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241226715674534:2101] [node 6] ICS10 traffic start
2025-12-24T02:10:58.679819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.679935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241226715674503:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.679943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.679946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241226715674503:2096] [node 4] ICP22 created new session: [2:7587241226715674536:2096]
2025-12-24T02:10:58.679948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241226715674536:2096] [node 4] ICS09 handshake done sender: [2:7587241226715674520:2116] self: [2:7587241226715674519:0] peer: [4:7587241226508152251:0] socket: 249 qp: -1
2025-12-24T02:10:58.679951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241226715674536:2096] [node 4] ICS10 traffic start
2025-12-24T02:10:58.679955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.682748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674510:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.682759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241226715674510:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.686831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225202009428:2105] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.686863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009460:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.686929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241225202009381:2061] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.686959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.686966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225202009381:2061] [node 6] ICP22 created new session: [5:7587241225202009461:2061]
2025-12-24T02:10:58.686973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241225202009461:2061] [node 6] ICS09 handshake done sender: [5:7587241225202009428:2105] self: [5:7587241225202009383:0] peer: [6:7587241228312477807:0] socket: 252 qp: -1
2025-12-24T02:10:58.686980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241225202009461:2061] [node 6] ICS10 traffic start
2025-12-24T02:10:58.686990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.688123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225202009460:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.688150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225202009366:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.687530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241226508152157:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.687552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.687570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241226508152157:2049] [node 2] ICP22 created new session: [4:7587241226508152253:2049]
2025-12-24T02:10:58.687577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241226508152253:2049] [node 2] ICS09 handshake done sender: [4:7587241226508152247:2121] self: [4:7587241226508152251:0] peer: [2:7587241226715674519:0] socket: 266 qp: -1
2025-12-24T02:10:58.687584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241226508152253:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.687591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.688890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241226508152157:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:58.689310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241226508152228:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.689444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241226508152230:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.689522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241226508152225:2067] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.689539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.689545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241226508152225:2067] [node 5] ICP22 created new session: [4:7587241226508152257:2067]
2025-12-24T02:10:58.689551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241226508152257:2067] [node 5] ICS09 handshake done sender: [4:7587241226508152228:2111] self: [4:7587241226508152227:0] peer: [5:7587241225202009458:0] socket: 272 qp: -1
2025-12-24T02:10:58.689557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241226508152257:2067] [node 5] ICS10 traffic start
2025-12-24T02:10:58.689563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.689672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241226508152226:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.689680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.689684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241226508152226:2073] [node 6] ICP22 created new session: [4:7587241226508152259:2073]
2025-12-24T02:10:58.689688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241226508152259:2073] [node 6] ICS09 handshake done sender: [4:7587241226508152230:2112] self: [4:7587241226508152229:0] peer: [6:7587241228312477808:0] socket: 289 qp: -1
2025-12-24T02:10:58.689692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241226508152259:2073] [node 6] ICS10 traffic start
2025-12-24T02:10:58.689697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.693914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241226715674521:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.693930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241226715674521:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.693863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477810:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.695142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241225202009426:2074] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63018) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241225202009450:2118] held: no
2025-12-24T02:10:58.696442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241225202009366:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241225202009460:2121] held: no
2025-12-24T02:10:58.695321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241226715674521:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.695390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241226715674505:2098] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.695403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.695409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241226715674505:2098] [node 5] ICP22 created new session: [2:7587241226715674540:2098]
2025-12-24T02:10:58.695414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241226715674540:2098] [node 5] ICS09 handshake done sender: [2:7587241226715674521:2117] self: [2:7587241226715674518:0] peer: [5:7587241225202009463:0] socket: 240 qp: -1
2025-12-24T02:10:58.695420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241226715674540:2098] [node 5] ICS10 traffic start
2025-12-24T02:10:58.695428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.700269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241228312477774:2063] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12308) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241228312477786:2111] held: no
2025-12-24T02:10:58.700305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241228312477760:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63018) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241228312477806:2121] held: no
2025-12-24T02:10:58.700385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477811:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.700532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241228312477786:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:58.700105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241226715674510:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.700180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241226715674440:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.700194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.700201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241226715674440:2049] [node 3] ICP22 created new session: [2:7587241226715674543:2049]
2025-12-24T02:10:58.700206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241226715674543:2049] [node 3] ICS09 handshake done sender: [2:7587241226715674510:2113] self: [2:7587241226715674509:0] peer: [3:7587241227527172328:0] socket: 257 qp: -1
2025-12-24T02:10:58.700211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241226715674543:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:58.700217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.700590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241228312477774:2063] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.700603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:58.700610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241228312477774:2063] [node 5] ICP22 created new session: [6:7587241228312477812:2063]
2025-12-24T02:10:58.700616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228312477812:2063] [node 5] ICS09 handshake done sender: [6:7587241228312477786:2111] self: [6:7587241228312477807:0] peer: [5:7587241225202009383:0] socket: 260 qp: -1
2025-12-24T02:10:58.700622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228312477812:2063] [node 5] ICS10 traffic start
2025-12-24T02:10:58.700628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:58.700743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241228312477787:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.700800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241228312477715:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.700808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.700814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241228312477715:2049] [node 2] ICP22 created new session: [6:7587241228312477814:2049]
2025-12-24T02:10:58.700820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228312477814:2049] [node 2] ICS09 handshake done sender: [6:7587241228312477787:2112] self: [6:7587241228312477809:0] peer: [2:7587241226715674511:0] socket: 261 qp: -1
2025-12-24T02:10:58.700825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228312477814:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.700830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.702899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241227527172329:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10257
2025-12-24T02:10:58.703908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241227527172230:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6747) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241227527172317:2120] held: no
2025-12-24T02:10:58.703952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241227527172317:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.710994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241227527172230:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.711021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.711033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241227527172230:2049] [node 2] ICP22 created new session: [3:7587241227527172330:2049]
2025-12-24T02:10:58.711040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241227527172330:2049] [node 2] ICS09 handshake done sender: [3:7587241227527172317:2120] self: [3:7587241227527172328:0] peer: [2:7587241226715674509:0] socket: 273 qp: -1
2025-12-24T02:10:58.711051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241227527172330:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.711062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.712464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241228312477816:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64856
2025-12-24T02:10:58.713808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241228312477806:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.718733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009464:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.718802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225202009465:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5168
2025-12-24T02:10:58.718745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241228312477760:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.718769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.718778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241228312477760:2061] [node 4] ICP22 created new session: [6:7587241228312477817:2061]
2025-12-24T02:10:58.718785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241228312477817:2061] [node 4] ICS09 handshake done sender: [6:7587241228312477806:2121] self: [6:7587241228312477808:0] peer: [4:7587241226508152229:0] socket: 290 qp: -1
2025-12-24T02:10:58.718795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241228312477817:2061] [node 4] ICS10 traffic start
2025-12-24T02:10:58.719008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225202009460:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:58.719062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225202009450:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:58.719091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225202009366:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.718808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.719102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:58.719114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225202009366:2049] [node 2] ICP22 created new session: [5:7587241225202009466:2049]
2025-12-24T02:10:58.719120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241225202009466:2049] [node 2] ICS09 handshake done sender: [5:7587241225202009460:2121] self: [5:7587241225202009463:0] peer: [2:7587241226715674518:0] socket: 248 qp: -1
2025-12-24T02:10:58.719139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241225202009466:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:58.719146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:58.720011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225202009426:2074] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:58.720021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:58.720026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225202009426:2074] [node 4] ICP22 created new session: [5:7587241225202009468:2074]
2025-12-24T02:10:58.720032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241225202009468:2074] [node 4] ICS09 handshake done sender: [5:7587241225202009450:2118] self: [5:7587241225202009458:0] peer: [4:7587241226508152227:0] socket: 250 qp: -1
2025-12-24T02:10:58.720037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241225202009468:2074] [node 4] ICS10 traffic start
2025-12-24T02:10:58.720043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:58.742828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:58.742929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241226715674524:2048] [node 1] ICS01 socket: 229 reason# EndOfStream
2025-12-24T02:10:58.742939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241226715674439:2048] [node 1] ICP30 unregister session Session# [2:7587241226715674524:2048] VirtualId# [2:7587241226715674514:0]
2025-12-24T02:10:58.742948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241226715674524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.743015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.743038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.749162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:58.749246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241226508152244:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-12-24T02:10:58.749254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241226508152156:2048] [node 1] ICP30 unregister session Session# [4:7587241226508152244:2048] VirtualId# [4:7587241226508152241:0]
2025-12-24T02:10:58.749260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241226508152244:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.749321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.749344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.749283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:58.749354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241228312477799:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-12-24T02:10:58.749360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241228312477714:2048] [node 1] ICP30 unregister session Session# [6:7587241228312477799:2048] VirtualId# [6:7587241228312477792:0]
2025-12-24T02:10:58.749363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241228312477799:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.749398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.749417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.750770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:58.750850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241227527172319:2048] [node 1] ICS01 socket: 245 reason# EndOfStream
2025-12-24T02:10:58.750857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241227527172229:2048] [node 1] ICP30 unregister session Session# [3:7587241227527172319:2048] VirtualId# [3:7587241227527172310:0]
2025-12-24T02:10:58.750864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241227527172319:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.750922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.750946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.750981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:58.751048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241225202009454:2048] [node 1] ICS01 socket: 282 reason# EndOfStream
2025-12-24T02:10:58.751052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225202009365:2048] [node 1] ICP30 unregister session Session# [5:7587241225202009454:2048] VirtualId# [5:7587241225202009447:0]
2025-12-24T02:10:58.751057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241225202009454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.751085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:58.751106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:58.762795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:58.762893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241227527172330:2049] [node 2] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:10:58.762904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241227527172230:2049] [node 2] ICP30 unregister session Session# [3:7587241227527172330:2049] VirtualId# [3:7587241227527172328:0]
2025-12-24T02:10:58.762911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241227527172330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.762998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.763026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.768281Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:58.768364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241228312477814:2049] [node 2] ICS01 socket: 261 reason# EndOfStream
2025-12-24T02:10:58.768373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241228312477715:2049] [node 2] ICP30 unregister session Session# [6:7587241228312477814:2049] VirtualId# [6:7587241228312477809:0]
2025-12-24T02:10:58.768379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241228312477814:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.768431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.768451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.769263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:58.769329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241226508152253:2049] [node 2] ICS01 socket: 266 reason# EndOfStream
2025-12-24T02:10:58.769335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241226508152157:2049] [node 2] ICP30 unregister session Session# [4:7587241226508152253:2049] VirtualId# [4:7587241226508152251:0]
2025-12-24T02:10:58.769340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241226508152253:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.769380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.769401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.772728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:58.772814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241225202009466:2049] [node 2] ICS01 socket: 248 reason# EndOfStream
2025-12-24T02:10:58.772822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225202009366:2049] [node 2] ICP30 unregister session Session# [5:7587241225202009466:2049] VirtualId# [5:7587241225202009463:0]
2025-12-24T02:10:58.772828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241225202009466:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.772885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:58.772911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:58.786784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:58.786885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241225202009448:2050] [node 3] ICS01 socket: 242 reason# EndOfStream
2025-12-24T02:10:58.786894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225202009367:2050] [node 3] ICP30 unregister session Session# [5:7587241225202009448:2050] VirtualId# [5:7587241225202009443:0]
2025-12-24T02:10:58.786901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241225202009448:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.786969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.786997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.787224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:58.787304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241228312477802:2050] [node 3] ICS01 socket: 239 reason# EndOfStream
2025-12-24T02:10:58.787311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241228312477716:2050] [node 3] ICP30 unregister session Session# [6:7587241228312477802:2050] VirtualId# [6:7587241228312477797:0]
2025-12-24T02:10:58.787316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241228312477802:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.787367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:58.787389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:58.801551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:58.801636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241228312477817:2061] [node 4] ICS01 socket: 290 reason# EndOfStream
2025-12-24T02:10:58.801644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241228312477760:2061] [node 4] ICP30 unregister session Session# [6:7587241228312477817:2061] VirtualId# [6:7587241228312477808:0]
2025-12-24T02:10:58.801650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241228312477817:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:58.801709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:58.801732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:58.849185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226219354421:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:58.849195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226219354422:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.850837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241229309271322:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.850852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241229309271323:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:58.853302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228774571298:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.853313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228774571299:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.854582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241227023995504:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.854590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241227023995505:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.854594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241227023995506:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.862851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225986513165:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.862862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225986513166:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.862867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225986513167:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.902796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241229375484881:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:58.906778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241229375484879:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:58.914730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241229375484880:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:58.946995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241229375484922:2068] [node 4] ICP01 ready to work
2025-12-24T02:10:58.947010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241229375484926:2070] [node 5] ICP01 ready to work
2025-12-24T02:10:58.946962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226219354483:2107] [node 4] ICP01 ready to work
2025-12-24T02:10:58.947064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241225986513226:2106] [node 6] ICP01 ready to work
2025-12-24T02:10:58.947981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226219354484:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:58.950963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241226219354488:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:58.951715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241229309271385:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.952304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225986513229:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:58.952609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228774571360:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:10:58.957772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241229309271393:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:58.954930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241227023995566:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:58.955046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241227023995565:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:58.961579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241229309271383:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.966173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241229375484879:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19622237ns) {Configure:104(19620935ns) {ProcessConfigured:135(18997643ns) {StartInitialHandshake:157(18996714ns) {DropHandshakes:470(1296ns) {DropIncomingHandshake:442(169ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(886ns) {SwitchToState:179(461ns)}}}}}
2025-12-24T02:10:58.968836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241229309271396:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57416
2025-12-24T02:10:58.970633Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241228774571299:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20621560ns) {Configure:104(13406963ns) {ProcessConfigured:135(13145055ns) {StartInitialHandshake:157(13144806ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:477(1114ns) {SwitchToState:179(734ns)}}}}}
2025-12-24T02:10:58.969812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241229375484922:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22740241ns) {Configure:104(22739667ns) {ProcessConfigured:135(22144776ns) {StartInitialHandshake:157(22144616ns) {DropHandshakes:470(358ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:477(452ns) {SwitchToState:179(153ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.972246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226219354499:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28701
2025-12-24T02:10:58.972255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228774571367:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:58.972524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241229375484956:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
2025-12-24T02:10:58.974801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241227023995566:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.977003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241229375484956:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.977036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241229375484879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.978773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226219354490:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.978787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226219354490:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.980142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226219354490:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:58.980240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226219354488:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.980254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.980261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226219354488:2109] [node 6] ICP22 created new session: [1:7587241226219354501:2109]
2025-12-24T02:10:58.980267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226219354501:2109] [node 6] ICS09 handshake done sender: [1:7587241226219354490:2110] self: [1:7587241226219354489:0] peer: [6:7587241229375484957:0] socket: 234 qp: -1
2025-12-24T02:10:58.980274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226219354501:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:58.980293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.977261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228774571367:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.977439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241228774571369:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:58.977804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241228774571369:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.979899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241229309271393:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18266705ns) {Configure:104(18266323ns) {ProcessConfigured:135(18002565ns) {StartInitialHandshake:157(18002196ns) {DropHandshakes:470(557ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(775ns) {SwitchToState:179(328ns)}}}}}
2025-12-24T02:10:58.980032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241229375484926:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32967444ns) {Configure:104(32966938ns) {ProcessConfigured:135(32153212ns) {StartInitialHandshake:157(32152633ns) {DropHandshakes:470(292ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(92ns) PrepareNewSessionHandshake:477(574ns) {SwitchToState:179(144ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.982946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241227023995565:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.982855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241225986513232:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:58.981399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241229309271396:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.983527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225986513232:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.983880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225986513165:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.984090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226219354495:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.984100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226219354495:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.986268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228774571371:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:58.986511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228774571371:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.988557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225986513229:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.988905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225986513166:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.989993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226219354499:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.990152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241227023995504:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.989314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241229375484942:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.989327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241229375484942:2104] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.993000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241229375484959:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
2025-12-24T02:10:58.993068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241229375484960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.993208Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241229309271323:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35389828ns) {Configure:104(35389077ns) {ProcessConfigured:135(35176903ns) {StartInitialHandshake:157(35176647ns) {DropHandshakes:470(194ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(40ns) PrepareNewSessionHandshake:477(890ns) {SwitchToState:179(456ns)}}}}}
2025-12-24T02:10:58.993336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241227023995569:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:58.993438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241228774571298:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43792867ns) {Configure:104(43792426ns) {ProcessConfigured:135(43453689ns) {StartInitialHandshake:157(43453475ns) {DropHandshakes:470(492ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(428ns) {SwitchToState:179(44ns)}}}}}
2025-12-24T02:10:58.993596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241228774571298:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.993941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226219354497:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.993952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226219354497:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.994725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241226219354504:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28701
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.995351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241228774571375:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:58.995328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241229375484960:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.995364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241229375484880:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.995446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226219354497:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:58.995573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228774571371:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:58.995486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241229309271398:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.995494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241229309271398:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.996108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241229309271398:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.996656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241227023995570:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:58.996196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241229309271383:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.996213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:58.996220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241229309271383:2107] [node 6] ICP22 created new session: [2:7587241229309271404:2107]
2025-12-24T02:10:58.996226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241229309271404:2107] [node 6] ICS09 handshake done sender: [2:7587241229309271398:2113] self: [2:7587241229309271397:0] peer: [6:7587241229375484962:0] socket: 256 qp: -1
2025-12-24T02:10:58.996232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241229309271404:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:58.996239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:58.998371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241227023995570:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.998733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241228774571298:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:58.998779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241229375484959:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.999095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241229375484880:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241229375484960:2117] held: no
2025-12-24T02:10:58.999159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241229375484926:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:58.999878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241229309271387:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.999893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241229309271387:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:58.999675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226219354422:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:58.999690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:58.999696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226219354422:2049] [node 3] ICP22 created new session: [1:7587241226219354506:2049]
2025-12-24T02:10:58.999701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241226219354506:2049] [node 3] ICS09 handshake done sender: [1:7587241226219354497:2113] self: [1:7587241226219354496:0] peer: [3:7587241228774571374:0] socket: 263 qp: -1
2025-12-24T02:10:58.999706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241226219354506:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:58.999713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:58.999928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241226219354492:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:58.999934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241226219354492:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.000948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241229375484961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
2025-12-24T02:10:58.998758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:58.998766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228774571298:2048] [node 1] ICP22 created new session: [3:7587241228774571376:2048]
2025-12-24T02:10:58.998772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241228774571376:2048] [node 1] ICS09 handshake done sender: [3:7587241228774571371:2112] self: [3:7587241228774571374:0] peer: [1:7587241226219354496:0] socket: 264 qp: -1
2025-12-24T02:10:58.998778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241228774571376:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:58.998795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.000848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241228774571362:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.000859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241228774571362:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.002760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241229375484963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
2025-12-24T02:10:59.004175Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241226219354421:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45353384ns) {Configure:104(45352722ns) {ProcessConfigured:135(44976733ns) {StartInitialHandshake:157(44976241ns) {DropHandshakes:470(342ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(755ns) {SwitchToState:179(344ns)}}}}}
2025-12-24T02:10:59.005413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241227023995506:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.005465Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241227023995506:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16155195ns) {Configure:104(16154882ns) {ProcessConfigured:135(15599270ns) {IncomingHandshake:261(15590105ns) {DropIncomingHandshake:442(30ns) PrepareNewSessionHandshake:477(284ns) {SwitchToState:179(92ns)} GenerateSessionVirtualId:988(399ns) IssueIncomingHandshakeReply:193(15586260ns) {SwitchToState:179(584ns)}}}}}
2025-12-24T02:10:59.006821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241227023995569:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.007013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231318962869:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:59.007278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241229309271400:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.007293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241229309271400:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.007336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231318962869:2112] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.007340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241227023995505:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.006895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226219354495:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.008616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226219354484:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.008631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.008637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226219354484:2108] [node 5] ICP22 created new session: [1:7587241230514321808:2108]
2025-12-24T02:10:59.008642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230514321808:2108] [node 5] ICS09 handshake done sender: [1:7587241226219354495:2112] self: [1:7587241226219354486:0] peer: [5:7587241225986513233:0] socket: 253 qp: -1
2025-12-24T02:10:59.008649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230514321808:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:59.008656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.009264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241229375484942:2104] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.010197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241233604238703:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57416
2025-12-24T02:10:59.010268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241233604238704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57416
2025-12-24T02:10:59.011004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241233604238704:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.011057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241229309271387:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.011036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241229375484954:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.011045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241229375484954:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.011338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241229309271385:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.011350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.011356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241229309271385:2108] [node 5] ICP22 created new session: [2:7587241233604238706:2108]
2025-12-24T02:10:59.011361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241233604238706:2108] [node 5] ICS09 handshake done sender: [2:7587241229309271387:2109] self: [2:7587241229309271386:0] peer: [5:7587241225986513234:0] socket: 237 qp: -1
2025-12-24T02:10:59.011367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241233604238706:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:59.011373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.011492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241229309271322:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.011893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231318962871:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:59.013973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231318962872:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:59.014903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228774571362:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.014910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230514321804:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.014920Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230514321804:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.015430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241226219354492:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.015690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241226219354422:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:59.015891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230514321804:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.015982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226219354421:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.016004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.016010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226219354421:2048] [node 2] ICP22 created new session: [1:7587241230514321812:2048]
2025-12-24T02:10:59.016015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230514321812:2048] [node 2] ICS09 handshake done sender: [1:7587241230514321804:2116] self: [1:7587241226219354494:0] peer: [2:7587241233604238708:0] socket: 275 qp: -1
2025-12-24T02:10:59.016022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230514321812:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:59.016028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.014875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241233604238703:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.016325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241226219354483:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.016341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.016348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241226219354483:2107] [node 4] ICP22 created new session: [1:7587241230514321814:2107]
2025-12-24T02:10:59.016354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230514321814:2107] [node 4] ICS09 handshake done sender: [1:7587241226219354492:2111] self: [1:7587241226219354485:0] peer: [4:7587241227023995568:0] socket: 240 qp: -1
2025-12-24T02:10:59.016360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230514321814:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:59.016365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.017789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241229375484960:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.018794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241227023995566:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.017849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241225986513230:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.017861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241225986513230:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.023460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241229309271322:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241233604238704:2117] held: no
2025-12-24T02:10:59.023546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241233604238710:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57416
2025-12-24T02:10:59.024220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241233604238704:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.024296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241229309271322:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.024312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.024320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241229309271322:2048] [node 1] ICP22 created new session: [2:7587241233604238712:2048]
2025-12-24T02:10:59.024326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241233604238712:2048] [node 1] ICS09 handshake done sender: [2:7587241233604238704:2117] self: [2:7587241233604238708:0] peer: [1:7587241226219354494:0] socket: 267 qp: -1
2025-12-24T02:10:59.024333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241233604238712:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.024339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.031096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241233069538675:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:59.030909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480531:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.031216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480532:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.032430Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225986513229:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.032569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225986513232:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.034933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241229309271400:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.035051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241229309271393:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.035066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.035074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241229309271393:2111] [node 4] ICP22 created new session: [2:7587241233604238715:2111]
2025-12-24T02:10:59.035080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241233604238715:2111] [node 4] ICS09 handshake done sender: [2:7587241229309271400:2114] self: [2:7587241229309271395:0] peer: [4:7587241227023995571:0] socket: 272 qp: -1
2025-12-24T02:10:59.035086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241233604238715:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:59.035098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.037179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241229375484881:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.037196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.037202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241229375484881:2050] [node 3] ICP22 created new session: [6:7587241233670452263:2050]
2025-12-24T02:10:59.037208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233670452263:2050] [node 3] ICS09 handshake done sender: [6:7587241229375484942:2104] self: [6:7587241229375484936:0] peer: [3:7587241228774571370:0] socket: 235 qp: -1
2025-12-24T02:10:59.037214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233670452263:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.037221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.037324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241229375484956:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.037562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241229375484880:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.037573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.037578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241229375484880:2049] [node 2] ICP22 created new session: [6:7587241233670452265:2049]
2025-12-24T02:10:59.037581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233670452265:2049] [node 2] ICS09 handshake done sender: [6:7587241229375484960:2117] self: [6:7587241229375484962:0] peer: [2:7587241229309271397:0] socket: 265 qp: -1
2025-12-24T02:10:59.037585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233670452265:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.037589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.037790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241229375484954:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.037960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225986513165:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.037976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.037983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225986513165:2048] [node 1] ICP22 created new session: [5:7587241230281480536:2048]
2025-12-24T02:10:59.037990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230281480536:2048] [node 1] ICS09 handshake done sender: [5:7587241225986513232:2111] self: [5:7587241225986513233:0] peer: [1:7587241226219354486:0] socket: 254 qp: -1
2025-12-24T02:10:59.037996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230281480536:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.038003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.037938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241229375484922:2068] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.037952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.037956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241229375484922:2068] [node 4] ICP22 created new session: [6:7587241233670452269:2068]
2025-12-24T02:10:59.037959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233670452269:2068] [node 4] ICS09 handshake done sender: [6:7587241229375484954:2113] self: [6:7587241229375484937:0] peer: [4:7587241231318962870:0] socket: 259 qp: -1
2025-12-24T02:10:59.037964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233670452269:2068] [node 4] ICS10 traffic start
2025-12-24T02:10:59.037969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.038053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241229375484879:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.038060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.038064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241229375484879:2048] [node 1] ICP22 created new session: [6:7587241233670452271:2048]
2025-12-24T02:10:59.038067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233670452271:2048] [node 1] ICS09 handshake done sender: [6:7587241229375484956:2114] self: [6:7587241229375484957:0] peer: [1:7587241226219354489:0] socket: 243 qp: -1
2025-12-24T02:10:59.038070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233670452271:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.038074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.037392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241229309271385:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26026725ns) {HandleHandshakeStatus:329(26026340ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(135ns) SwitchToState:179(87ns) ProcessPendingSessionEvents:495(25959106ns) {ForwardSessionEventToSession:612(25946383ns) ForwardSessionEventToSession:612(1409ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(400ns) ForwardSessionEventToSession:612(411ns) ForwardSessionEventToSession:612(1631ns) ForwardSessionEventToSession:612(459ns) ForwardSessionEventToSession:612(400ns) ForwardSessionEventToSession:612(469ns) ForwardSessionEventToSession:612(436ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.039074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241230514321816:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28701
2025-12-24T02:10:59.041677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225986513166:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.041703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.041714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225986513166:2049] [node 2] ICP22 created new session: [5:7587241230281480538:2049]
2025-12-24T02:10:59.041721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230281480538:2049] [node 2] ICS09 handshake done sender: [5:7587241225986513229:2109] self: [5:7587241225986513234:0] peer: [2:7587241229309271386:0] socket: 238 qp: -1
2025-12-24T02:10:59.041729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230281480538:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.041762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.041726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241228774571367:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.041828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241228774571360:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.041840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.041846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228774571360:2107] [node 4] ICP22 created new session: [3:7587241233069538677:2107]
2025-12-24T02:10:59.041852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241233069538677:2107] [node 4] ICS09 handshake done sender: [3:7587241228774571362:2108] self: [3:7587241228774571361:0] peer: [4:7587241227023995567:0] socket: 241 qp: -1
2025-12-24T02:10:59.041862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241233069538677:2107] [node 4] ICS10 traffic start
2025-12-24T02:10:59.041879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.048321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241227023995565:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.048441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241227023995504:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.048451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.048458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241227023995504:2048] [node 1] ICP22 created new session: [4:7587241231318962874:2048]
2025-12-24T02:10:59.048464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231318962874:2048] [node 1] ICS09 handshake done sender: [4:7587241227023995565:2108] self: [4:7587241227023995568:0] peer: [1:7587241226219354485:0] socket: 242 qp: -1
2025-12-24T02:10:59.048470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231318962874:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.048488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.048770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231318962876:2115] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.048935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231318962877:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.049098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231318962878:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:59.049502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241227023995570:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.049657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241227023995569:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.049714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241227023995505:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.049728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.049734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241227023995505:2049] [node 2] ICP22 created new session: [4:7587241231318962881:2049]
2025-12-24T02:10:59.049748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231318962881:2049] [node 2] ICS09 handshake done sender: [4:7587241227023995570:2111] self: [4:7587241227023995571:0] peer: [2:7587241229309271395:0] socket: 273 qp: -1
2025-12-24T02:10:59.049754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231318962881:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.049760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.049872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231318962869:2112] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.049882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.049887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231318962869:2112] [node 6] ICP22 created new session: [4:7587241231318962883:2112]
2025-12-24T02:10:59.049891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231318962883:2112] [node 6] ICS09 handshake done sender: [4:7587241227023995569:2110] self: [4:7587241231318962870:0] peer: [6:7587241229375484937:0] socket: 261 qp: -1
2025-12-24T02:10:59.049896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231318962883:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:59.049901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.047910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230281480541:2115] [node 4] ICP01 ready to work
2025-12-24T02:10:59.049599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241225986513230:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.050319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241225986513226:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.050329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.050335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225986513226:2106] [node 6] ICP22 created new session: [5:7587241230281480544:2106]
2025-12-24T02:10:59.050339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230281480544:2106] [node 6] ICS09 handshake done sender: [5:7587241225986513230:2110] self: [5:7587241225986513227:0] peer: [6:7587241229375484964:0] socket: 247 qp: -1
2025-12-24T02:10:59.050345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230281480544:2106] [node 6] ICS10 traffic start
2025-12-24T02:10:59.050361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.051957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241228774571369:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.051974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.051982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228774571369:2111] [node 6] ICP22 created new session: [3:7587241233069538680:2111]
2025-12-24T02:10:59.051988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241233069538680:2111] [node 6] ICS09 handshake done sender: [3:7587241228774571367:2110] self: [3:7587241228774571370:0] peer: [6:7587241229375484936:0] socket: 244 qp: -1
2025-12-24T02:10:59.051994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241233069538680:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:59.052009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.052415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241233069538682:2117] [node 5] ICP01 ready to work
2025-12-24T02:10:59.052488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241233069538676:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:59.053203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241233069538676:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.054547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241228774571299:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.054710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241229309271402:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.054723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241229309271402:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.059233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241233069538687:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
2025-12-24T02:10:59.061933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241227023995506:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.061960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.061972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241227023995506:2050] [node 3] ICP22 created new session: [4:7587241231318962886:2050]
2025-12-24T02:10:59.061979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231318962886:2050] [node 3] ICS09 handshake done sender: [4:7587241227023995566:2109] self: [4:7587241227023995567:0] peer: [3:7587241228774571361:0] socket: 245 qp: -1
2025-12-24T02:10:59.061990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231318962886:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.062017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.062359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241233069538687:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.070912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480546:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.073056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241227023995506:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(46289676ns) {HandleHandshakeStatus:329(46289245ns) {DropIncomingHandshake:442(903ns) DropOutgoingHandshake:457(17ns) SwitchToState:179(118ns) ProcessPendingSessionEvents:495(572ns)}}
2025-12-24T02:10:59.073885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241229309271402:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.072060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.072817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230281480548:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.072850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241225986513167:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.073154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230281480546:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.073172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241225986513226:2106] [node 6] ICP18 handshake (actor [5:7587241230281480546:2116]) is too old
2025-12-24T02:10:59.074869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.075546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241229309271323:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.075568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.075575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241229309271323:2049] [node 3] ICP22 created new session: [2:7587241233604238718:2049]
2025-12-24T02:10:59.075581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241233604238718:2049] [node 3] ICS09 handshake done sender: [2:7587241229309271402:2115] self: [2:7587241229309271394:0] peer: [3:7587241233069538686:0] socket: 279 qp: -1
2025-12-24T02:10:59.075587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241233604238718:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:59.075596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.078622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241228774571299:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241233069538676:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.078931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233670452273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21081
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.079186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231318962888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18699
2025-12-24T02:10:59.078772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241230281480541:2115] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30794983ns) {Configure:104(30794185ns) {ProcessConfigured:135(30464487ns) {StartInitialHandshake:157(30463730ns) {DropHandshakes:470(656ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(743ns) {SwitchToState:179(531ns)}}}}}
2025-12-24T02:10:59.079071Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241225986513166:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(41146038ns) {HandleHandshakeStatus:329(41145361ns) {DropIncomingHandshake:442(914ns) DropOutgoingHandshake:457(130ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(16267233ns) {ForwardSessionEventToSession:612(16265585ns)}}}
2025-12-24T02:10:59.083049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241233069538684:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.083066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241233069538684:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.084205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230281480547:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.084241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241230281480541:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.084340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231318962888:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.086869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241229375484926:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30282) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241229375484959:2116] held: no
2025-12-24T02:10:59.090271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241229375484959:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.090324Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241228774571360:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(75246549ns) {HandleHandshakeStatus:329(75245828ns) {DropIncomingHandshake:442(421ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(27772687ns) {ForwardSessionEventToSession:612(27761077ns) ForwardSessionEventToSession:612(1873ns) ForwardSessionEventToSession:612(800ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(969ns) ForwardSessionEventToSession:612(818ns) ForwardSessionEventToSession:612(1225ns) ForwardSessionEventToSession:612(477ns)}}}
2025-12-24T02:10:59.090559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231318962880:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.090579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241231318962880:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.090633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241229375484926:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.090647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.090654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241229375484926:2070] [node 5] ICP22 created new session: [6:7587241233670452274:2070]
2025-12-24T02:10:59.090662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233670452274:2070] [node 5] ICS09 handshake done sender: [6:7587241229375484959:2116] self: [6:7587241229375484964:0] peer: [5:7587241225986513227:0] socket: 260 qp: -1
2025-12-24T02:10:59.090669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233670452274:2070] [node 5] ICS10 traffic start
2025-12-24T02:10:59.090681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.100299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241233069538684:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.100427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241233069538682:2117] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.100449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.100457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241233069538682:2117] [node 5] ICP22 created new session: [3:7587241233069538690:2117]
2025-12-24T02:10:59.100465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241233069538690:2117] [node 5] ICS09 handshake done sender: [3:7587241233069538684:2118] self: [3:7587241233069538683:0] peer: [5:7587241230281480549:0] socket: 297 qp: -1
2025-12-24T02:10:59.100471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241233069538690:2117] [node 5] ICS10 traffic start
2025-12-24T02:10:59.100480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.105644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241233069538692:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.108975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241233069538676:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.109386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231318962880:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.109491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241228774571299:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.109505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.109511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241228774571299:2049] [node 2] ICP22 created new session: [3:7587241233069538693:2049]
2025-12-24T02:10:59.109518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241233069538693:2049] [node 2] ICS09 handshake done sender: [3:7587241233069538676:2116] self: [3:7587241233069538686:0] peer: [2:7587241229309271394:0] socket: 252 qp: -1
2025-12-24T02:10:59.109525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241233069538693:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.109535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.110491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241231318962876:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.110517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.110523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231318962876:2115] [node 5] ICP22 created new session: [4:7587241231318962891:2115]
2025-12-24T02:10:59.110529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231318962891:2115] [node 5] ICS09 handshake done sender: [4:7587241231318962880:2118] self: [4:7587241231318962879:0] peer: [5:7587241230281480552:0] socket: 295 qp: -1
2025-12-24T02:10:59.110537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231318962891:2115] [node 5] ICS10 traffic start
2025-12-24T02:10:59.110543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.108761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480554:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.108786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230281480553:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19062
2025-12-24T02:10:59.109013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230281480548:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.109122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241225986513167:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.109138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.109147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241225986513167:2050] [node 3] ICP22 created new session: [5:7587241230281480555:2050]
2025-12-24T02:10:59.109154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230281480555:2050] [node 3] ICS09 handshake done sender: [5:7587241230281480548:2118] self: [5:7587241230281480549:0] peer: [3:7587241233069538683:0] socket: 299 qp: -1
2025-12-24T02:10:59.109161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230281480555:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.109169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.115104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230281480547:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.119813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241230281480541:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.119832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.119838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230281480541:2115] [node 4] ICP22 created new session: [5:7587241230281480557:2115]
2025-12-24T02:10:59.119844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230281480557:2115] [node 4] ICS09 handshake done sender: [5:7587241230281480547:2117] self: [5:7587241230281480552:0] peer: [4:7587241231318962879:0] socket: 298 qp: -1
2025-12-24T02:10:59.119851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230281480557:2115] [node 4] ICS10 traffic start
2025-12-24T02:10:59.119860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.130859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241231318962876:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(20329504ns) {HandleHandshakeStatus:329(20328639ns) {DropIncomingHandshake:442(9504ns) DropOutgoingHandshake:457(126ns) SwitchToState:179(87ns) ProcessPendingSessionEvents:495(20235175ns) {ForwardSessionEventToSession:612(20224512ns) ForwardSessionEventToSession:612(2055ns) ForwardSessionEventToSession:612(588ns) ForwardSessionEventToSession:612(845ns) ForwardSessionEventToSession:612(1275ns) ForwardSessionEventToSession:612(877ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(726ns) ForwardSessionEventToSession:612(720ns)}}}
2025-12-24T02:10:59.150801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:59.150907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241228774571376:2048] [node 1] ICS01 socket: 264 reason# EndOfStream
2025-12-24T02:10:59.150917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241228774571298:2048] [node 1] ICP30 unregister session Session# [3:7587241228774571376:2048] VirtualId# [3:7587241228774571374:0]
2025-12-24T02:10:59.150924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241228774571376:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.151001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.151034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.151084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:59.151164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241233604238712:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:10:59.151170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241229309271322:2048] [node 1] ICP30 unregister session Session# [2:7587241233604238712:2048] VirtualId# [2:7587241233604238708:0]
2025-12-24T02:10:59.151175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241233604238712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.151211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.151233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.158185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:59.158278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241233670452271:2048] [node 1] ICS07 socket disconnect 243 reason# EndOfStream
2025-12-24T02:10:59.158285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233670452271:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.158351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.158377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241233670452271:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:59.158792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241229375484879:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.158804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233670452271:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.158808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241229375484879:2048] [node 1] ICP30 unregister session Session# [6:7587241233670452271:2048] VirtualId# [6:7587241229375484957:0]
2025-12-24T02:10:59.158822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.158828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241229375484879:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.158831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.158777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:59.158858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241230281480536:2048] [node 1] ICS07 socket disconnect 254 reason# EndOfStream
2025-12-24T02:10:59.158863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230281480536:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.158916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.158937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241230281480536:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:59.158763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:59.158931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241231318962874:2048] [node 1] ICS07 socket disconnect 242 reason# EndOfStream
2025-12-24T02:10:59.158935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231318962874:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.158990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.159012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241231318962874:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:59.159327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241227023995504:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.159334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231318962874:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.159338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241227023995504:2048] [node 1] ICP30 unregister session Session# [4:7587241231318962874:2048] VirtualId# [4:7587241227023995568:0]
2025-12-24T02:10:59.159351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.159355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241227023995504:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.159357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.163115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:59.163196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241233069538693:2049] [node 2] ICS01 socket: 252 reason# EndOfStream
2025-12-24T02:10:59.163202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241228774571299:2049] [node 2] ICP30 unregister session Session# [3:7587241233069538693:2049] VirtualId# [3:7587241233069538686:0]
2025-12-24T02:10:59.163207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241233069538693:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.163235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.163252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.166861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241225986513165:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.166877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230281480536:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.166883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225986513165:2048] [node 1] ICP30 unregister session Session# [5:7587241230281480536:2048] VirtualId# [5:7587241225986513233:0]
2025-12-24T02:10:59.166900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.166906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241225986513165:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.166908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.168921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241227023995504:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:59.172836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241229375484879:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:59.172941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:59.173021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241233670452265:2049] [node 2] ICS07 socket disconnect 265 reason# EndOfStream
2025-12-24T02:10:59.173027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233670452265:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.173086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.173105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241233670452265:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:59.173484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241229375484880:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.173494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233670452265:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.173499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241229375484880:2049] [node 2] ICP30 unregister session Session# [6:7587241233670452265:2049] VirtualId# [6:7587241229375484962:0]
2025-12-24T02:10:59.173510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.173515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241229375484880:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.173517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.174891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:59.174965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241230281480538:2049] [node 2] ICS07 socket disconnect 238 reason# EndOfStream
2025-12-24T02:10:59.174970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230281480538:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.175024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.175045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241230281480538:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:59.175303Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241225986513166:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.175312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230281480538:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.175318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225986513166:2049] [node 2] ICP30 unregister session Session# [5:7587241230281480538:2049] VirtualId# [5:7587241225986513234:0]
2025-12-24T02:10:59.175332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.175337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241225986513166:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.175338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.175100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:59.175195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241231318962881:2049] [node 2] ICS07 socket disconnect 273 reason# EndOfStream
2025-12-24T02:10:59.175200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231318962881:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.175249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.175271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241231318962881:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:59.177437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241227023995505:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.177453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231318962881:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.177457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241227023995505:2049] [node 2] ICP30 unregister session Session# [4:7587241231318962881:2049] VirtualId# [4:7587241227023995571:0]
2025-12-24T02:10:59.177470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.177474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241227023995505:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.177476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18656) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.180027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241225986513165:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:59.183738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241229375484880:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:59.187990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:59.188080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241231318962886:2050] [node 3] ICS07 socket disconnect 245 reason# EndOfStream
2025-12-24T02:10:59.188086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231318962886:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.188140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.188166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241231318962886:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:59.188391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241227023995505:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:59.188542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241227023995506:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.188550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231318962886:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.188556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241227023995506:2050] [node 3] ICP30 unregister session Session# [4:7587241231318962886:2050] VirtualId# [4:7587241227023995567:0]
2025-12-24T02:10:59.188569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.188574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241227023995506:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.188576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.188791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:59.188868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233670452263:2050] [node 3] ICS01 socket: 235 reason# EndOfStream
2025-12-24T02:10:59.188875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241229375484881:2050] [node 3] ICP30 unregister session Session# [6:7587241233670452263:2050] VirtualId# [6:7587241229375484936:0]
2025-12-24T02:10:59.188881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233670452263:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.188932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.188956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.186955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241225986513166:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:59.191068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:59.191157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241230281480555:2050] [node 3] ICS07 socket disconnect 299 reason# EndOfStream
2025-12-24T02:10:59.191162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230281480555:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.191214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.191235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241230281480555:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:59.191520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241225986513167:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.191526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230281480555:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.191529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241225986513167:2050] [node 3] ICP30 unregister session Session# [5:7587241230281480555:2050] VirtualId# [5:7587241230281480549:0]
2025-12-24T02:10:59.191540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.191544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241225986513167:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.191547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.198170Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:59.198257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241233670452269:2068] [node 4] ICS07 socket disconnect 259 reason# EndOfStream
2025-12-24T02:10:59.198265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233670452269:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.198330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:59.198364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241233670452269:2068] [node 4] ICS15 start handshake
2025-12-24T02:10:59.198806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241229375484922:2068] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2889) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.198818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233670452269:2068] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.198822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241229375484922:2068] [node 4] ICP30 unregister session Session# [6:7587241233670452269:2068] VirtualId# [6:7587241229375484937:0]
2025-12-24T02:10:59.198834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:59.198840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241229375484922:2068] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2889) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.198842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2889) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.211592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241229375484922:2068] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:59.262129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241231147273813:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:59.262152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241231147273814:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:59.266779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241232307310845:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.266810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241232307310846:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:59.281822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231916662228:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.281838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231916662229:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.281842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231916662230:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.277522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231010061873:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.277533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231010061874:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.293344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230142161252:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.293355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230142161253:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.293361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230142161254:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.301325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241233511979787:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.301336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241233511979788:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.301342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241233511979789:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.328448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241233511979801:2059] [node 5] ICP01 ready to work
2025-12-24T02:10:59.329474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241233511979809:2064] [node 4] ICP01 ready to work
2025-12-24T02:10:59.331167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241231147273877:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:59.333631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230142161312:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.334824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241231147273875:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:59.346951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161311:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.350839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241231147273887:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48136
2025-12-24T02:10:59.351106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273887:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.350851Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241230142161312:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17146124ns) {Configure:104(17145441ns) {ProcessConfigured:135(16755342ns) {StartInitialHandshake:157(16754814ns) {DropHandshakes:470(505ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1364ns) {SwitchToState:179(846ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.354074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241231147273875:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19184170ns) {Configure:104(19183183ns) {ProcessConfigured:135(18879357ns) {StartInitialHandshake:157(18878764ns) {DropHandshakes:470(457ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1486ns) {SwitchToState:179(950ns)}}}}}
2025-12-24T02:10:59.354315Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241233511979788:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19544704ns) {Configure:104(19544166ns) {ProcessConfigured:135(18868805ns) {StartInitialHandshake:157(18868426ns) {DropHandshakes:470(430ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(584ns) {SwitchToState:179(253ns)}}}}}
2025-12-24T02:10:59.354408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979864:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.355258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241233511979864:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.355294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241233511979801:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.360757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241232307310909:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.362985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061933:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.363010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061934:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.363994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231010061934:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.366814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230142161317:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.366838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241230142161317:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.367203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241232307310913:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25632
2025-12-24T02:10:59.366729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241231147273886:2109] [node 6] ICP01 ready to work
2025-12-24T02:10:59.367635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241231010061873:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.371390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310913:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.371765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241232307310845:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.371208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241231147273892:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48136
2025-12-24T02:10:59.371872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273882:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.371876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241231147273882:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.372457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241231147273882:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.376999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241232307310921:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.378785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241231147273877:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47551144ns) {Configure:104(47550659ns) {ProcessConfigured:135(47309034ns) {StartInitialHandshake:157(47308365ns) {DropHandshakes:470(465ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1195ns) {SwitchToState:179(583ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.379193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662289:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.381822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273884:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.381839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241231147273884:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.382721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662290:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.382839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273892:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.383397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241231147273886:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16580541ns) {Configure:104(16579504ns) {ProcessConfigured:135(16285439ns) {StartInitialHandshake:157(16284481ns) {DropHandshakes:470(648ns) {DropIncomingHandshake:442(98ns) DropOutgoingHandshake:457(127ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:477(857ns) {SwitchToState:179(668ns)}}}}}
2025-12-24T02:10:59.384340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979867:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.386875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230142161317:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.390742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241230142161312:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.390770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.390782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230142161312:2105] [node 6] ICP22 created new session: [5:7587241230142161322:2105]
2025-12-24T02:10:59.390790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230142161322:2105] [node 6] ICS09 handshake done sender: [5:7587241230142161317:2110] self: [5:7587241230142161313:0] peer: [6:7587241233511979865:0] socket: 254 qp: -1
2025-12-24T02:10:59.390799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230142161322:2105] [node 6] ICS10 traffic start
2025-12-24T02:10:59.390822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.393974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241231147273813:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.394007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.394014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241231147273813:2048] [node 2] ICP22 created new session: [1:7587241231147273898:2048]
2025-12-24T02:10:59.394021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241231147273898:2048] [node 2] ICS09 handshake done sender: [1:7587241231147273882:2110] self: [1:7587241231147273876:0] peer: [2:7587241232307310914:0] socket: 247 qp: -1
2025-12-24T02:10:59.394029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241231147273898:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:59.394040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.390819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061940:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.392306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241232307310919:2112] [node 6] ICP01 ready to work
2025-12-24T02:10:59.391520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161320:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.391595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161319:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.394964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231010061940:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.394781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231916662290:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.397184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061943:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.397380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310911:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.397387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241232307310911:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.398137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241232307310911:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.398211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241232307310909:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.398223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.398228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241232307310909:2108] [node 5] ICP22 created new session: [2:7587241232307310925:2108]
2025-12-24T02:10:59.398233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241232307310925:2108] [node 5] ICS09 handshake done sender: [2:7587241232307310911:2109] self: [2:7587241232307310910:0] peer: [5:7587241230142161325:0] socket: 259 qp: -1
2025-12-24T02:10:59.398238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241232307310925:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:59.398245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.399274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241231010061874:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32302618ns) {Configure:104(32301948ns) {ProcessConfigured:135(31899035ns) {StartInitialHandshake:157(31898265ns) {DropHandshakes:470(588ns) {DropIncomingHandshake:442(193ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(801ns) {SwitchToState:179(743ns)}}}}}
2025-12-24T02:10:59.396731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230142161319:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.399439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241231010061874:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.396766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230142161320:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.397062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241230142161252:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.397244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241230142161253:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.400196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161326:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.400480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230142161320:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.400608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241230142161253:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.400618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.400623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230142161253:2049] [node 2] ICP22 created new session: [5:7587241230142161327:2049]
2025-12-24T02:10:59.400629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230142161327:2049] [node 2] ICS09 handshake done sender: [5:7587241230142161320:2112] self: [5:7587241230142161325:0] peer: [2:7587241232307310910:0] socket: 270 qp: -1
2025-12-24T02:10:59.400635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230142161327:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.400645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.401448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241231147273884:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.401843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231010061934:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.401926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241231010061873:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.401938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.401945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231010061873:2048] [node 1] ICP22 created new session: [3:7587241231010061947:2048]
2025-12-24T02:10:59.401950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231010061947:2048] [node 1] ICS09 handshake done sender: [3:7587241231010061934:2108] self: [3:7587241231010061938:0] peer: [1:7587241231147273878:0] socket: 260 qp: -1
2025-12-24T02:10:59.401956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231010061947:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.401973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.402732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241233511979864:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.405961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241231147273814:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.405985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.405994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241231147273814:2049] [node 3] ICP22 created new session: [1:7587241231147273901:2049]
2025-12-24T02:10:59.406001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241231147273901:2049] [node 3] ICS09 handshake done sender: [1:7587241231147273884:2111] self: [1:7587241231147273878:0] peer: [3:7587241231010061938:0] socket: 248 qp: -1
2025-12-24T02:10:59.406007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241231147273901:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:59.406023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.402777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231010061942:2111] [node 4] ICP01 ready to work
2025-12-24T02:10:59.407483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241233511979801:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.407505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.407513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241233511979801:2059] [node 5] ICP22 created new session: [6:7587241233511979868:2059]
2025-12-24T02:10:59.407518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233511979868:2059] [node 5] ICS09 handshake done sender: [6:7587241233511979864:2114] self: [6:7587241233511979865:0] peer: [5:7587241230142161313:0] socket: 257 qp: -1
2025-12-24T02:10:59.407525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233511979868:2059] [node 5] ICS10 traffic start
2025-12-24T02:10:59.407543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.406894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231010061953:2115] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.409452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241232307310919:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17063953ns) {Configure:104(17062937ns) {ProcessConfigured:135(16750960ns) {StartInitialHandshake:157(16750343ns) {DropHandshakes:470(672ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:477(864ns) {SwitchToState:179(398ns)}}}}}
2025-12-24T02:10:59.409576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241232307310845:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241232307310913:2110] held: no
2025-12-24T02:10:59.409637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241232307310929:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25632
2025-12-24T02:10:59.409707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241232307310928:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25632
2025-12-24T02:10:59.410016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241230142161312:2105] [node 6] ICP35 event processing took too much time PendingConnection:279(19271266ns) {HandleHandshakeStatus:329(19270468ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(179ns) ProcessPendingSessionEvents:495(19100949ns) {ForwardSessionEventToSession:612(19096051ns) ForwardSessionEventToSession:612(1482ns) ForwardSessionEventToSession:612(923ns) ForwardSessionEventToSession:612(664ns)}}}
2025-12-24T02:10:59.410750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231916662228:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.412185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273889:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.412200Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241231147273889:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.412792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241232307310921:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35733493ns) {Configure:104(35731990ns) {ProcessConfigured:135(35437439ns) {StartInitialHandshake:157(35437091ns) {DropHandshakes:470(594ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(686ns) {SwitchToState:179(301ns)}}}}}
2025-12-24T02:10:59.412823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241232307310913:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.412287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231010061933:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.412473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231010061957:2117] [node 6] ICP01 ready to work
2025-12-24T02:10:59.412815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241231010061957:2117] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.413215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241231147273889:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.413193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310929:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.413205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241232307310845:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.413214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.413220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241232307310845:2048] [node 1] ICP22 created new session: [2:7587241232307310932:2048]
2025-12-24T02:10:59.413225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241232307310932:2048] [node 1] ICS09 handshake done sender: [2:7587241232307310913:2110] self: [2:7587241232307310914:0] peer: [1:7587241231147273876:0] socket: 249 qp: -1
2025-12-24T02:10:59.413230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241232307310932:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.413255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.413296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241231147273875:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.413307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.413313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241231147273875:2107] [node 5] ICP22 created new session: [1:7587241231147273905:2107]
2025-12-24T02:10:59.413319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241231147273905:2107] [node 5] ICS09 handshake done sender: [1:7587241231147273889:2113] self: [1:7587241231147273881:0] peer: [5:7587241230142161324:0] socket: 256 qp: -1
2025-12-24T02:10:59.413322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241231147273905:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:59.413337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.414557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241231147273813:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(41659987ns) {HandleHandshakeStatus:329(41659289ns) {DropIncomingHandshake:442(9617ns) DropOutgoingHandshake:457(176ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(13409256ns) {ForwardSessionEventToSession:612(13399675ns) ForwardSessionEventToSession:612(2150ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(799ns) ForwardSessionEventToSession:612(590ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.415340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161329:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.415709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230142161319:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.415802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241230142161252:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.415811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.415818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230142161252:2048] [node 1] ICP22 created new session: [5:7587241230142161330:2048]
2025-12-24T02:10:59.415824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230142161330:2048] [node 1] ICS09 handshake done sender: [5:7587241230142161319:2111] self: [5:7587241230142161324:0] peer: [1:7587241231147273881:0] socket: 258 qp: -1
2025-12-24T02:10:59.415831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230142161330:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.415838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.419903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241233511979813:2067] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.419918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241233511979813:2067] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.418984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662292:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.420432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231916662292:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.420778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231916662230:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.423019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241231010061873:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21072551ns) {HandleHandshakeStatus:329(21072038ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(1309ns) SwitchToState:179(157ns) ProcessPendingSessionEvents:495(12793319ns) {ForwardSessionEventToSession:612(12790900ns) ForwardSessionEventToSession:612(1037ns)}}}
2025-12-24T02:10:59.422912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231916662289:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.423102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231916662295:2111] [node 6] ICP01 ready to work
2025-12-24T02:10:59.423410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231916662295:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.424067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273894:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.424078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241231147273894:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.424672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241231147273894:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.424744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241231147273877:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.424758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.424763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241231147273877:2108] [node 4] ICP22 created new session: [1:7587241231147273908:2108]
2025-12-24T02:10:59.424769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241231147273908:2108] [node 4] ICS09 handshake done sender: [1:7587241231147273894:2115] self: [1:7587241231147273879:0] peer: [4:7587241231916662291:0] socket: 268 qp: -1
2025-12-24T02:10:59.424774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241231147273908:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:59.424780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.426468Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662297:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.426727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231916662290:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.426796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241231147273814:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20802248ns) {HandleHandshakeStatus:329(20801627ns) {DropIncomingHandshake:442(4444ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(215ns) ProcessPendingSessionEvents:495(15983815ns) {ForwardSessionEventToSession:612(15971620ns) ForwardSessionEventToSession:612(1764ns) ForwardSessionEventToSession:612(1383ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(1519ns) ForwardSessionEventToSession:612(685ns) ForwardSessionEventToSession:612(519ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.427606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241233511979815:2068] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.427621Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241233511979815:2068] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.428414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231010061950:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.428431Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241231010061950:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.430725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979870:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.432172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241233511979870:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.435476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241233511979813:2067] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.435709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061960:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.435908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231010061933:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.435990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241231010061957:2117] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.436002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.436009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231010061957:2117] [node 6] ICP22 created new session: [3:7587241231010061961:2117]
2025-12-24T02:10:59.436016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231010061961:2117] [node 6] ICS09 handshake done sender: [3:7587241231010061933:2107] self: [3:7587241231010061958:0] peer: [6:7587241233511979811:0] socket: 244 qp: -1
2025-12-24T02:10:59.436023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231010061961:2117] [node 6] ICS10 traffic start
2025-12-24T02:10:59.436032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.437193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310927:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.437207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241232307310927:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.436961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241233511979789:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.436978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.436984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241233511979789:2050] [node 3] ICP22 created new session: [6:7587241233511979873:2050]
2025-12-24T02:10:59.436989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233511979873:2050] [node 3] ICS09 handshake done sender: [6:7587241233511979813:2067] self: [6:7587241233511979811:0] peer: [3:7587241231010061958:0] socket: 241 qp: -1
2025-12-24T02:10:59.436995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233511979873:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.437001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.438834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241232307310927:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.438934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241232307310919:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.438948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.438955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241232307310919:2112] [node 6] ICP22 created new session: [2:7587241232307310937:2112]
2025-12-24T02:10:59.438961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241232307310937:2112] [node 6] ICS09 handshake done sender: [2:7587241232307310927:2114] self: [2:7587241232307310923:0] peer: [6:7587241233511979871:0] socket: 280 qp: -1
2025-12-24T02:10:59.438967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241232307310937:2112] [node 6] ICS10 traffic start
2025-12-24T02:10:59.438979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.442055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241233511979788:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.442725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231916662228:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.442756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.442769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231916662228:2048] [node 1] ICP22 created new session: [4:7587241231916662298:2048]
2025-12-24T02:10:59.442778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231916662298:2048] [node 1] ICS09 handshake done sender: [4:7587241231916662290:2109] self: [4:7587241231916662291:0] peer: [1:7587241231147273879:0] socket: 269 qp: -1
2025-12-24T02:10:59.442788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231916662298:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.442817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.441963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310916:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.441981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241232307310916:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.449218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231916662304:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.453482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241232307310939:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25632
2025-12-24T02:10:59.453562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241232307310940:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25632
2025-12-24T02:10:59.456967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241233511979815:2068] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.457068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231010061950:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.457165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241231010061942:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.457179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.457185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231010061942:2111] [node 4] ICP22 created new session: [3:7587241231010061964:2111]
2025-12-24T02:10:59.457190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231010061964:2111] [node 4] ICS09 handshake done sender: [3:7587241231010061950:2114] self: [3:7587241231010061949:0] peer: [4:7587241231916662293:0] socket: 278 qp: -1
2025-12-24T02:10:59.457196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231010061964:2111] [node 4] ICS10 traffic start
2025-12-24T02:10:59.457204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.458829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241233511979809:2064] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.458842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.458848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241233511979809:2064] [node 4] ICP22 created new session: [6:7587241233511979878:2064]
2025-12-24T02:10:59.458853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233511979878:2064] [node 4] ICS09 handshake done sender: [6:7587241233511979815:2068] self: [6:7587241233511979812:0] peer: [4:7587241231916662296:0] socket: 243 qp: -1
2025-12-24T02:10:59.458858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233511979878:2064] [node 4] ICS10 traffic start
2025-12-24T02:10:59.458877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.460564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.460585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161333:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.462233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310939:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.462273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310940:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.462320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241232307310919:2112] [node 6] ICP18 handshake (actor [2:7587241232307310940:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.463424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662308:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.463490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662309:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.463632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241230142161312:2105] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.467746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241231916662228:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40875733ns) {HandleHandshakeStatus:329(40875243ns) {DropIncomingHandshake:442(1081ns) DropOutgoingHandshake:457(126ns) SwitchToState:179(330ns) ProcessPendingSessionEvents:495(25ns)}}
2025-12-24T02:10:59.467780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231916662289:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.467789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231916662308:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.467814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231916662229:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.467911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231916662295:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.467925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.467932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231916662295:2111] [node 6] ICP22 created new session: [4:7587241231916662311:2111]
2025-12-24T02:10:59.467938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231916662311:2111] [node 6] ICS09 handshake done sender: [4:7587241231916662289:2108] self: [4:7587241231916662296:0] peer: [6:7587241233511979812:0] socket: 245 qp: -1
2025-12-24T02:10:59.467945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231916662311:2111] [node 6] ICS10 traffic start
2025-12-24T02:10:59.467954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.467090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230142161332:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.467473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241230142161254:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.468787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241230142161333:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.468945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241230142161336:2117] [node 4] ICP01 ready to work
2025-12-24T02:10:59.469241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241230142161336:2117] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.467646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241233511979788:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241233511979870:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.470835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231010061955:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.470846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241231010061955:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.475498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231010061966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6947
2025-12-24T02:10:59.478876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241232307310916:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.479007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241232307310846:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.479029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.479038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241232307310846:2049] [node 3] ICP22 created new session: [2:7587241232307310943:2049]
2025-12-24T02:10:59.479046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241232307310943:2049] [node 3] ICS09 handshake done sender: [2:7587241232307310916:2111] self: [2:7587241232307310915:0] peer: [3:7587241231010061946:0] socket: 264 qp: -1
2025-12-24T02:10:59.479053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241232307310943:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:59.479074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.478951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241231147273910:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48136
2025-12-24T02:10:59.479399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241232307310931:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.479411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241232307310931:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.482791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979880:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.482895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979881:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.482973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231010061940:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.483087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241231010061874:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.483109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.483116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231010061874:2049] [node 2] ICP22 created new session: [3:7587241231010061967:2049]
2025-12-24T02:10:59.483121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231010061967:2049] [node 2] ICS09 handshake done sender: [3:7587241231010061940:2110] self: [3:7587241231010061946:0] peer: [2:7587241232307310915:0] socket: 267 qp: -1
2025-12-24T02:10:59.483128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231010061967:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.483136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.483594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273910:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.483905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241233511979880:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.483941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241233511979787:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.484078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241231147273896:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.484087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241231147273896:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.485471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241233511979870:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.485617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241233511979883:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:10:59.485705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241233511979788:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.485720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.485726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241233511979788:2049] [node 2] ICP22 created new session: [6:7587241233511979884:2049]
2025-12-24T02:10:59.485733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233511979884:2049] [node 2] ICS09 handshake done sender: [6:7587241233511979870:2117] self: [6:7587241233511979871:0] peer: [2:7587241232307310923:0] socket: 281 qp: -1
2025-12-24T02:10:59.485740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233511979884:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.485746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.486912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241233511979880:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.486836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231916662306:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.486847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241231916662306:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.487433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662313:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.486798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241231147273896:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.488887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241231147273886:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.488909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.488916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241231147273886:2109] [node 6] ICP22 created new session: [1:7587241231147273913:2109]
2025-12-24T02:10:59.488923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241231147273913:2109] [node 6] ICS09 handshake done sender: [1:7587241231147273896:2116] self: [1:7587241231147273891:0] peer: [6:7587241233511979882:0] socket: 263 qp: -1
2025-12-24T02:10:59.488930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241231147273913:2109] [node 6] ICS10 traffic start
2025-12-24T02:10:59.488937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.490296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231916662306:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.490485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241231916662304:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.490497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.490505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231916662304:2114] [node 5] ICP22 created new session: [4:7587241231916662315:2114]
2025-12-24T02:10:59.490511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231916662315:2114] [node 5] ICS09 handshake done sender: [4:7587241231916662306:2115] self: [4:7587241231916662305:0] peer: [5:7587241230142161337:0] socket: 302 qp: -1
2025-12-24T02:10:59.490517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231916662315:2114] [node 5] ICS10 traffic start
2025-12-24T02:10:59.490524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.490659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231010061955:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.490737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241231010061953:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.490748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.490754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231010061953:2115] [node 5] ICP22 created new session: [3:7587241231010061970:2115]
2025-12-24T02:10:59.490759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231010061970:2115] [node 5] ICS09 handshake done sender: [3:7587241231010061955:2116] self: [3:7587241231010061954:0] peer: [5:7587241230142161334:0] socket: 287 qp: -1
2025-12-24T02:10:59.490764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231010061970:2115] [node 5] ICS10 traffic start
2025-12-24T02:10:59.490774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.491215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241233511979787:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.491233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.491240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241233511979787:2048] [node 1] ICP22 created new session: [6:7587241233511979886:2048]
2025-12-24T02:10:59.491247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241233511979886:2048] [node 1] ICS09 handshake done sender: [6:7587241233511979880:2118] self: [6:7587241233511979882:0] peer: [1:7587241231147273891:0] socket: 286 qp: -1
2025-12-24T02:10:59.491252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241233511979886:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.491260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.495574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231916662292:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.501043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241231916662295:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(33101246ns) {HandleHandshakeStatus:329(33100831ns) {DropIncomingHandshake:442(291ns) DropOutgoingHandshake:457(69ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(18216015ns) {ForwardSessionEventToSession:612(18212732ns) ForwardSessionEventToSession:612(1567ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.502281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231916662317:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60446
2025-12-24T02:10:59.502806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241232307310931:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.506792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231916662230:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.506815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.506827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231916662230:2050] [node 3] ICP22 created new session: [4:7587241231916662318:2050]
2025-12-24T02:10:59.506835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231916662318:2050] [node 3] ICS09 handshake done sender: [4:7587241231916662292:2110] self: [4:7587241231916662293:0] peer: [3:7587241231010061949:0] socket: 279 qp: -1
2025-12-24T02:10:59.506842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231916662318:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.506864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.507526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231916662308:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.507193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241232307310846:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(28156325ns) {HandleHandshakeStatus:329(28155583ns) {DropIncomingHandshake:442(4355ns) DropOutgoingHandshake:457(240ns) SwitchToState:179(156ns) ProcessPendingSessionEvents:495(17024598ns) {ForwardSessionEventToSession:612(17015083ns) ForwardSessionEventToSession:612(1832ns) ForwardSessionEventToSession:612(945ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(1060ns) ForwardSessionEventToSession:612(549ns) ForwardSessionEventToSession:612(707ns)}}}
2025-12-24T02:10:59.507635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231916662229:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.507651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.507657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231916662229:2049] [node 2] ICP22 created new session: [4:7587241231916662320:2049]
2025-12-24T02:10:59.507663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231916662320:2049] [node 2] ICS09 handshake done sender: [4:7587241231916662308:2116] self: [4:7587241231916662310:0] peer: [2:7587241232307310922:0] socket: 289 qp: -1
2025-12-24T02:10:59.507668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231916662320:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.507674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.507993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241232307310921:2113] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.508015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.508021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241232307310921:2113] [node 4] ICP22 created new session: [2:7587241232307310946:2113]
2025-12-24T02:10:59.508027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241232307310946:2113] [node 4] ICS09 handshake done sender: [2:7587241232307310931:2117] self: [2:7587241232307310922:0] peer: [4:7587241231916662310:0] socket: 251 qp: -1
2025-12-24T02:10:59.508034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241232307310946:2113] [node 4] ICS10 traffic start
2025-12-24T02:10:59.508046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.511007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161338:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.511122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241230142161339:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8553
2025-12-24T02:10:59.511671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230142161333:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.511765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241230142161332:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.512128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241230142161254:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.512144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.512152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230142161254:2050] [node 3] ICP22 created new session: [5:7587241230142161340:2050]
2025-12-24T02:10:59.512159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230142161340:2050] [node 3] ICS09 handshake done sender: [5:7587241230142161332:2115] self: [5:7587241230142161334:0] peer: [3:7587241231010061954:0] socket: 288 qp: -1
2025-12-24T02:10:59.512165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230142161340:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.512176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.512517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241231916662304:2114] [node 5] ICP35 event processing took too much time PendingConnection:279(22003277ns) {HandleHandshakeStatus:329(22002588ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(193ns) SwitchToState:179(170ns) ProcessPendingSessionEvents:495(21934251ns) {ForwardSessionEventToSession:612(21930239ns) ForwardSessionEventToSession:612(1462ns)}}}
2025-12-24T02:10:59.512701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241230142161336:2117] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.512716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.512727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241230142161336:2117] [node 4] ICP22 created new session: [5:7587241230142161342:2117]
2025-12-24T02:10:59.512733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241230142161342:2117] [node 4] ICS09 handshake done sender: [5:7587241230142161333:2116] self: [5:7587241230142161337:0] peer: [4:7587241231916662305:0] socket: 240 qp: -1
2025-12-24T02:10:59.512741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241230142161342:2117] [node 4] ICS10 traffic start
2025-12-24T02:10:59.512753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.512822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241231010061953:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(22064289ns) {HandleHandshakeStatus:329(22063759ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(171ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(21997317ns) {ForwardSessionEventToSession:612(21988916ns) ForwardSessionEventToSession:612(1584ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(1154ns) ForwardSessionEventToSession:612(559ns)}}}
2025-12-24T02:10:59.513036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241233511979787:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21799043ns) {HandleHandshakeStatus:329(21798565ns) {DropIncomingHandshake:442(128ns) DropOutgoingHandshake:457(1922ns) SwitchToState:179(86ns) ProcessPendingSessionEvents:495(21718397ns) {ForwardSessionEventToSession:612(21709263ns) ForwardSessionEventToSession:612(1249ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(985ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(1033ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(467ns) ForwardSessionEventToSession:612(588ns)}}}
2025-12-24T02:10:59.591640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:59.591728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241232307310932:2048] [node 1] ICS01 socket: 249 reason# EndOfStream
2025-12-24T02:10:59.591738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241232307310845:2048] [node 1] ICP30 unregister session Session# [2:7587241232307310932:2048] VirtualId# [2:7587241232307310914:0]
2025-12-24T02:10:59.591745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241232307310932:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.591791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.591821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.594777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:59.594873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241231010061947:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-12-24T02:10:59.594881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241231010061873:2048] [node 1] ICP30 unregister session Session# [3:7587241231010061947:2048] VirtualId# [3:7587241231010061938:0]
2025-12-24T02:10:59.594888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241231010061947:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.594959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.594986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.600326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:59.600424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233511979886:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-12-24T02:10:59.600433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241233511979787:2048] [node 1] ICP30 unregister session Session# [6:7587241233511979886:2048] VirtualId# [6:7587241233511979882:0]
2025-12-24T02:10:59.600439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233511979886:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.600492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.600517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.606763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:59.606858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241231916662298:2048] [node 1] ICS07 socket disconnect 269 reason# EndOfStream
2025-12-24T02:10:59.606864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231916662298:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.606939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.606961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241231916662298:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:59.607204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:59.607266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241230142161330:2048] [node 1] ICS07 socket disconnect 258 reason# EndOfStream
2025-12-24T02:10:59.607270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230142161330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.607304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.607323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241230142161330:2048] [node 1] ICS15 start handshake
2025-12-24T02:10:59.610792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:59.610871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241231010061967:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:10:59.610877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241231010061874:2049] [node 2] ICP30 unregister session Session# [3:7587241231010061967:2049] VirtualId# [3:7587241231010061946:0]
2025-12-24T02:10:59.610882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241231010061967:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.610933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.610959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.610993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:59.611059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233511979884:2049] [node 2] ICS01 socket: 281 reason# EndOfStream
2025-12-24T02:10:59.611064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241233511979788:2049] [node 2] ICP30 unregister session Session# [6:7587241233511979884:2049] VirtualId# [6:7587241233511979871:0]
2025-12-24T02:10:59.611067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233511979884:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.611094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.611113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.610950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:59.611008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241231916662320:2049] [node 2] ICS07 socket disconnect 289 reason# EndOfStream
2025-12-24T02:10:59.611012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231916662320:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.611014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241231916662228:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.611023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231916662298:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.611028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241231916662228:2048] [node 1] ICP30 unregister session Session# [4:7587241231916662298:2048] VirtualId# [4:7587241231916662291:0]
2025-12-24T02:10:59.611045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.611052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.611052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241231916662228:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.611055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.611070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241231916662320:2049] [node 2] ICS15 start handshake
2025-12-24T02:10:59.611224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:59.611288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230142161327:2049] [node 2] ICS01 socket: 270 reason# EndOfStream
2025-12-24T02:10:59.611294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241230142161253:2049] [node 2] ICP30 unregister session Session# [5:7587241230142161327:2049] VirtualId# [5:7587241230142161325:0]
2025-12-24T02:10:59.611298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230142161327:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.611341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.611377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.611405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241230142161252:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.611413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230142161330:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.611418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241230142161252:2048] [node 1] ICP30 unregister session Session# [5:7587241230142161330:2048] VirtualId# [5:7587241230142161324:0]
2025-12-24T02:10:59.611426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.611430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241230142161252:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.611432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.611775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241231916662229:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8292) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.611787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231916662320:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.611792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241231916662229:2049] [node 2] ICP30 unregister session Session# [4:7587241231916662320:2049] VirtualId# [4:7587241231916662310:0]
2025-12-24T02:10:59.611802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.611806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241231916662229:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8292) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.611807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8292) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.622747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241231916662228:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:59.622767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241231916662229:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:10:59.622741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241230142161252:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:10:59.626844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:59.626940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233511979873:2050] [node 3] ICS01 socket: 241 reason# EndOfStream
2025-12-24T02:10:59.626949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241233511979789:2050] [node 3] ICP30 unregister session Session# [6:7587241233511979873:2050] VirtualId# [6:7587241233511979811:0]
2025-12-24T02:10:59.626955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233511979873:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.626993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.627024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.629967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:59.630055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241230142161340:2050] [node 3] ICS07 socket disconnect 288 reason# EndOfStream
2025-12-24T02:10:59.630060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241230142161340:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.630122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.630143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241230142161340:2050] [node 3] ICS15 start handshake
2025-12-24T02:10:59.638529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241230142161254:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8987) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:10:59.638548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241230142161340:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:10:59.638554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241230142161254:2050] [node 3] ICP30 unregister session Session# [5:7587241230142161340:2050] VirtualId# [5:7587241230142161334:0]
2025-12-24T02:10:59.638572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.638578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241230142161254:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.638582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8987) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:10:59.640564Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:59.640644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233511979878:2064] [node 4] ICS01 socket: 243 reason# EndOfStream
2025-12-24T02:10:59.640652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241233511979809:2064] [node 4] ICP30 unregister session Session# [6:7587241233511979878:2064] VirtualId# [6:7587241233511979812:0]
2025-12-24T02:10:59.640657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233511979878:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.640698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:59.640724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:10:59.666785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:10:59.666881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241233511979868:2059] [node 5] ICS01 socket: 257 reason# EndOfStream
2025-12-24T02:10:59.666891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241233511979801:2059] [node 5] ICP30 unregister session Session# [6:7587241233511979868:2059] VirtualId# [6:7587241233511979865:0]
2025-12-24T02:10:59.666900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241233511979868:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.666954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:10:59.666981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:10:59.694271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241230853947393:2048] [node 2] ICP01 ready to work
2025-12-24T02:10:59.694279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241230853947394:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:59.695584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241231220555216:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.695603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241231220555217:2049] [node 3] ICP01 ready to work
2025-12-24T02:10:59.699082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231053942563:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.699098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231053942564:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.708734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231665227950:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.708745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231665227951:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.708751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231665227952:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.711227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241232075174578:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.711244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241232075174579:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.711249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241232075174580:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.727564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241232550620157:2048] [node 1] ICP01 ready to work
2025-12-24T02:10:59.727579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241232550620158:2049] [node 2] ICP01 ready to work
2025-12-24T02:10:59.727584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241232550620159:2050] [node 3] ICP01 ready to work
2025-12-24T02:10:59.759053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241231220555277:2107] [node 5] ICP01 ready to work
2025-12-24T02:10:59.759288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241232550620179:2061] [node 5] ICP01 ready to work
2025-12-24T02:10:59.759373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241232550620180:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.759280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241230853947455:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:59.759369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241230853947456:2108] [node 4] ICP01 ready to work
2025-12-24T02:10:59.760781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620188:2070] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.763930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231665227994:2077] [node 6] ICP01 ready to work
2025-12-24T02:10:59.764040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241231665227995:2079] [node 5] ICP01 ready to work
2025-12-24T02:10:59.765686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241231220555282:2109] [node 4] ICP01 ready to work
2025-12-24T02:10:59.762963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241230853947464:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.783318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241231665227951:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19229243ns) {Configure:104(19228717ns) {ProcessConfigured:135(18881294ns) {StartInitialHandshake:157(18881167ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(808ns) {SwitchToState:179(597ns)}}}}}
2025-12-24T02:10:59.784219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228023:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.784231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228024:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.799825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241230853947394:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20279932ns) {Configure:104(20279320ns) {ProcessConfigured:135(14441791ns) {StartInitialHandshake:157(14440248ns) {DropHandshakes:470(483ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:477(2443ns) {SwitchToState:179(1006ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.804825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228024:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.805331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620236:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.805819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241232075174579:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43241983ns) {Configure:104(43240808ns) {ProcessConfigured:135(42799013ns) {StartInitialHandshake:157(42797907ns) {DropHandshakes:470(544ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1807ns) {SwitchToState:179(1163ns)}}}}}
2025-12-24T02:10:59.811032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241232550620188:2070] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.811073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241232550620157:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.811222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947458:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.811233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230853947458:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.811281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231053942628:2107] [node 6] ICP01 ready to work
2025-12-24T02:10:59.811386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231053942629:2108] [node 5] ICP01 ready to work
2025-12-24T02:10:59.811466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241231053942630:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.815594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942626:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.814937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241231220555216:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49166545ns) {Configure:104(49165982ns) {ProcessConfigured:135(48839227ns) {StartInitialHandshake:157(48838286ns) {DropHandshakes:470(401ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1745ns) {SwitchToState:179(1053ns)}}}}}
2025-12-24T02:10:59.819937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241232075174646:2064] [node 4] ICP01 ready to work
2025-12-24T02:10:59.820047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241232075174647:2066] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.821250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174643:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.821280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174644:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.823356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174645:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.823430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174644:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.827758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942627:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.834824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241231220555217:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19796875ns) {Configure:104(19795315ns) {ProcessConfigured:135(19443419ns) {StartInitialHandshake:157(19442741ns) {DropHandshakes:470(1125ns) {DropIncomingHandshake:442(224ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(2084ns) {SwitchToState:179(1332ns)}}}}}
2025-12-24T02:10:59.836902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241231220555290:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.840125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620238:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.850820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241231053942563:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22918007ns) {Configure:104(22916044ns) {ProcessConfigured:135(22523214ns) {StartInitialHandshake:157(22522425ns) {DropHandshakes:470(876ns) {DropIncomingHandshake:442(161ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(1139ns) PrepareNewSessionHandshake:477(2370ns) {SwitchToState:179(1233ns)}}}}}
2025-12-24T02:10:59.851972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942626:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.852306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942641:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.854777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241231665227950:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31976883ns) {Configure:104(31975797ns) {ProcessConfigured:135(23814813ns) {StartInitialHandshake:157(23814379ns) {DropHandshakes:470(929ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:477(1326ns) {SwitchToState:179(758ns)}}}}}
2025-12-24T02:10:59.854842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228023:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.854991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231665227950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.856183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241231220555290:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17719232ns) {Configure:104(17717860ns) {ProcessConfigured:135(16937309ns) {StartInitialHandshake:157(16936714ns) {DropHandshakes:470(1432ns) {DropIncomingHandshake:442(660ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(1578ns) {SwitchToState:179(648ns)}}}}}
2025-12-24T02:10:59.856214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555291:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.855195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942627:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.855539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947460:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.855554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230853947460:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.855953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241230853947473:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4428
2025-12-24T02:10:59.855969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241230853947474:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4428
2025-12-24T02:10:59.856378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230853947458:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.856518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230853947460:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.856579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241230853947455:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.856593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.856601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241230853947455:2107] [node 6] ICP22 created new session: [1:7587241230853947476:2107]
2025-12-24T02:10:59.856606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230853947476:2107] [node 6] ICS09 handshake done sender: [1:7587241230853947458:2109] self: [1:7587241230853947457:0] peer: [6:7587241232550620237:0] socket: 248 qp: -1
2025-12-24T02:10:59.856613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230853947476:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:59.856622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.856750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241230853947456:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.856757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.856762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241230853947456:2108] [node 4] ICP22 created new session: [1:7587241230853947478:2108]
2025-12-24T02:10:59.856766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230853947478:2108] [node 4] ICS09 handshake done sender: [1:7587241230853947460:2110] self: [1:7587241230853947459:0] peer: [4:7587241231665228031:0] socket: 249 qp: -1
2025-12-24T02:10:59.856770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230853947478:2108] [node 4] ICS10 traffic start
2025-12-24T02:10:59.856775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.858827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241232550620236:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.859131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228032:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.860401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241232550620188:2070] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.860507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231665228023:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.861143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942647:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.861250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947474:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.861361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942647:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.861378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241231053942563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.861465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947470:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.861470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230853947470:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.861921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230853947470:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.862082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241230853947394:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.862099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.862107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241230853947394:2049] [node 3] ICP22 created new session: [1:7587241230853947484:2049]
2025-12-24T02:10:59.862112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230853947484:2049] [node 3] ICS09 handshake done sender: [1:7587241230853947470:2114] self: [1:7587241230853947469:0] peer: [3:7587241231053942648:0] socket: 286 qp: -1
2025-12-24T02:10:59.862119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230853947484:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:59.862127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.863693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241232550620180:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.863715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241232550620157:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.863727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.863733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241232550620157:2048] [node 1] ICP22 created new session: [6:7587241232550620241:2048]
2025-12-24T02:10:59.863739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241232550620241:2048] [node 1] ICS09 handshake done sender: [6:7587241232550620188:2070] self: [6:7587241232550620237:0] peer: [1:7587241230853947457:0] socket: 252 qp: -1
2025-12-24T02:10:59.863748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241232550620241:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.863755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.864136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620239:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.862961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555292:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.863243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555291:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.863461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.863528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.866109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555300:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.866490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174645:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.866562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174643:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.866592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241232075174578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.866602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241232075174579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.866424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228020:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.866435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241231665228020:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.867160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231665227950:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.867177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.867182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231665227950:2048] [node 1] ICP22 created new session: [4:7587241231665228033:2048]
2025-12-24T02:10:59.867188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231665228033:2048] [node 1] ICS09 handshake done sender: [4:7587241231665228023:2114] self: [4:7587241231665228031:0] peer: [1:7587241230853947459:0] socket: 255 qp: -1
2025-12-24T02:10:59.867194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231665228033:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.867201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.867859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947466:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.867869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230853947466:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.868453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241230853947486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4428
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.868583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174662:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.868615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174663:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.871019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230853947466:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.871312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620244:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.871404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555279:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.871414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241231220555279:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.871775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555298:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.871839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555300:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.871862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555297:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.871980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241231220555216:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.872050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555292:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.872277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947473:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.872323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947486:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.872350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241230853947468:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.872353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241230853947468:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.872764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241230853947464:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.872776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.872782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241230853947464:2111] [node 5] ICP22 created new session: [1:7587241230853947488:2111]
2025-12-24T02:10:59.872789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230853947488:2111] [node 5] ICS09 handshake done sender: [1:7587241230853947466:2112] self: [1:7587241230853947465:0] peer: [5:7587241232075174660:0] socket: 264 qp: -1
2025-12-24T02:10:59.872796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230853947488:2111] [node 5] ICS10 traffic start
2025-12-24T02:10:59.872805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.873057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241230853947464:2111] [node 5] ICP18 handshake (actor [1:7587241230853947473:2115]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.871251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942641:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.871285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241231053942563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241231053942647:2118] held: no
2025-12-24T02:10:59.871334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.873454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231053942647:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.873522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942650:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.873618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241231053942563:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.873627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.873632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231053942563:2048] [node 1] ICP22 created new session: [3:7587241231053942653:2048]
2025-12-24T02:10:59.873638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231053942653:2048] [node 1] ICS09 handshake done sender: [3:7587241231053942647:2118] self: [3:7587241231053942648:0] peer: [1:7587241230853947469:0] socket: 289 qp: -1
2025-12-24T02:10:59.873643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231053942653:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.873649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.871436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231665228020:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.874788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241231665227994:2077] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.874804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.874812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231665227994:2077] [node 6] ICP22 created new session: [4:7587241231665228037:2077]
2025-12-24T02:10:59.874819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231665228037:2077] [node 6] ICS09 handshake done sender: [4:7587241231665228020:2112] self: [4:7587241231665228019:0] peer: [6:7587241232550620240:0] socket: 259 qp: -1
2025-12-24T02:10:59.874824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231665228037:2077] [node 6] ICS10 traffic start
2025-12-24T02:10:59.874832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.874954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241232550620239:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.878649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241230853947468:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.878928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241232075174578:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241232075174645:2113] held: no
2025-12-24T02:10:59.878987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.879023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241232075174645:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.880251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241230853947393:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.880270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.880278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241230853947393:2048] [node 2] ICP22 created new session: [1:7587241230853947492:2048]
2025-12-24T02:10:59.880284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241230853947492:2048] [node 2] ICS09 handshake done sender: [1:7587241230853947468:2113] self: [1:7587241230853947467:0] peer: [2:7587241231220555302:0] socket: 285 qp: -1
2025-12-24T02:10:59.880290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241230853947492:2048] [node 2] ICS10 traffic start
2025-12-24T02:10:59.880299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.879600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241232550620236:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.881452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228039:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.882096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241232550620159:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.883613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241232550620180:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.883632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.883638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241232550620180:2065] [node 4] ICP22 created new session: [6:7587241232550620246:2065]
2025-12-24T02:10:59.883645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241232550620246:2065] [node 4] ICS09 handshake done sender: [6:7587241232550620236:2116] self: [6:7587241232550620240:0] peer: [4:7587241231665228019:0] socket: 261 qp: -1
2025-12-24T02:10:59.883653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241232550620246:2065] [node 4] ICS10 traffic start
2025-12-24T02:10:59.883661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.883733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241232550620180:2065] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.884693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241232075174578:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.884711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.884718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241232075174578:2048] [node 1] ICP22 created new session: [5:7587241232075174667:2048]
2025-12-24T02:10:59.884724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241232075174667:2048] [node 1] ICS09 handshake done sender: [5:7587241232075174645:2113] self: [5:7587241232075174660:0] peer: [1:7587241230853947465:0] socket: 267 qp: -1
2025-12-24T02:10:59.884731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241232075174667:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.884739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:10:59.884873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174665:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.884925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241232075174580:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.884972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174664:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.886347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228040:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.883104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942650:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.883156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241231053942564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.884028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942636:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.884033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241231053942636:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.884703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231053942636:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.884865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241231053942628:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.884877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.884882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231053942628:2107] [node 6] ICP22 created new session: [3:7587241231053942658:2107]
2025-12-24T02:10:59.884888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231053942658:2107] [node 6] ICS09 handshake done sender: [3:7587241231053942636:2114] self: [3:7587241231053942635:0] peer: [6:7587241232550620245:0] socket: 281 qp: -1
2025-12-24T02:10:59.884893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231053942658:2107] [node 6] ICS10 traffic start
2025-12-24T02:10:59.884899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.885247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942638:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.885250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241231053942638:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.886333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241231220555216:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19473) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241231220555298:2118] held: no
2025-12-24T02:10:59.886389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555287:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.886392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241231220555287:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.886741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241231220555279:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.886840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241231220555277:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.886849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.886855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241231220555277:2107] [node 5] ICP22 created new session: [2:7587241231220555308:2107]
2025-12-24T02:10:59.886861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241231220555308:2107] [node 5] ICS09 handshake done sender: [2:7587241231220555279:2108] self: [2:7587241231220555278:0] peer: [5:7587241232075174661:0] socket: 246 qp: -1
2025-12-24T02:10:59.886866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241231220555308:2107] [node 5] ICS10 traffic start
2025-12-24T02:10:59.886871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.889518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174662:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.889818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241231053942661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19998
2025-12-24T02:10:59.887474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241231220555306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35642
2025-12-24T02:10:59.889226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241231220555287:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.889304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241231220555217:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.889317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.889325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241231220555217:2049] [node 3] ICP22 created new session: [2:7587241231220555312:2049]
2025-12-24T02:10:59.889331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241231220555312:2049] [node 3] ICS09 handshake done sender: [2:7587241231220555287:2112] self: [2:7587241231220555286:0] peer: [3:7587241231053942655:0] socket: 293 qp: -1
2025-12-24T02:10:59.889335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241231220555312:2049] [node 3] ICS10 traffic start
2025-12-24T02:10:59.889342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.889438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241231220555298:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:10:59.890938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241232075174643:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.891020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241232075174579:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.891035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.891042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241232075174579:2049] [node 2] ICP22 created new session: [5:7587241232075174670:2049]
2025-12-24T02:10:59.891048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241232075174670:2049] [node 2] ICS09 handshake done sender: [5:7587241232075174643:2111] self: [5:7587241232075174661:0] peer: [2:7587241231220555278:0] socket: 265 qp: -1
2025-12-24T02:10:59.891054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241232075174670:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.891060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.891591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620249:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.891649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241232550620159:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241232550620239:2118] held: no
2025-12-24T02:10:59.891692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620248:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.891863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241232550620239:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.891930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241232550620159:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.891943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.891948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241232550620159:2050] [node 3] ICP22 created new session: [6:7587241232550620250:2050]
2025-12-24T02:10:59.891954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241232550620250:2050] [node 3] ICS09 handshake done sender: [6:7587241232550620239:2118] self: [6:7587241232550620245:0] peer: [3:7587241231053942635:0] socket: 287 qp: -1
2025-12-24T02:10:59.891960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241232550620250:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.891967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.891266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228039:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.891392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228042:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.891332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241231220555216:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.891349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:10:59.891356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241231220555216:2048] [node 1] ICP22 created new session: [2:7587241231220555314:2048]
2025-12-24T02:10:59.892967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231053942650:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.891362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241231220555314:2048] [node 1] ICS09 handshake done sender: [2:7587241231220555298:2118] self: [2:7587241231220555302:0] peer: [1:7587241230853947467:0] socket: 291 qp: -1
2025-12-24T02:10:59.891368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241231220555314:2048] [node 1] ICS10 traffic start
2025-12-24T02:10:59.891376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.895811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241232075174579:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:59.895838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241232075174646:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.895861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174673:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.900376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241232075174675:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24398
2025-12-24T02:10:59.899916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620252:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.899120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228022:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.899135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241231665228022:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.899682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228040:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.899707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231665227952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.901926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241232550620249:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.901957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241232550620252:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.901985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241232550620158:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.901997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241232550620179:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.902821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241231053942640:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.902836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241231053942640:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.903136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241231053942564:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.903158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.903165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231053942564:2049] [node 2] ICP22 created new session: [3:7587241231053942662:2049]
2025-12-24T02:10:59.903171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231053942662:2049] [node 2] ICS09 handshake done sender: [3:7587241231053942650:2120] self: [3:7587241231053942655:0] peer: [2:7587241231220555286:0] socket: 303 qp: -1
2025-12-24T02:10:59.903182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231053942662:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.903191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.902134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555294:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.902143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241231220555294:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.903082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241231220555294:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.903194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241231220555290:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.903209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.903217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241231220555290:2110] [node 6] ICP22 created new session: [2:7587241231220555317:2110]
2025-12-24T02:10:59.903223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241231220555317:2110] [node 6] ICS09 handshake done sender: [2:7587241231220555294:2116] self: [2:7587241231220555293:0] peer: [6:7587241232550620253:0] socket: 310 qp: -1
2025-12-24T02:10:59.903229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241231220555317:2110] [node 6] ICS10 traffic start
2025-12-24T02:10:59.903236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.903160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241232075174655:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.903177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241232075174655:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.903695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620255:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.903845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241232550620249:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.903902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241232550620158:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.903917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.903922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241232550620158:2049] [node 2] ICP22 created new session: [6:7587241232550620256:2049]
2025-12-24T02:10:59.903928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241232550620256:2049] [node 2] ICS09 handshake done sender: [6:7587241232550620249:2121] self: [6:7587241232550620253:0] peer: [2:7587241231220555293:0] socket: 311 qp: -1
2025-12-24T02:10:59.903934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241232550620256:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.903943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.902992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231665228022:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.907043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231053942638:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.911013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241232075174662:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.911274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241232550620258:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60256
2025-12-24T02:10:59.911490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241231665228042:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.911506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241231665227995:2079] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.911519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.911525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231665227995:2079] [node 5] ICP22 created new session: [4:7587241231665228046:2079]
2025-12-24T02:10:59.911531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231665228046:2079] [node 5] ICS09 handshake done sender: [4:7587241231665228022:2113] self: [4:7587241231665228021:0] peer: [5:7587241232075174674:0] socket: 297 qp: -1
2025-12-24T02:10:59.911539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231665228046:2079] [node 5] ICS10 traffic start
2025-12-24T02:10:59.911547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.915446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241231053942640:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.916171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241231053942630:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.916197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.916205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231053942630:2109] [node 4] ICP22 created new session: [3:7587241231053942665:2109]
2025-12-24T02:10:59.916211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231053942665:2109] [node 4] ICS09 handshake done sender: [3:7587241231053942640:2116] self: [3:7587241231053942639:0] peer: [4:7587241231665228044:0] socket: 292 qp: -1
2025-12-24T02:10:59.916218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231053942665:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:59.916228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.916604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241231053942564:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:59.916626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241231053942629:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.916637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.916642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241231053942629:2108] [node 5] ICP22 created new session: [3:7587241231053942667:2108]
2025-12-24T02:10:59.916646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241231053942667:2108] [node 5] ICS09 handshake done sender: [3:7587241231053942638:2115] self: [3:7587241231053942637:0] peer: [5:7587241232075174669:0] socket: 306 qp: -1
2025-12-24T02:10:59.916651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241231053942667:2108] [node 5] ICS10 traffic start
2025-12-24T02:10:59.916658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.917966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241231220555289:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:10:59.917991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241231220555289:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:10:59.914870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228045:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.914910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241231665227951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:10:59.919075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241231220555289:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:10:59.918811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241232075174646:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.918835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.918844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241232075174646:2064] [node 4] ICP22 created new session: [5:7587241232075174677:2064]
2025-12-24T02:10:59.918851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241232075174677:2064] [node 4] ICS09 handshake done sender: [5:7587241232075174662:2118] self: [5:7587241232075174674:0] peer: [4:7587241231665228021:0] socket: 300 qp: -1
2025-12-24T02:10:59.918861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241232075174677:2064] [node 4] ICS10 traffic start
2025-12-24T02:10:59.918871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.919099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241232075174655:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:10:59.919488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241232550620252:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:10:59.919221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241231220555282:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.919253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:10:59.919259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241231220555282:2109] [node 4] ICP22 created new session: [2:7587241231220555320:2109]
2025-12-24T02:10:59.919266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241231220555320:2109] [node 4] ICS09 handshake done sender: [2:7587241231220555289:2113] self: [2:7587241231220555288:0] peer: [4:7587241231665228048:0] socket: 274 qp: -1
2025-12-24T02:10:59.919272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241231220555320:2109] [node 4] ICS10 traffic start
2025-12-24T02:10:59.919280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:10:59.919551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241232550620179:2061] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.919565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:10:59.919569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241232550620179:2061] [node 5] ICP22 created new session: [6:7587241232550620259:2061]
2025-12-24T02:10:59.919574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241232550620259:2061] [node 5] ICS09 handshake done sender: [6:7587241232550620252:2122] self: [6:7587241232550620254:0] peer: [5:7587241232075174654:0] socket: 315 qp: -1
2025-12-24T02:10:59.919579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241232550620259:2061] [node 5] ICS10 traffic start
2025-12-24T02:10:59.919586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:10:59.919385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231665228040:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.922209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231665227952:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.922222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.922230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231665227952:2050] [node 3] ICP22 created new session: [4:7587241231665228049:2050]
2025-12-24T02:10:59.922237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231665228049:2050] [node 3] ICS09 handshake done sender: [4:7587241231665228040:2119] self: [4:7587241231665228044:0] peer: [3:7587241231053942639:0] socket: 254 qp: -1
2025-12-24T02:10:59.922243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231665228049:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.922251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:10:59.922406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241231665227951:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241231665228042:2120] held: no
2025-12-24T02:10:59.923009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241232075174647:2066] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:10:59.923027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:10:59.923032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241232075174647:2066] [node 6] ICP22 created new session: [5:7587241232075174679:2066]
2025-12-24T02:10:59.923038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241232075174679:2066] [node 6] ICS09 handshake done sender: [5:7587241232075174655:2117] self: [5:7587241232075174654:0] peer: [6:7587241232550620254:0] socket: 247 qp: -1
2025-12-24T02:10:59.923042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241232075174679:2066] [node 6] ICS10 traffic start
2025-12-24T02:10:59.923049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:10:59.923380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241232075174646:2064] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:59.923404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241232075174665:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:10:59.923817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241232075174580:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241232075174665:2121] held: no
2025-12-24T02:10:59.923837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241232075174580:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.923844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:10:59.923848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241232075174580:2050] [node 3] ICP22 created new session: [5:7587241232075174681:2050]
2025-12-24T02:10:59.923853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241232075174681:2050] [node 3] ICS09 handshake done sender: [5:7587241232075174665:2121] self: [5:7587241232075174669:0] peer: [3:7587241231053942637:0] socket: 308 qp: -1
2025-12-24T02:10:59.923860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241232075174681:2050] [node 3] ICS10 traffic start
2025-12-24T02:10:59.923867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:10:59.926524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241231665227952:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:10:59.926610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241231665228051:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4903
2025-12-24T02:10:59.926830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241231665228042:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:10:59.926886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241231665227951:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:10:59.926896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:10:59.926902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241231665227951:2049] [node 2] ICP22 created new session: [4:7587241231665228052:2049]
2025-12-24T02:10:59.926907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241231665228052:2049] [node 2] ICS09 handshake done sender: [4:7587241231665228042:2120] self: [4:7587241231665228048:0] peer: [2:7587241231220555288:0] socket: 283 qp: -1
2025-12-24T02:10:59.926914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241231665228052:2049] [node 2] ICS10 traffic start
2025-12-24T02:10:59.926921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:10:59.939185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:10:59.939283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241231220555314:2048] [node 1] ICS01 socket: 291 reason# EndOfStream
2025-12-24T02:10:59.939293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241231220555216:2048] [node 1] ICP30 unregister session Session# [2:7587241231220555314:2048] VirtualId# [2:7587241231220555302:0]
2025-12-24T02:10:59.939300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241231220555314:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.939347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.939370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.940244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:10:59.940322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241231053942653:2048] [node 1] ICS01 socket: 289 reason# EndOfStream
2025-12-24T02:10:59.940329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241231053942563:2048] [node 1] ICP30 unregister session Session# [3:7587241231053942653:2048] VirtualId# [3:7587241231053942648:0]
2025-12-24T02:10:59.940335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241231053942653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.940396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.940421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.941555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:10:59.941631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231665228033:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-12-24T02:10:59.941638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241231665227950:2048] [node 1] ICP30 unregister session Session# [4:7587241231665228033:2048] VirtualId# [4:7587241231665228031:0]
2025-12-24T02:10:59.941643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231665228033:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.941689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.941713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.942749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:10:59.942821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241232550620241:2048] [node 1] ICS01 socket: 252 reason# EndOfStream
2025-12-24T02:10:59.942828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241232550620157:2048] [node 1] ICP30 unregister session Session# [6:7587241232550620241:2048] VirtualId# [6:7587241232550620237:0]
2025-12-24T02:10:59.942832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241232550620241:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.942881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.942902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.942934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:10:59.942990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241232075174667:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:10:59.942993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241232075174578:2048] [node 1] ICP30 unregister session Session# [5:7587241232075174667:2048] VirtualId# [5:7587241232075174660:0]
2025-12-24T02:10:59.942997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241232075174667:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.943021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:10:59.943045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:10:59.954766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:10:59.954873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241231053942662:2049] [node 2] ICS01 socket: 303 reason# EndOfStream
2025-12-24T02:10:59.954881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241231053942564:2049] [node 2] ICP30 unregister session Session# [3:7587241231053942662:2049] VirtualId# [3:7587241231053942655:0]
2025-12-24T02:10:59.954888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241231053942662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.954952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.954986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.960338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:10:59.960438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241232075174670:2049] [node 2] ICS01 socket: 265 reason# EndOfStream
2025-12-24T02:10:59.960448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241232075174579:2049] [node 2] ICP30 unregister session Session# [5:7587241232075174670:2049] VirtualId# [5:7587241232075174661:0]
2025-12-24T02:10:59.960458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241232075174670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.960508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.960538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.961383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:10:59.961467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231665228052:2049] [node 2] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:10:59.961479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241231665227951:2049] [node 2] ICP30 unregister session Session# [4:7587241231665228052:2049] VirtualId# [4:7587241231665228048:0]
2025-12-24T02:10:59.961486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231665228052:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.961535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.961562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.961969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:10:59.962043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241232550620256:2049] [node 2] ICS01 socket: 311 reason# EndOfStream
2025-12-24T02:10:59.962051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241232550620158:2049] [node 2] ICP30 unregister session Session# [6:7587241232550620256:2049] VirtualId# [6:7587241232550620253:0]
2025-12-24T02:10:59.962057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241232550620256:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.962098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:10:59.962121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:10:59.981778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:10:59.981872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241231665228049:2050] [node 3] ICS01 socket: 254 reason# EndOfStream
2025-12-24T02:10:59.981881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241231665227952:2050] [node 3] ICP30 unregister session Session# [4:7587241231665228049:2050] VirtualId# [4:7587241231665228044:0]
2025-12-24T02:10:59.981888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241231665228049:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.981948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.981977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.984697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:10:59.984790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241232550620250:2050] [node 3] ICS01 socket: 287 reason# EndOfStream
2025-12-24T02:10:59.984797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241232550620159:2050] [node 3] ICP30 unregister session Session# [6:7587241232550620250:2050] VirtualId# [6:7587241232550620245:0]
2025-12-24T02:10:59.984805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241232550620250:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.984868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.984894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.984969Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:10:59.985048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241232075174681:2050] [node 3] ICS01 socket: 308 reason# EndOfStream
2025-12-24T02:10:59.985054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241232075174580:2050] [node 3] ICP30 unregister session Session# [5:7587241232075174681:2050] VirtualId# [5:7587241232075174669:0]
2025-12-24T02:10:59.985060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241232075174681:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.985114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:10:59.985149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:10:59.999713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:10:59.999819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241232550620246:2065] [node 4] ICS01 socket: 261 reason# EndOfStream
2025-12-24T02:10:59.999829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241232550620180:2065] [node 4] ICP30 unregister session Session# [6:7587241232550620246:2065] VirtualId# [6:7587241232550620240:0]
2025-12-24T02:10:59.999838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241232550620246:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:10:59.999902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:10:59.999928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:00.000665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:00.000743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241232075174677:2064] [node 4] ICS01 socket: 300 reason# EndOfStream
2025-12-24T02:11:00.000750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241232075174646:2064] [node 4] ICP30 unregister session Session# [5:7587241232075174677:2064] VirtualId# [5:7587241232075174674:0]
2025-12-24T02:11:00.000756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241232075174677:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.000809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:00.000832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:00.055108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241234556714879:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:00.055118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241234556714880:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.066244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241235968579178:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.066253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241235968579179:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.066513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237012792309:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.066527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237012792310:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.069212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241237627624675:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.069230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241237627624676:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.069236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241237627624677:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.083102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234457683686:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.083112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234457683687:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.083116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234457683688:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.110980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236610370009:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.110992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236610370010:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.131048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241235968579197:2060] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.132622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241234556714934:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64008
2025-12-24T02:11:00.132561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237012792326:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:00.134875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241237627624693:2063] [node 6] ICP01 ready to work
2025-12-24T02:11:00.134508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241234556714942:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:00.134600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241234556714943:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:00.110997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236610370011:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.131159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236610370027:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:00.131250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236610370028:2064] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.136924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241235968579218:2074] [node 4] ICP01 ready to work
2025-12-24T02:11:00.137008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241235968579219:2076] [node 6] ICP01 ready to work
2025-12-24T02:11:00.138012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579220:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.133546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237012792331:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:00.142233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234457683714:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:00.150917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241237012792331:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17303198ns) {Configure:104(17302486ns) {ProcessConfigured:135(17048836ns) {StartInitialHandshake:157(17047792ns) {DropHandshakes:470(421ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(70ns) PrepareNewSessionHandshake:477(2366ns) {SwitchToState:179(1671ns)}}}}}
2025-12-24T02:11:00.151095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237012792365:2070] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.152424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792366:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.154990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241237627624733:2089] [node 5] ICP01 ready to work
2025-12-24T02:11:00.155595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241234556714942:2106] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20949251ns) {Configure:104(20948856ns) {ProcessConfigured:135(20621864ns) {StartInitialHandshake:157(20621468ns) {DropHandshakes:470(337ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(73ns) PrepareNewSessionHandshake:477(1026ns) {SwitchToState:179(601ns)}}}}}
2025-12-24T02:11:00.156544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241234556714955:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64008
2025-12-24T02:11:00.166837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241234556714941:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:00.166984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241234556714954:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64008
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.174501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624731:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.174529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624732:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.174781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241234457683688:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32460938ns) {Configure:104(32459858ns) {ProcessConfigured:135(32033145ns) {StartInitialHandshake:157(32032704ns) {DropHandshakes:470(878ns) {DropIncomingHandshake:442(102ns) DropOutgoingHandshake:457(154ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(212ns)}}}}}
2025-12-24T02:11:00.175725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234457683758:2083] [node 6] ICP01 ready to work
2025-12-24T02:11:00.176383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683764:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.176411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.176474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.179001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624754:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.183227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579259:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.183258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579260:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.185461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683765:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.185511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234457683688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.190895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579220:2082] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.191516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624732:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.198879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241237012792310:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(66239290ns) {Configure:104(66238358ns) {ProcessConfigured:135(65844079ns) {StartInitialHandshake:157(65843323ns) {DropHandshakes:470(507ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:477(2026ns) {SwitchToState:179(1043ns)}}}}}
2025-12-24T02:11:00.199225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792389:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
2025-12-24T02:11:00.199239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
2025-12-24T02:11:00.203024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241234556714941:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36038009ns) {Configure:104(36036804ns) {ProcessConfigured:135(35619770ns) {StartInitialHandshake:157(35619357ns) {DropHandshakes:470(1240ns) {DropIncomingHandshake:442(131ns) DropOutgoingHandshake:457(184ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(993ns) {SwitchToState:179(867ns)}}}}}
2025-12-24T02:11:00.202951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792364:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.202961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237012792364:2098] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.205173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579224:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.205195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241235968579224:2085] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.205347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714954:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.202792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683766:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.202808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241237627624733:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47737533ns) {Configure:104(47736873ns) {ProcessConfigured:135(47410291ns) {StartInitialHandshake:157(47409655ns) {DropHandshakes:470(485ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(1477ns) {SwitchToState:179(681ns)}}}}}
2025-12-24T02:11:00.202833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234457683687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.206720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579203:2070] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.206737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241235968579203:2070] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.214625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237012792364:2098] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.214800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237012792331:2066] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.214814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.214819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237012792331:2066] [node 5] ICP22 created new session: [2:7587241237012792392:2066]
2025-12-24T02:11:00.214825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237012792392:2066] [node 5] ICS09 handshake done sender: [2:7587241237012792364:2098] self: [2:7587241237012792332:0] peer: [5:7587241234457683768:0] socket: 268 qp: -1
2025-12-24T02:11:00.214831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237012792392:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:00.214839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.215906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714934:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.216046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241235968579224:2085] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.213665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241236610370011:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(78536669ns) {Configure:104(78536226ns) {ProcessConfigured:135(78265426ns) {StartInitialHandshake:157(78264926ns) {DropHandshakes:470(289ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(1379ns) {SwitchToState:179(804ns)}}}}}
2025-12-24T02:11:00.213750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370086:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.213816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370088:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.213828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370087:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.216406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236610370087:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.216432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236610370010:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.216482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236610370086:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.216496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236610370028:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.218783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241237627624677:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.218823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241237627624677:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27243342ns) {IncomingHandshake:261(27242779ns) {DropIncomingHandshake:442(177ns) PrepareNewSessionHandshake:477(680ns) {SwitchToState:179(257ns)} GenerateSessionVirtualId:988(246ns) IssueIncomingHandshakeReply:193(27236612ns) {SwitchToState:179(739ns)}}}
2025-12-24T02:11:00.216272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714955:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.218964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683769:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.222894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624697:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.222909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241237627624697:2067] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.226901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624757:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.226923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.227437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241235968579218:2074] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.227462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.227470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241235968579218:2074] [node 4] ICP22 created new session: [3:7587241235968579263:2074]
2025-12-24T02:11:00.227478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241235968579263:2074] [node 4] ICS09 handshake done sender: [3:7587241235968579224:2085] self: [3:7587241235968579223:0] peer: [4:7587241237627624755:0] socket: 265 qp: -1
2025-12-24T02:11:00.227487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241235968579263:2074] [node 4] ICS10 traffic start
2025-12-24T02:11:00.227498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.228754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241235968579203:2070] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.229679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792369:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.229696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237012792369:2101] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.230337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.229855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234457683766:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.229995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234457683687:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.230009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.230014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234457683687:2049] [node 2] ICP22 created new session: [5:7587241234457683771:2049]
2025-12-24T02:11:00.230020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234457683771:2049] [node 2] ICS09 handshake done sender: [5:7587241234457683766:2117] self: [5:7587241234457683768:0] peer: [2:7587241237012792332:0] socket: 284 qp: -1
2025-12-24T02:11:00.230026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234457683771:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.230043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.230454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683770:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.230764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234457683765:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.230808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234457683688:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.230824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.230829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234457683688:2050] [node 3] ICP22 created new session: [5:7587241234457683774:2050]
2025-12-24T02:11:00.230833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234457683774:2050] [node 3] ICS09 handshake done sender: [5:7587241234457683765:2116] self: [5:7587241234457683767:0] peer: [3:7587241235968579202:0] socket: 283 qp: -1
2025-12-24T02:11:00.230839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234457683774:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.230844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.231168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241234556714964:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64008
2025-12-24T02:11:00.243273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241237627624732:2097] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.243360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241237627624677:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241237627624732:2097] held: no
2025-12-24T02:11:00.243380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241237627624677:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.243390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.243397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241237627624677:2050] [node 3] ICP22 created new session: [4:7587241237627624760:2050]
2025-12-24T02:11:00.243403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241237627624760:2050] [node 3] ICS09 handshake done sender: [4:7587241237627624732:2097] self: [4:7587241237627624755:0] peer: [3:7587241235968579223:0] socket: 272 qp: -1
2025-12-24T02:11:00.243425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241237627624760:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.243433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.248268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241235968579197:2060] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.248287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.248297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241235968579197:2060] [node 5] ICP22 created new session: [3:7587241235968579267:2060]
2025-12-24T02:11:00.248305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241235968579267:2060] [node 5] ICS09 handshake done sender: [3:7587241235968579203:2070] self: [3:7587241235968579202:0] peer: [5:7587241234457683767:0] socket: 252 qp: -1
2025-12-24T02:11:00.248312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241235968579267:2060] [node 5] ICS10 traffic start
2025-12-24T02:11:00.248321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.249348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237012792369:2101] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.249524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237012792365:2070] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.249535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.249540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237012792365:2070] [node 6] ICP22 created new session: [2:7587241237012792397:2070]
2025-12-24T02:11:00.249546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237012792397:2070] [node 6] ICS09 handshake done sender: [2:7587241237012792369:2101] self: [2:7587241237012792368:0] peer: [6:7587241236610370090:0] socket: 269 qp: -1
2025-12-24T02:11:00.249551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237012792397:2070] [node 6] ICS10 traffic start
2025-12-24T02:11:00.249556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.248800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370093:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.250927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624757:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.251013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241237627624676:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.251069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683764:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.251081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370094:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.251165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370095:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.251460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236610370093:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.251481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236610370088:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.251497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236610370087:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.251617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236610370010:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.251629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.251635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236610370010:2049] [node 2] ICP22 created new session: [6:7587241236610370096:2049]
2025-12-24T02:11:00.251641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236610370096:2049] [node 2] ICS09 handshake done sender: [6:7587241236610370087:2115] self: [6:7587241236610370090:0] peer: [2:7587241237012792368:0] socket: 291 qp: -1
2025-12-24T02:11:00.251647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236610370096:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.251655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.251841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236610370027:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.251854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236610370009:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.254000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579259:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.254047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579270:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.254090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241235968579178:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.252435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624731:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.252526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241237627624697:2067] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.253717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624754:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.254314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241237627624693:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.254327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.254333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241237627624693:2063] [node 6] ICP22 created new session: [4:7587241237627624766:2063]
2025-12-24T02:11:00.254340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241237627624766:2063] [node 6] ICS09 handshake done sender: [4:7587241237627624697:2067] self: [4:7587241237627624696:0] peer: [6:7587241236610370091:0] socket: 260 qp: -1
2025-12-24T02:11:00.254347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241237627624766:2063] [node 6] ICS10 traffic start
2025-12-24T02:11:00.254354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.254618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792366:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.254748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792373:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.254752Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237012792373:2103] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.256292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579270:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.256325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241235968579179:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.255401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792389:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.255818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241237012792365:2070] [node 6] ICP18 handshake (actor [2:7587241237012792366:2099]) is too old
2025-12-24T02:11:00.255864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241237012792309:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.257916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.257210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241236610370028:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241236610370086:2114] held: no
2025-12-24T02:11:00.257256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236610370086:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.257317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236610370028:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.257326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.257332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236610370028:2064] [node 4] ICP22 created new session: [6:7587241236610370101:2064]
2025-12-24T02:11:00.257337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236610370101:2064] [node 4] ICS09 handshake done sender: [6:7587241236610370086:2114] self: [6:7587241236610370091:0] peer: [4:7587241237627624696:0] socket: 279 qp: -1
2025-12-24T02:11:00.257343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236610370101:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:00.257350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.259666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792399:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
2025-12-24T02:11:00.259712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
2025-12-24T02:11:00.259778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237012792373:2103] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.260091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792388:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.260097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237012792388:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.260475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792400:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.260493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237012792399:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.260500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237012792326:2063] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.260508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.260515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237012792326:2063] [node 4] ICP22 created new session: [2:7587241237012792403:2063]
2025-12-24T02:11:00.260520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237012792403:2063] [node 4] ICS09 handshake done sender: [2:7587241237012792373:2103] self: [2:7587241237012792372:0] peer: [4:7587241237627624763:0] socket: 296 qp: -1
2025-12-24T02:11:00.260525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237012792403:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:00.260532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.260710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241237012792326:2063] [node 4] ICP18 handshake (actor [2:7587241237012792399:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.260988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579274:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.262433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579273:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.262466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241235968579226:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.262470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241235968579226:2086] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.259498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624768:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.262172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241237627624757:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.262285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236610370092:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.262352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236610370011:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.263080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241235968579226:2086] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.263173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241235968579219:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.263187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.263193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241235968579219:2076] [node 6] ICP22 created new session: [3:7587241235968579277:2076]
2025-12-24T02:11:00.263199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241235968579277:2076] [node 6] ICS09 handshake done sender: [3:7587241235968579226:2086] self: [3:7587241235968579225:0] peer: [6:7587241236610370104:0] socket: 301 qp: -1
2025-12-24T02:11:00.263205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241235968579277:2076] [node 6] ICS10 traffic start
2025-12-24T02:11:00.263212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.266817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714951:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.266831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241234556714951:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.267765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241237627624676:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.267785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.267795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241237627624676:2049] [node 2] ICP22 created new session: [4:7587241237627624770:2049]
2025-12-24T02:11:00.267801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241237627624770:2049] [node 2] ICS09 handshake done sender: [4:7587241237627624757:2118] self: [4:7587241237627624763:0] peer: [2:7587241237012792372:0] socket: 297 qp: -1
2025-12-24T02:11:00.267809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241237627624770:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.267816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.267889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241237627624676:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.266740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683760:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.266759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241234457683760:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.269169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683778:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.270915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237012792388:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.271342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714949:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.271356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241234556714949:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.271019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241236610370011:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241236610370092:2118] held: no
2025-12-24T02:11:00.271090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370105:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.272911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241235968579270:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.272977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241234556714965:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64008
2025-12-24T02:11:00.273066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241234556714949:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.273680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241235968579179:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.273698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.273705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241235968579179:2049] [node 2] ICP22 created new session: [3:7587241235968579280:2049]
2025-12-24T02:11:00.273711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241235968579280:2049] [node 2] ICS09 handshake done sender: [3:7587241235968579270:2119] self: [3:7587241235968579272:0] peer: [2:7587241237012792330:0] socket: 288 qp: -1
2025-12-24T02:11:00.273717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241235968579280:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.273726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.274514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241235968579279:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12872
2025-12-24T02:11:00.274567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714947:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.274578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241234556714947:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.275113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241234556714951:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.275383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241235968579259:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.275232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236610370092:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.275269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370106:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.275281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236610370011:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.275297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.275305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236610370011:2050] [node 3] ICP22 created new session: [6:7587241236610370107:2050]
2025-12-24T02:11:00.275312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236610370107:2050] [node 3] ICS09 handshake done sender: [6:7587241236610370092:2118] self: [6:7587241236610370104:0] peer: [3:7587241235968579225:0] socket: 302 qp: -1
2025-12-24T02:11:00.275320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236610370107:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.275337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.275627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237012792310:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.275648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.275653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237012792310:2049] [node 3] ICP22 created new session: [2:7587241237012792408:2049]
2025-12-24T02:11:00.275659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237012792408:2049] [node 3] ICS09 handshake done sender: [2:7587241237012792388:2115] self: [2:7587241237012792330:0] peer: [3:7587241235968579272:0] socket: 304 qp: -1
2025-12-24T02:11:00.275665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237012792408:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:00.275671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.276156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237012792407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60231
2025-12-24T02:11:00.275807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624756:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.275816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241237627624756:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.276591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241237627624756:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.277197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241237627624733:2089] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.277213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.277219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241237627624733:2089] [node 5] ICP22 created new session: [4:7587241237627624773:2089]
2025-12-24T02:11:00.277225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241237627624773:2089] [node 5] ICS09 handshake done sender: [4:7587241237627624756:2117] self: [4:7587241237627624734:0] peer: [5:7587241234457683782:0] socket: 315 qp: -1
2025-12-24T02:11:00.277230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241237627624773:2089] [node 5] ICS10 traffic start
2025-12-24T02:11:00.277237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.277586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241235968579178:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.277604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.277618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241235968579178:2048] [node 1] ICP22 created new session: [3:7587241235968579282:2048]
2025-12-24T02:11:00.277625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241235968579282:2048] [node 1] ICS09 handshake done sender: [3:7587241235968579259:2117] self: [3:7587241235968579271:0] peer: [1:7587241234556714950:0] socket: 287 qp: -1
2025-12-24T02:11:00.277632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241235968579282:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.277640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.275422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234457683760:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.275513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683778:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.275532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241234457683758:2083] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.275544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.275550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234457683758:2083] [node 6] ICP22 created new session: [5:7587241234457683780:2083]
2025-12-24T02:11:00.275561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234457683780:2083] [node 6] ICS09 handshake done sender: [5:7587241234457683760:2114] self: [5:7587241234457683759:0] peer: [6:7587241236610370098:0] socket: 280 qp: -1
2025-12-24T02:11:00.275566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234457683780:2083] [node 6] ICS10 traffic start
2025-12-24T02:11:00.275600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.275720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234457683714:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.278125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241234457683714:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22697) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241234457683778:2120] held: no
2025-12-24T02:11:00.278719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714965:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.278757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714964:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.278765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241234556714879:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.278777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.278783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241234556714879:2048] [node 2] ICP22 created new session: [1:7587241234556714969:2048]
2025-12-24T02:11:00.278790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241234556714969:2048] [node 2] ICS09 handshake done sender: [1:7587241234556714949:2112] self: [1:7587241234556714948:0] peer: [2:7587241237012792402:0] socket: 276 qp: -1
2025-12-24T02:11:00.278795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241234556714969:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:00.278802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.277632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241236610370027:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26314) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241236610370093:2119] held: no
2025-12-24T02:11:00.277668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236610370093:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.277736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236610370027:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.277744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.277748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236610370027:2059] [node 5] ICP22 created new session: [6:7587241236610370109:2059]
2025-12-24T02:11:00.277752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236610370109:2059] [node 5] ICS09 handshake done sender: [6:7587241236610370093:2119] self: [6:7587241236610370098:0] peer: [5:7587241234457683759:0] socket: 303 qp: -1
2025-12-24T02:11:00.277757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236610370109:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:00.277762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.281574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683783:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.281728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234457683778:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.281780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234457683714:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.281790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.281796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234457683714:2063] [node 4] ICP22 created new session: [5:7587241234457683784:2063]
2025-12-24T02:11:00.281801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234457683784:2063] [node 4] ICS09 handshake done sender: [5:7587241234457683778:2120] self: [5:7587241234457683782:0] peer: [4:7587241237627624734:0] socket: 318 qp: -1
2025-12-24T02:11:00.281807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234457683784:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:00.281816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.282093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236610370111:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64803
2025-12-24T02:11:00.283010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237012792389:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.287370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241234556714880:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.287396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.287403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241234556714880:2049] [node 3] ICP22 created new session: [1:7587241234556714974:2049]
2025-12-24T02:11:00.287411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241234556714974:2049] [node 3] ICS09 handshake done sender: [1:7587241234556714951:2113] self: [1:7587241234556714950:0] peer: [3:7587241235968579271:0] socket: 277 qp: -1
2025-12-24T02:11:00.287418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241234556714974:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:00.287438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.288090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241234556714947:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.286819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241237012792309:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.286847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.286857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237012792309:2048] [node 1] ICP22 created new session: [2:7587241237012792410:2048]
2025-12-24T02:11:00.286863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237012792410:2048] [node 1] ICS09 handshake done sender: [2:7587241237012792389:2116] self: [2:7587241237012792402:0] peer: [1:7587241234556714948:0] socket: 289 qp: -1
2025-12-24T02:11:00.286870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237012792410:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.286881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.288328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241234556714942:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.288346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.288354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241234556714942:2106] [node 6] ICP22 created new session: [1:7587241234556714976:2106]
2025-12-24T02:11:00.288360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241234556714976:2106] [node 6] ICS09 handshake done sender: [1:7587241234556714947:2111] self: [1:7587241234556714944:0] peer: [6:7587241236610370099:0] socket: 275 qp: -1
2025-12-24T02:11:00.288365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241234556714976:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:00.288372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.289286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683786:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.290560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234457683786:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.290592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234457683686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.290979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236610370088:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.290742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714953:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.290750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241234556714953:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.292266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241234556714953:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.292381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241234556714943:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.292396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.292402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241234556714943:2108] [node 5] ICP22 created new session: [1:7587241234556714980:2108]
2025-12-24T02:11:00.292408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241234556714980:2108] [node 5] ICS09 handshake done sender: [1:7587241234556714953:2114] self: [1:7587241234556714952:0] peer: [5:7587241234457683787:0] socket: 274 qp: -1
2025-12-24T02:11:00.292414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241234556714980:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:00.292423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.292540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234457683788:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51814
2025-12-24T02:11:00.292627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234457683786:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.294853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236610370009:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.294873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.294879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236610370009:2048] [node 1] ICP22 created new session: [6:7587241236610370112:2048]
2025-12-24T02:11:00.294887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236610370112:2048] [node 1] ICS09 handshake done sender: [6:7587241236610370088:2116] self: [6:7587241236610370099:0] peer: [1:7587241234556714944:0] socket: 292 qp: -1
2025-12-24T02:11:00.294894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236610370112:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.294900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.295764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241236610370009:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:00.296571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234457683686:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.296592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.296599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234457683686:2048] [node 1] ICP22 created new session: [5:7587241234457683789:2048]
2025-12-24T02:11:00.296606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234457683789:2048] [node 1] ICS09 handshake done sender: [5:7587241234457683786:2122] self: [5:7587241234457683787:0] peer: [1:7587241234556714952:0] socket: 257 qp: -1
2025-12-24T02:11:00.296614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234457683789:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.296622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.297435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241234457683686:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.299064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624775:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.301213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241237627624775:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.301249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241237627624675:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.301375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241234556714960:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.301384Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241234556714960:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.302070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241237627624777:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43352
2025-12-24T02:11:00.302185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241234556714960:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.302346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241237627624775:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.302427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241237627624675:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.302442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.302449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241237627624675:2048] [node 1] ICP22 created new session: [4:7587241237627624778:2048]
2025-12-24T02:11:00.302454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241237627624778:2048] [node 1] ICS09 handshake done sender: [4:7587241237627624775:2121] self: [4:7587241237627624776:0] peer: [1:7587241234556714959:0] socket: 270 qp: -1
2025-12-24T02:11:00.302460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241237627624778:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.302467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.303417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241234556714941:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.303433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.303438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241234556714941:2104] [node 4] ICP22 created new session: [1:7587241234556714983:2104]
2025-12-24T02:11:00.303443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241234556714983:2104] [node 4] ICS09 handshake done sender: [1:7587241234556714960:2117] self: [1:7587241234556714959:0] peer: [4:7587241237627624776:0] socket: 282 qp: -1
2025-12-24T02:11:00.303447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241234556714983:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:00.303453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.314776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:00.314854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241237012792410:2048] [node 1] ICS01 socket: 289 reason# EndOfStream
2025-12-24T02:11:00.314861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241237012792309:2048] [node 1] ICP30 unregister session Session# [2:7587241237012792410:2048] VirtualId# [2:7587241237012792402:0]
2025-12-24T02:11:00.314867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241237012792410:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.314911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.314933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.315962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:00.316043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241235968579282:2048] [node 1] ICS01 socket: 287 reason# EndOfStream
2025-12-24T02:11:00.316051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241235968579178:2048] [node 1] ICP30 unregister session Session# [3:7587241235968579282:2048] VirtualId# [3:7587241235968579271:0]
2025-12-24T02:11:00.316057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241235968579282:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.316106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.316133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.316769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:00.316831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241237627624778:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-12-24T02:11:00.316839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241237627624675:2048] [node 1] ICP30 unregister session Session# [4:7587241237627624778:2048] VirtualId# [4:7587241237627624776:0]
2025-12-24T02:11:00.316844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241237627624778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.316878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.316896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.318071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:00.318147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234457683789:2048] [node 1] ICS01 socket: 257 reason# EndOfStream
2025-12-24T02:11:00.318153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234457683686:2048] [node 1] ICP30 unregister session Session# [5:7587241234457683789:2048] VirtualId# [5:7587241234457683787:0]
2025-12-24T02:11:00.318158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234457683789:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.318201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.318225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.316167Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:00.316239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236610370112:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-12-24T02:11:00.316244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236610370009:2048] [node 1] ICP30 unregister session Session# [6:7587241236610370112:2048] VirtualId# [6:7587241236610370099:0]
2025-12-24T02:11:00.316248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236610370112:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.316274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.316293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.327919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:00.327999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241235968579280:2049] [node 2] ICS01 socket: 288 reason# EndOfStream
2025-12-24T02:11:00.328007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241235968579179:2049] [node 2] ICP30 unregister session Session# [3:7587241235968579280:2049] VirtualId# [3:7587241235968579272:0]
2025-12-24T02:11:00.328012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241235968579280:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.328042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.328057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.330742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:00.330810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241237627624770:2049] [node 2] ICS01 socket: 297 reason# EndOfStream
2025-12-24T02:11:00.330816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241237627624676:2049] [node 2] ICP30 unregister session Session# [4:7587241237627624770:2049] VirtualId# [4:7587241237627624763:0]
2025-12-24T02:11:00.330821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241237627624770:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.330856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.330878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.334195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:00.334281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234457683771:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-12-24T02:11:00.334288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234457683687:2049] [node 2] ICP30 unregister session Session# [5:7587241234457683771:2049] VirtualId# [5:7587241234457683768:0]
2025-12-24T02:11:00.334294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234457683771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.334353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.334382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.334423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:00.334488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236610370096:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-12-24T02:11:00.334493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236610370010:2049] [node 2] ICP30 unregister session Session# [6:7587241236610370096:2049] VirtualId# [6:7587241236610370090:0]
2025-12-24T02:11:00.334497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236610370096:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.334528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.334555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.342828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:00.342906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234457683774:2050] [node 3] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:11:00.342914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234457683688:2050] [node 3] ICP30 unregister session Session# [5:7587241234457683774:2050] VirtualId# [5:7587241234457683767:0]
2025-12-24T02:11:00.342920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234457683774:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.342971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:00.342995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:00.343029Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:00.343095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236610370107:2050] [node 3] ICS01 socket: 302 reason# EndOfStream
2025-12-24T02:11:00.343099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236610370011:2050] [node 3] ICP30 unregister session Session# [6:7587241236610370107:2050] VirtualId# [6:7587241236610370104:0]
2025-12-24T02:11:00.343104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236610370107:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.343131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:00.343169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:00.358801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:00.358894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236610370101:2064] [node 4] ICS01 socket: 279 reason# EndOfStream
2025-12-24T02:11:00.358905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236610370028:2064] [node 4] ICP30 unregister session Session# [6:7587241236610370101:2064] VirtualId# [6:7587241236610370091:0]
2025-12-24T02:11:00.358912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236610370101:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.358971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:00.359000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:00.359041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:00.359107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234457683784:2063] [node 4] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:00.359112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234457683714:2063] [node 4] ICP30 unregister session Session# [5:7587241234457683784:2063] VirtualId# [5:7587241234457683782:0]
2025-12-24T02:11:00.359116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234457683784:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.359145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:00.359181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:00.418812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237115761176:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.418827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237115761177:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.421682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241237091205438:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.421691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241237091205439:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.418899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241237187547344:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:00.418908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241237187547345:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.435107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241235465421208:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.435116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241235465421209:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.435120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241235465421210:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.435741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234017675341:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.435757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234017675342:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.435762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234017675343:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.446103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241237632916383:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.446115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241237632916384:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.446121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241237632916385:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.473961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241237091205497:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:00.478120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241237632916433:2091] [node 5] ICP01 ready to work
2025-12-24T02:11:00.482323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241237187547407:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.482875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421274:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.482932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241237187547410:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36714
2025-12-24T02:11:00.488915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241237187547415:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:00.494105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237115761237:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:00.495008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241235465421274:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.495365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241235465421210:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.495499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205500:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.495512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241237091205500:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.496208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241237091205500:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.496358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241237091205497:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.496370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.496377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241237091205497:2105] [node 4] ICP22 created new session: [3:7587241237091205507:2105]
2025-12-24T02:11:00.496383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241237091205507:2105] [node 4] ICS09 handshake done sender: [3:7587241237091205500:2107] self: [3:7587241237091205499:0] peer: [4:7587241235465421276:0] socket: 258 qp: -1
2025-12-24T02:11:00.496390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241237091205507:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:00.496399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.498986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241237632916443:2100] [node 4] ICP01 ready to work
2025-12-24T02:11:00.498311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237115761244:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.504467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241237187547419:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:00.505693Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241237187547344:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30815868ns) {Configure:104(30815346ns) {ProcessConfigured:135(29389342ns) {StartInitialHandshake:157(29389061ns) {DropHandshakes:470(482ns) {DropIncomingHandshake:442(37ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(564ns) {SwitchToState:179(362ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.503319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241237091205509:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13812
2025-12-24T02:11:00.503573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237115761246:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61214
2025-12-24T02:11:00.503593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237115761247:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61214
2025-12-24T02:11:00.503916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421277:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.503937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421278:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.506959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205509:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.507398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241237091205438:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.506366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675402:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.507178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916452:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
2025-12-24T02:11:00.507575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241235465421274:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.510905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241235465421210:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.510925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.510935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241235465421210:2050] [node 3] ICP22 created new session: [4:7587241235465421279:2050]
2025-12-24T02:11:00.510942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241235465421279:2050] [node 3] ICS09 handshake done sender: [4:7587241235465421274:2109] self: [4:7587241235465421276:0] peer: [3:7587241237091205499:0] socket: 259 qp: -1
2025-12-24T02:11:00.510951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241235465421279:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.510973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.511145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241235465421277:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.511806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241237632916383:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34520941ns) {Configure:104(34519808ns) {ProcessConfigured:135(34028341ns) {StartInitialHandshake:157(34027801ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(1185ns) {SwitchToState:179(803ns)}}}}}
2025-12-24T02:11:00.514488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241235465421208:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.512667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761247:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.515415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234017675402:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.515534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547411:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.515556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241237187547411:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.516335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241237632916452:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.516659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241237632916384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.522724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241237115761250:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:00.522808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761239:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.522813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237115761239:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.523235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761246:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.525036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237115761239:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.523627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547409:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.523644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241237187547409:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.525264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241237187547409:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.525147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237115761237:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.525162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.525169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237115761237:2107] [node 6] ICP22 created new session: [2:7587241237115761258:2107]
2025-12-24T02:11:00.525175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237115761258:2107] [node 6] ICS09 handshake done sender: [2:7587241237115761239:2108] self: [2:7587241237115761238:0] peer: [6:7587241237632916456:0] socket: 275 qp: -1
2025-12-24T02:11:00.525182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237115761258:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:00.525202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.525489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241237187547411:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.525581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241237187547407:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.525597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.525604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241237187547407:2107] [node 4] ICP22 created new session: [1:7587241237187547428:2107]
2025-12-24T02:11:00.525611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241237187547428:2107] [node 4] ICS09 handshake done sender: [1:7587241237187547409:2108] self: [1:7587241237187547408:0] peer: [4:7587241235465421281:0] socket: 269 qp: -1
2025-12-24T02:11:00.525617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241237187547428:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:00.525626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.525758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241237187547345:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.525766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.525794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241237187547345:2049] [node 3] ICP22 created new session: [1:7587241237187547430:2049]
2025-12-24T02:11:00.525799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241237187547430:2049] [node 3] ICS09 handshake done sender: [1:7587241237187547411:2110] self: [1:7587241237187547405:0] peer: [3:7587241237091205510:0] socket: 271 qp: -1
2025-12-24T02:11:00.525804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241237187547430:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:00.525811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.525919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241237187547427:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36714
2025-12-24T02:11:00.522773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916457:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.527090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675405:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.527121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675406:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.527073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237115761261:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61214
2025-12-24T02:11:00.527578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547410:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.530208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241237187547407:2107] [node 4] ICP18 handshake (actor [1:7587241237187547410:2109]) is too old
2025-12-24T02:11:00.530653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234017675406:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.530884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241235465421208:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241235465421277:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.530884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241237632916457:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.530924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241237632916383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.531318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241237187547424:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36714
2025-12-24T02:11:00.531385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241237091205511:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13812
2025-12-24T02:11:00.531472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547427:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.531481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421283:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.531270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234017675404:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:00.531694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234017675404:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.531733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234017675405:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.532044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234017675342:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.531663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241237091205509:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.534785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547421:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.534799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241237187547421:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.534725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421282:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.534828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241235465421283:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.534992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241235465421286:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:00.535385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241235465421286:2116] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.535420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241235465421282:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.535469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241235465421209:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.536433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916461:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
2025-12-24T02:11:00.536598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241237632916452:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.536678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241237632916384:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.536690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.536696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241237632916384:2049] [node 2] ICP22 created new session: [6:7587241237632916462:2049]
2025-12-24T02:11:00.536701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241237632916462:2049] [node 2] ICS09 handshake done sender: [6:7587241237632916452:2111] self: [6:7587241237632916456:0] peer: [2:7587241237115761238:0] socket: 276 qp: -1
2025-12-24T02:11:00.536707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241237632916462:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.536715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.536129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547424:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.536548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241237187547421:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.536642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241237187547419:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.536652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.536658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241237187547419:2112] [node 5] ICP22 created new session: [1:7587241237187547437:2112]
2025-12-24T02:11:00.536663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241237187547437:2112] [node 5] ICS09 handshake done sender: [1:7587241237187547421:2114] self: [1:7587241237187547420:0] peer: [5:7587241234017675407:0] socket: 295 qp: -1
2025-12-24T02:11:00.536675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241237187547437:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:00.536686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.537585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241237632916458:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.537592Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241237632916458:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.537676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241237632916435:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.537683Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241237632916435:2096] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.537711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761256:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.537723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237115761256:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.538314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761252:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.538334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237115761252:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.538092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547418:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.538110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241237187547418:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.538170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421284:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.537275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675410:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.538777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241235465421277:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.538896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761261:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.538921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241237115761176:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.539231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241237091205438:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.539255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.539263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241237091205438:2048] [node 1] ICP22 created new session: [3:7587241237091205512:2048]
2025-12-24T02:11:00.539272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241237091205512:2048] [node 1] ICS09 handshake done sender: [3:7587241237091205509:2110] self: [3:7587241237091205510:0] peer: [1:7587241237187547405:0] socket: 273 qp: -1
2025-12-24T02:11:00.539279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241237091205512:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.539301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.540754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234017675341:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.540812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234017675406:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.540958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234017675341:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.540971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.540979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234017675341:2048] [node 1] ICP22 created new session: [5:7587241234017675411:2048]
2025-12-24T02:11:00.540986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234017675411:2048] [node 1] ICS09 handshake done sender: [5:7587241234017675406:2111] self: [5:7587241234017675407:0] peer: [1:7587241237187547420:0] socket: 299 qp: -1
2025-12-24T02:11:00.540995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234017675411:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.541017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.542768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241237187547423:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.542786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241237187547423:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.544274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241237632916458:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.544587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241237632916435:2096] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.544735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241237632916433:2091] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.544745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.544753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241237632916433:2091] [node 5] ICP22 created new session: [6:7587241237632916466:2091]
2025-12-24T02:11:00.544759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241237632916466:2091] [node 5] ICS09 handshake done sender: [6:7587241237632916435:2096] self: [6:7587241237632916434:0] peer: [5:7587241234017675408:0] socket: 260 qp: -1
2025-12-24T02:11:00.544765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241237632916466:2091] [node 5] ICS10 traffic start
2025-12-24T02:11:00.544773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.544957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237115761256:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.544903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241237632916443:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.544918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.544923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241237632916443:2100] [node 4] ICP22 created new session: [6:7587241237632916468:2100]
2025-12-24T02:11:00.544941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241237632916468:2100] [node 4] ICS09 handshake done sender: [6:7587241237632916458:2114] self: [6:7587241237632916453:0] peer: [4:7587241235465421287:0] socket: 289 qp: -1
2025-12-24T02:11:00.544947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241237632916468:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:00.544952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.545077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237115761250:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.545089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.545095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237115761250:2113] [node 5] ICP22 created new session: [2:7587241237115761266:2113]
2025-12-24T02:11:00.545100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237115761266:2113] [node 5] ICS09 handshake done sender: [2:7587241237115761256:2115] self: [2:7587241237115761255:0] peer: [5:7587241234017675409:0] socket: 294 qp: -1
2025-12-24T02:11:00.545106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237115761266:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:00.545112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.546468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241237187547418:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.546599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237115761252:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.546857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241237091205514:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.546730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241235465421208:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.546755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.546763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241235465421208:2048] [node 1] ICP22 created new session: [4:7587241235465421289:2048]
2025-12-24T02:11:00.546773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241235465421289:2048] [node 1] ICS09 handshake done sender: [4:7587241235465421277:2111] self: [4:7587241235465421281:0] peer: [1:7587241237187547408:0] socket: 274 qp: -1
2025-12-24T02:11:00.546782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241235465421289:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.546805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.547057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421292:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.550975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241235465421282:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.551042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241235465421209:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.551059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.551066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241235465421209:2049] [node 2] ICP22 created new session: [4:7587241235465421293:2049]
2025-12-24T02:11:00.551073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241235465421293:2049] [node 2] ICS09 handshake done sender: [4:7587241235465421282:2113] self: [4:7587241235465421288:0] peer: [2:7587241237115761245:0] socket: 288 qp: -1
2025-12-24T02:11:00.551079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241235465421293:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.551086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.555810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241237187547423:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.555930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241237187547344:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.555954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.555971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241237187547344:2048] [node 2] ICP22 created new session: [1:7587241237187547441:2048]
2025-12-24T02:11:00.555977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241237187547441:2048] [node 2] ICS09 handshake done sender: [1:7587241237187547423:2115] self: [1:7587241237187547404:0] peer: [2:7587241237115761264:0] socket: 296 qp: -1
2025-12-24T02:11:00.555983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241237187547441:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:00.555991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.560035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237115761244:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.560063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.560073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237115761244:2110] [node 4] ICP22 created new session: [2:7587241237115761271:2110]
2025-12-24T02:11:00.560080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237115761271:2110] [node 4] ICS09 handshake done sender: [2:7587241237115761252:2114] self: [2:7587241237115761245:0] peer: [4:7587241235465421288:0] socket: 286 qp: -1
2025-12-24T02:11:00.560086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237115761271:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:00.560096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.560324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241237115761176:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241237115761261:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.560607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916470:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.560809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241237115761270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61214
2025-12-24T02:11:00.561061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237115761261:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.561151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241237115761176:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.561163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.561170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237115761176:2048] [node 1] ICP22 created new session: [2:7587241237115761273:2048]
2025-12-24T02:11:00.561177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237115761273:2048] [node 1] ICS09 handshake done sender: [2:7587241237115761261:2116] self: [2:7587241237115761264:0] peer: [1:7587241237187547404:0] socket: 297 qp: -1
2025-12-24T02:11:00.561183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237115761273:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.561190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.562905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241235465421291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59676
2025-12-24T02:11:00.562986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205514:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.563030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241237091205439:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.562992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241237187547415:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.563022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.563029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241237187547415:2111] [node 6] ICP22 created new session: [1:7587241237187547443:2111]
2025-12-24T02:11:00.563035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241237187547443:2111] [node 6] ICS09 handshake done sender: [1:7587241237187547418:2113] self: [1:7587241237187547416:0] peer: [6:7587241237632916460:0] socket: 283 qp: -1
2025-12-24T02:11:00.563040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241237187547443:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:00.563050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.561825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241237632916457:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.561942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241237632916383:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.561962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.561970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241237632916383:2048] [node 1] ICP22 created new session: [6:7587241237632916471:2048]
2025-12-24T02:11:00.561977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241237632916471:2048] [node 1] ICS09 handshake done sender: [6:7587241237632916457:2113] self: [6:7587241237632916460:0] peer: [1:7587241237187547416:0] socket: 284 qp: -1
2025-12-24T02:11:00.561985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241237632916471:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.561993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.566890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241237091205517:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.571924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675414:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.576905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675415:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.577132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234017675402:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.577246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234017675405:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.577298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234017675404:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.577317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.577324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234017675404:2109] [node 6] ICP22 created new session: [5:7587241234017675416:2109]
2025-12-24T02:11:00.577332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234017675416:2109] [node 6] ICS09 handshake done sender: [5:7587241234017675402:2108] self: [5:7587241234017675408:0] peer: [6:7587241237632916434:0] socket: 267 qp: -1
2025-12-24T02:11:00.577340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234017675416:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:00.577351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.577765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234017675342:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.577775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.577780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234017675342:2049] [node 2] ICP22 created new session: [5:7587241234017675418:2049]
2025-12-24T02:11:00.577784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234017675418:2049] [node 2] ICS09 handshake done sender: [5:7587241234017675405:2110] self: [5:7587241234017675409:0] peer: [2:7587241237115761255:0] socket: 298 qp: -1
2025-12-24T02:11:00.577789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234017675418:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.577795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.579172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241237115761262:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.579191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241237115761262:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.583305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241235465421296:2119] [node 5] ICP01 ready to work
2025-12-24T02:11:00.583339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241235465421283:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.582943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241237091205520:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:00.586896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241234017675343:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24112078ns) {Configure:104(24111274ns) {ProcessConfigured:135(23782367ns) {StartInitialHandshake:157(23781661ns) {DropHandshakes:470(401ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(277ns) PrepareNewSessionHandshake:477(1221ns) {SwitchToState:179(880ns)}}}}}
2025-12-24T02:11:00.586957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241235465421286:2116] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.586975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.586985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241235465421286:2116] [node 6] ICP22 created new session: [4:7587241235465421297:2116]
2025-12-24T02:11:00.586992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241235465421297:2116] [node 6] ICS09 handshake done sender: [4:7587241235465421283:2114] self: [4:7587241235465421287:0] peer: [6:7587241237632916453:0] socket: 270 qp: -1
2025-12-24T02:11:00.587000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241235465421297:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:00.587010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.594341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241237187547415:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(47706661ns) {HandleHandshakeStatus:329(47705562ns) {DropIncomingHandshake:442(9430ns) DropOutgoingHandshake:457(137ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(31191603ns) {ForwardSessionEventToSession:612(31177576ns) ForwardSessionEventToSession:612(2455ns) ForwardSessionEventToSession:612(854ns) ForwardSessionEventToSession:612(983ns) ForwardSessionEventToSession:612(971ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(646ns) ForwardSessionEventToSession:612(853ns) ForwardSessionEventToSession:612(956ns)}}}
2025-12-24T02:11:00.602865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241237115761262:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.603042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241237115761177:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.603070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.603078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241237115761177:2049] [node 3] ICP22 created new session: [2:7587241237115761276:2049]
2025-12-24T02:11:00.603085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241237115761276:2049] [node 3] ICS09 handshake done sender: [2:7587241237115761262:2117] self: [2:7587241237115761251:0] peer: [3:7587241237091205515:0] socket: 306 qp: -1
2025-12-24T02:11:00.603094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241237115761276:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:00.603103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.602987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675422:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.614814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234017675422:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.615022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241234017675424:2117] [node 4] ICP01 ready to work
2025-12-24T02:11:00.615409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234017675424:2117] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.614808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241237091205517:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(47837055ns) {Configure:104(47836097ns) {ProcessConfigured:135(47435849ns) {StartInitialHandshake:157(47435138ns) {DropHandshakes:470(914ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(365ns) PrepareNewSessionHandshake:477(1526ns) {SwitchToState:179(791ns)}}}}}
2025-12-24T02:11:00.615364Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241237091205439:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241237091205514:2112] held: no
2025-12-24T02:11:00.615780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916473:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
2025-12-24T02:11:00.618875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241235465421300:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.618890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241235465421300:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.615586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241237091205523:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13812
2025-12-24T02:11:00.615635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241237091205524:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13812
2025-12-24T02:11:00.618808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241237091205520:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35778949ns) {Configure:104(35778078ns) {ProcessConfigured:135(35438934ns) {StartInitialHandshake:157(35438383ns) {DropHandshakes:470(1273ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(1102ns) {SwitchToState:179(504ns)}}}}}
2025-12-24T02:11:00.618894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241237091205514:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.620256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205523:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.620270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241237091205439:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.620283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.620291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241237091205439:2049] [node 2] ICP22 created new session: [3:7587241237091205528:2049]
2025-12-24T02:11:00.620296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241237091205528:2049] [node 2] ICS09 handshake done sender: [3:7587241237091205514:2112] self: [3:7587241237091205515:0] peer: [2:7587241237115761251:0] socket: 310 qp: -1
2025-12-24T02:11:00.620303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241237091205528:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.620311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.630975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675426:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.631047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.628341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241235465421300:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.630914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241237632916473:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.631330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241237632916385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.634803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241235465421296:2119] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.634822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.634831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241235465421296:2119] [node 5] ICP22 created new session: [4:7587241235465421303:2119]
2025-12-24T02:11:00.634839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241235465421303:2119] [node 5] ICS09 handshake done sender: [4:7587241235465421300:2120] self: [4:7587241235465421299:0] peer: [5:7587241234017675425:0] socket: 285 qp: -1
2025-12-24T02:11:00.634848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241235465421303:2119] [node 5] ICS10 traffic start
2025-12-24T02:11:00.634862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.635050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205522:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.635064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241237091205522:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.635952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234017675422:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.638376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241234017675426:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.638389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234017675424:2117] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.638400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.638407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234017675424:2117] [node 4] ICP22 created new session: [5:7587241234017675428:2117]
2025-12-24T02:11:00.638412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234017675428:2117] [node 4] ICS09 handshake done sender: [5:7587241234017675422:2116] self: [5:7587241234017675425:0] peer: [4:7587241235465421299:0] socket: 314 qp: -1
2025-12-24T02:11:00.638417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234017675428:2117] [node 4] ICS10 traffic start
2025-12-24T02:11:00.638425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.638574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241234017675343:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.638798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241237091205522:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.642829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241237091205526:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.642845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241237091205526:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.646866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241237091205517:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.646887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.646894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241237091205517:2113] [node 6] ICP22 created new session: [3:7587241237091205532:2113]
2025-12-24T02:11:00.646900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241237091205532:2113] [node 6] ICS09 handshake done sender: [3:7587241237091205522:2115] self: [3:7587241237091205518:0] peer: [6:7587241237632916474:0] socket: 317 qp: -1
2025-12-24T02:11:00.646909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241237091205532:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:00.646919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.643191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241237632916475:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42025
2025-12-24T02:11:00.643386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241237632916473:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.646792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241237632916385:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.646808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.646815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241237632916385:2050] [node 3] ICP22 created new session: [6:7587241237632916476:2050]
2025-12-24T02:11:00.646822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241237632916476:2050] [node 3] ICS09 handshake done sender: [6:7587241237632916473:2117] self: [6:7587241237632916474:0] peer: [3:7587241237091205518:0] socket: 318 qp: -1
2025-12-24T02:11:00.646834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241237632916476:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.646845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.651927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241234017675431:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15964
2025-12-24T02:11:00.652195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241237091205526:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.658995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241237091205520:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.659028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.659036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241237091205520:2114] [node 5] ICP22 created new session: [3:7587241237091205535:2114]
2025-12-24T02:11:00.659042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241237091205535:2114] [node 5] ICS09 handshake done sender: [3:7587241237091205526:2118] self: [3:7587241237091205521:0] peer: [5:7587241234017675430:0] socket: 319 qp: -1
2025-12-24T02:11:00.659049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241237091205535:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:00.659058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.662865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241234017675343:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62517) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241234017675426:2118] held: no
2025-12-24T02:11:00.662914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241234017675426:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.666840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241234017675343:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.666866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.666876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241234017675343:2050] [node 3] ICP22 created new session: [5:7587241234017675432:2050]
2025-12-24T02:11:00.666882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241234017675432:2050] [node 3] ICS09 handshake done sender: [5:7587241234017675426:2118] self: [5:7587241234017675430:0] peer: [3:7587241237091205521:0] socket: 320 qp: -1
2025-12-24T02:11:00.666891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241234017675432:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:00.666901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.714814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:00.714901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241237091205512:2048] [node 1] ICS07 socket disconnect 273 reason# EndOfStream
2025-12-24T02:11:00.714910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241237091205512:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.714954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.714974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241237091205512:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:00.716882Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:00.716955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241234017675411:2048] [node 1] ICS07 socket disconnect 299 reason# EndOfStream
2025-12-24T02:11:00.716960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234017675411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.717003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.717021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241234017675411:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:00.717334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241234017675341:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.717342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234017675411:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.717347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234017675341:2048] [node 1] ICP30 unregister session Session# [5:7587241234017675411:2048] VirtualId# [5:7587241234017675407:0]
2025-12-24T02:11:00.717362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.717370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241234017675341:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.717372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.717946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:00.718018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241237632916471:2048] [node 1] ICS07 socket disconnect 284 reason# EndOfStream
2025-12-24T02:11:00.718023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241237632916471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.718066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.718090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241237632916471:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:00.718440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241237632916383:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.718451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241237632916471:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.718455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241237632916383:2048] [node 1] ICP30 unregister session Session# [6:7587241237632916471:2048] VirtualId# [6:7587241237632916460:0]
2025-12-24T02:11:00.718468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.718474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241237632916383:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.718476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.718921Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:00.718993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241235465421289:2048] [node 1] ICS07 socket disconnect 274 reason# EndOfStream
2025-12-24T02:11:00.718997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241235465421289:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.719046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:00.719066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241235465421289:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:00.719424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241235465421208:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.719436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241235465421289:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.719440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241235465421208:2048] [node 1] ICP30 unregister session Session# [4:7587241235465421289:2048] VirtualId# [4:7587241235465421281:0]
2025-12-24T02:11:00.719451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.719457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241235465421208:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.719458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.722877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241237091205438:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.722891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241237091205512:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.722896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241237091205438:2048] [node 1] ICP30 unregister session Session# [3:7587241237091205512:2048] VirtualId# [3:7587241237091205510:0]
2025-12-24T02:11:00.722907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:00.722911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241237091205438:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.722914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27279) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.727491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241234017675341:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:00.728539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241237632916383:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:00.730749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:00.730826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241237091205528:2049] [node 2] ICS07 socket disconnect 310 reason# EndOfStream
2025-12-24T02:11:00.730832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241237091205528:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.730877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.730896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241237091205528:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:00.732538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241237091205438:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:00.734877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241237091205439:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.734891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241237091205528:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.734895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241237091205439:2049] [node 2] ICP30 unregister session Session# [3:7587241237091205528:2049] VirtualId# [3:7587241237091205515:0]
2025-12-24T02:11:00.734905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.734910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241237091205439:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.734913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.736293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:00.736385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241234017675418:2049] [node 2] ICS07 socket disconnect 298 reason# EndOfStream
2025-12-24T02:11:00.736390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234017675418:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.736442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.736463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241234017675418:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:00.736828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241234017675342:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.736839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234017675418:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.736844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234017675342:2049] [node 2] ICP30 unregister session Session# [5:7587241234017675418:2049] VirtualId# [5:7587241234017675409:0]
2025-12-24T02:11:00.736856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.736861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241234017675342:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.736864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.736851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:00.736913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241237632916462:2049] [node 2] ICS07 socket disconnect 276 reason# EndOfStream
2025-12-24T02:11:00.736918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241237632916462:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.736960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.736986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241237632916462:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:00.737286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241237632916384:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.737295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241237632916462:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.737299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241237632916384:2049] [node 2] ICP30 unregister session Session# [6:7587241237632916462:2049] VirtualId# [6:7587241237632916456:0]
2025-12-24T02:11:00.737309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:00.737313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241237632916384:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.737314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.738729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241235465421208:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:00.746925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241234017675342:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:00.746762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:00.746867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241235465421293:2049] [node 2] ICS07 socket disconnect 288 reason# EndOfStream
2025-12-24T02:11:00.746875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241235465421293:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.746950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:00.746987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241235465421293:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:00.748466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241237632916384:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:00.750233Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:00.750330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241234017675432:2050] [node 3] ICS01 socket: 320 reason# EndOfStream
2025-12-24T02:11:00.750337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241234017675343:2050] [node 3] ICP30 unregister session Session# [5:7587241234017675432:2050] VirtualId# [5:7587241234017675430:0]
2025-12-24T02:11:00.750344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241234017675432:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.750400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:00.750425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:00.752609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:00.752681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241237632916476:2050] [node 3] ICS07 socket disconnect 318 reason# EndOfStream
2025-12-24T02:11:00.752687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241237632916476:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.752737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:00.752760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241237632916476:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:00.758941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241237632916385:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62517) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.758958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241237632916476:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.758962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241237632916385:2050] [node 3] ICP30 unregister session Session# [6:7587241237632916476:2050] VirtualId# [6:7587241237632916474:0]
2025-12-24T02:11:00.758979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:00.758984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241237632916385:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.758987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62517) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.766737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:00.766820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241237632916468:2100] [node 4] ICS07 socket disconnect 289 reason# EndOfStream
2025-12-24T02:11:00.766826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241237632916468:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:00.766878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:00.766897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241237632916468:2100] [node 4] ICS15 start handshake
2025-12-24T02:11:00.768539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241237632916385:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:00.768699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241237632916443:2100] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7401) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:00.768712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241237632916468:2100] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:00.768716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241237632916443:2100] [node 4] ICP30 unregister session Session# [6:7587241237632916468:2100] VirtualId# [6:7587241237632916453:0]
2025-12-24T02:11:00.768731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:00.768737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241237632916443:2100] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7401) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:00.768739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7401) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:00.819269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241236950364420:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.819279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241236950364421:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.823439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241236818541575:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:00.823448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241236818541576:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.834820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241236724726826:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.834831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241236724726827:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.834840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241236724726828:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.825792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241235874848580:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.825807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241235874848581:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:00.835078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241236211255642:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.835092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241236211255643:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:00.835113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241236211255641:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.838806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236257573597:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:00.838822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236257573598:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:00.838826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236257573599:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.879018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241236724726875:2093] [node 4] ICP01 ready to work
2025-12-24T02:11:00.879050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241236724726876:2095] [node 6] ICP01 ready to work
2025-12-24T02:11:00.880579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255702:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.888050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241236724726878:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.888061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241236724726878:2096] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.887301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241236211255702:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.887449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241236211255704:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:00.887841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241236211255704:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.888808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241236724726878:2096] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.888635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255706:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.888973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241236211255702:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.889063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241236211255704:2109] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.889073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.889080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241236211255704:2109] [node 5] ICP22 created new session: [4:7587241236211255707:2109]
2025-12-24T02:11:00.889086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241236211255707:2109] [node 5] ICS09 handshake done sender: [4:7587241236211255702:2108] self: [4:7587241236211255705:0] peer: [5:7587241236724726877:0] socket: 266 qp: -1
2025-12-24T02:11:00.889093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241236211255707:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:00.889117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.889401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241236724726875:2093] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.889417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.889422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241236724726875:2093] [node 4] ICP22 created new session: [5:7587241236724726887:2093]
2025-12-24T02:11:00.889428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241236724726887:2093] [node 4] ICS09 handshake done sender: [5:7587241236724726878:2096] self: [5:7587241236724726877:0] peer: [4:7587241236211255705:0] socket: 264 qp: -1
2025-12-24T02:11:00.889434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241236724726887:2093] [node 4] ICS10 traffic start
2025-12-24T02:11:00.889441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:00.890863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236257573621:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:00.902776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241236818541636:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:00.907228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241236257573597:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16286685ns) {Configure:104(16285997ns) {ProcessConfigured:135(14827444ns) {StartInitialHandshake:157(14826797ns) {DropHandshakes:470(470ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(357ns) PrepareNewSessionHandshake:477(1451ns) {SwitchToState:179(663ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.908011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241236257573621:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17078308ns) {Configure:104(17076868ns) {ProcessConfigured:135(15585234ns) {StartInitialHandshake:157(15584889ns) {DropHandshakes:470(289ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(109ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:477(485ns) {SwitchToState:179(178ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.907910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241235874848640:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59930
2025-12-24T02:11:00.908659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255709:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.910743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241236257573637:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:00.914955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241236724726876:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(35847607ns) {Configure:104(35846664ns) {ProcessConfigured:135(32069050ns) {StartInitialHandshake:157(32068109ns) {DropHandshakes:470(755ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(1109ns) {SwitchToState:179(648ns)}}}}}
2025-12-24T02:11:00.914900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241236818541643:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.915625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241236724726900:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:00.915754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241236818541648:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.917417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241236724726901:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.917933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241236950364486:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:00.917964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241236950364487:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:00.916323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241235874848642:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:00.916410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241235874848644:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:00.917807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241235874848656:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59930
2025-12-24T02:11:00.919355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241235874848658:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:00.918891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241236818541575:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35833770ns) {Configure:104(35832570ns) {ProcessConfigured:135(35491930ns) {StartInitialHandshake:157(35491719ns) {DropHandshakes:470(336ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:477(1223ns) {SwitchToState:179(806ns)}}}}}
2025-12-24T02:11:00.922851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241236950364486:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.923089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241236724726900:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.931469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241236257573637:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20634578ns) {Configure:104(20633880ns) {ProcessConfigured:135(20312061ns) {StartInitialHandshake:157(20311150ns) {DropHandshakes:470(617ns) {DropIncomingHandshake:442(109ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(1415ns) {SwitchToState:179(866ns)}}}}}
2025-12-24T02:11:00.930939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241236818541653:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37673
2025-12-24T02:11:00.930988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241236818541654:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37673
2025-12-24T02:11:00.931065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241236818541655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37673
2025-12-24T02:11:00.934216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541639:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.934229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241236818541639:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.934393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241236211255709:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.934532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241236211255711:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:00.934785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241236211255711:2112] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.934931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541645:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.934947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241236818541645:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.938838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541654:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.945505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541653:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.945551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541655:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.943549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848640:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.946769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241236950364487:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.946807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241236950364421:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.947313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241236950364420:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.947341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241236950364420:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24436148ns) {IncomingHandshake:261(24435758ns) {DropIncomingHandshake:442(70ns) PrepareNewSessionHandshake:477(506ns) {SwitchToState:179(132ns)} GenerateSessionVirtualId:988(235ns) IssueIncomingHandshakeReply:193(24429841ns) {SwitchToState:179(52ns)}}}
2025-12-24T02:11:00.947783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241236724726826:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.947839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241236724726826:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24674559ns) {Configure:104(24673972ns) {ProcessConfigured:135(24338088ns) {IncomingHandshake:261(24322075ns) {DropIncomingHandshake:442(59ns) PrepareNewSessionHandshake:477(328ns) {SwitchToState:179(103ns)} GenerateSessionVirtualId:988(425ns) IssueIncomingHandshakeReply:193(24316201ns) {SwitchToState:179(667ns)}}}}}
2025-12-24T02:11:00.946874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236257573676:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:00.950598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236257573675:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:00.950759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236257573670:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.950779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241236257573670:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.950930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236257573676:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.950969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236257573597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.951068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241236257573675:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.951090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236257573637:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.957035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241236818541636:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(54161493ns) {Configure:104(54159811ns) {ProcessConfigured:135(53840497ns) {StartInitialHandshake:157(53839878ns) {DropHandshakes:470(945ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(145ns)} GenerateSessionVirtualId:988(612ns) PrepareNewSessionHandshake:477(811ns) {SwitchToState:179(327ns)}}}}}
2025-12-24T02:11:00.958880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241236818541639:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:00.958941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241236818541645:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.958957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541650:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.958964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241236818541650:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.959090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241236950364490:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:00.959237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241236724726898:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.959251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241236724726898:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.959026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241236257573599:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(42725155ns) {Configure:104(42724605ns) {ProcessConfigured:135(41532421ns) {StartInitialHandshake:157(41531906ns) {DropHandshakes:470(281ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:477(562ns) {SwitchToState:179(196ns)}}}}}
2025-12-24T02:11:00.961879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848648:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.961902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241235874848648:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.959423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241236818541643:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.959444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.959454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241236818541643:2109] [node 5] ICP22 created new session: [1:7587241236818541662:2109]
2025-12-24T02:11:00.959461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241236818541662:2109] [node 5] ICS09 handshake done sender: [1:7587241236818541645:2110] self: [1:7587241236818541644:0] peer: [5:7587241236724726902:0] socket: 280 qp: -1
2025-12-24T02:11:00.959470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241236818541662:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:00.959480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.962894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241236724726901:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.963320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241236724726827:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:00.968840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848646:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:00.968858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241235874848646:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:00.970939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236257573670:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:00.971067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241236257573621:2068] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.971083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:00.971091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236257573621:2068] [node 4] ICP22 created new session: [6:7587241236257573680:2068]
2025-12-24T02:11:00.971098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236257573680:2068] [node 4] ICS09 handshake done sender: [6:7587241236257573670:2112] self: [6:7587241236257573632:0] peer: [4:7587241236211255712:0] socket: 275 qp: -1
2025-12-24T02:11:00.971105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236257573680:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:00.971117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.974204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241236818541576:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.974231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:00.974240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241236818541576:2049] [node 3] ICP22 created new session: [1:7587241236818541666:2049]
2025-12-24T02:11:00.974249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241236818541666:2049] [node 3] ICS09 handshake done sender: [1:7587241236818541639:2108] self: [1:7587241236818541638:0] peer: [3:7587241236950364488:0] socket: 267 qp: -1
2025-12-24T02:11:00.974254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241236818541666:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:00.974262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:00.974800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848656:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.974682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241236724726904:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:00.975020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241236818541650:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.975735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241236818541648:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.975750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.975756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241236818541648:2111] [node 6] ICP22 created new session: [1:7587241236818541669:2111]
2025-12-24T02:11:00.975761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241236818541669:2111] [node 6] ICS09 handshake done sender: [1:7587241236818541650:2112] self: [1:7587241236818541649:0] peer: [6:7587241236257573677:0] socket: 282 qp: -1
2025-12-24T02:11:00.975766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241236818541669:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:00.975784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.975613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241236724726900:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.978780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241236950364420:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241236950364486:2109] held: no
2025-12-24T02:11:00.978925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241236950364490:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19758826ns) {Configure:104(19757964ns) {ProcessConfigured:135(19410041ns) {StartInitialHandshake:157(19409251ns) {DropHandshakes:470(695ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(166ns)}}}}}
2025-12-24T02:11:00.979140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241236818541643:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19699716ns) {HandleHandshakeStatus:329(19698654ns) {DropIncomingHandshake:442(116ns) DropOutgoingHandshake:457(673ns) SwitchToState:179(180ns) ProcessPendingSessionEvents:495(19584663ns) {ForwardSessionEventToSession:612(19577898ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(481ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(941ns)}}}
2025-12-24T02:11:00.982000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241236724726905:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:00.982100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241236724726898:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.987380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255713:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.988311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241236257573597:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241236257573676:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.991115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241235874848646:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:00.992235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241236724726876:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.992252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:00.992259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241236724726876:2095] [node 6] ICP22 created new session: [5:7587241236724726907:2095]
2025-12-24T02:11:00.992264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241236724726907:2095] [node 6] ICS09 handshake done sender: [5:7587241236724726898:2111] self: [5:7587241236724726881:0] peer: [6:7587241236257573678:0] socket: 279 qp: -1
2025-12-24T02:11:00.992270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241236724726907:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:00.992279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:00.992473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241236724726909:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:00.990792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255715:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.990821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241236211255709:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:00.991281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236257573684:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:00.991424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236257573676:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:00.991485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241236257573685:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:00.991594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241236257573675:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:00.991637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236257573597:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.991649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:00.991657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236257573597:2048] [node 1] ICP22 created new session: [6:7587241236257573686:2048]
2025-12-24T02:11:00.991663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236257573686:2048] [node 1] ICS09 handshake done sender: [6:7587241236257573676:2116] self: [6:7587241236257573677:0] peer: [1:7587241236818541649:0] socket: 296 qp: -1
2025-12-24T02:11:00.991670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236257573686:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:00.991689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:00.992148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236257573637:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.992157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.992163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236257573637:2077] [node 5] ICP22 created new session: [6:7587241236257573688:2077]
2025-12-24T02:11:00.992168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241236257573688:2077] [node 5] ICS09 handshake done sender: [6:7587241236257573675:2115] self: [6:7587241236257573678:0] peer: [5:7587241236724726881:0] socket: 292 qp: -1
2025-12-24T02:11:00.992173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241236257573688:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:00.992178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.996498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241236724726901:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:00.998734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241235874848642:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:00.998754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:00.998761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241235874848642:2108] [node 5] ICP22 created new session: [2:7587241235874848664:2108]
2025-12-24T02:11:00.998768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241235874848664:2108] [node 5] ICS09 handshake done sender: [2:7587241235874848646:2110] self: [2:7587241235874848645:0] peer: [5:7587241236724726903:0] socket: 285 qp: -1
2025-12-24T02:11:00.998776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241235874848664:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:00.998785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:00.998828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241236724726827:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:00.998846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:00.998854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241236724726827:2049] [node 2] ICP22 created new session: [5:7587241236724726910:2049]
2025-12-24T02:11:00.998860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241236724726910:2049] [node 2] ICS09 handshake done sender: [5:7587241236724726901:2113] self: [5:7587241236724726903:0] peer: [2:7587241235874848645:0] socket: 287 qp: -1
2025-12-24T02:11:00.998866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241236724726910:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:00.998874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:00.998973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241236211255714:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:00.999367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241236950364496:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:01.002926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241236211255714:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.007505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241245331799:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:01.007593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241245331796:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:01.007647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241245331797:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:01.007657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241245331798:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:01.007998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241245331795:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:01.008177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241236950364486:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.008226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241236950364420:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.008236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.008242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241236950364420:2048] [node 1] ICP22 created new session: [3:7587241241245331802:2048]
2025-12-24T02:11:01.008248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241245331802:2048] [node 1] ICS09 handshake done sender: [3:7587241236950364486:2109] self: [3:7587241236950364488:0] peer: [1:7587241236818541638:0] socket: 271 qp: -1
2025-12-24T02:11:01.008255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241245331802:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.008273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.009059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241245331796:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.012265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241235874848648:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.013848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241240169815964:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59930
2025-12-24T02:11:01.014859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241236950364487:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.014940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241236257573621:2068] [node 4] ICP35 event processing took too much time PendingConnection:279(43828058ns) {HandleHandshakeStatus:329(43827348ns) {DropIncomingHandshake:442(218ns) DropOutgoingHandshake:457(377ns) SwitchToState:179(432ns) ProcessPendingSessionEvents:495(32863301ns) {ForwardSessionEventToSession:612(32847537ns) ForwardSessionEventToSession:612(1797ns) ForwardSessionEventToSession:612(973ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(711ns) ForwardSessionEventToSession:612(1151ns) ForwardSessionEventToSession:612(545ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(1129ns)}}}
2025-12-24T02:11:01.014927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241236211255711:2112] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.014957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.014969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241236211255711:2112] [node 6] ICP22 created new session: [4:7587241240506223017:2112]
2025-12-24T02:11:01.014979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241240506223017:2112] [node 6] ICS09 handshake done sender: [4:7587241236211255709:2111] self: [4:7587241236211255712:0] peer: [6:7587241236257573632:0] socket: 276 qp: -1
2025-12-24T02:11:01.014988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241240506223017:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:01.015012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.018720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241236724726826:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.018753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.018765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241236724726826:2048] [node 1] ICP22 created new session: [5:7587241241019694208:2048]
2025-12-24T02:11:01.018773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241241019694208:2048] [node 1] ICS09 handshake done sender: [5:7587241236724726900:2112] self: [5:7587241236724726902:0] peer: [1:7587241236818541644:0] socket: 281 qp: -1
2025-12-24T02:11:01.018781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241241019694208:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.018808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.018771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241240169815964:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.018816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241235874848580:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.022792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241236211255642:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23478123ns) {Configure:104(23476901ns) {ProcessConfigured:135(23088370ns) {StartInitialHandshake:157(23087956ns) {DropHandshakes:470(408ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(197ns)}}}}}
2025-12-24T02:11:01.023016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241236211255641:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23257062ns) {Configure:104(23256660ns) {ProcessConfigured:135(22957761ns) {StartInitialHandshake:157(22957582ns) {DropHandshakes:470(184ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(435ns) PrepareNewSessionHandshake:477(568ns) {SwitchToState:179(42ns)}}}}}
2025-12-24T02:11:01.023051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241240506223020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:01.023155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241236211255641:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.024156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241241019694210:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:01.022560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241236950364496:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23136490ns) {Configure:104(23135474ns) {ProcessConfigured:135(22833431ns) {StartInitialHandshake:157(22832921ns) {DropHandshakes:470(383ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(130ns) PrepareNewSessionHandshake:477(922ns) {SwitchToState:179(423ns)}}}}}
2025-12-24T02:11:01.022751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241236950364421:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.022761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.022768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241236950364421:2049] [node 2] ICP22 created new session: [3:7587241241245331806:2049]
2025-12-24T02:11:01.022774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241245331806:2049] [node 2] ICS09 handshake done sender: [3:7587241236950364487:2110] self: [3:7587241236950364491:0] peer: [2:7587241235874848647:0] socket: 291 qp: -1
2025-12-24T02:11:01.022781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241245331806:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.022806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.026766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241240506223020:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.026814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241236211255715:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.026844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241236211255642:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.026854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241236211255643:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.022966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241235874848581:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.022986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.022993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241235874848581:2049] [node 3] ICP22 created new session: [2:7587241240169815967:2049]
2025-12-24T02:11:01.022998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241240169815967:2049] [node 3] ICS09 handshake done sender: [2:7587241235874848648:2111] self: [2:7587241235874848647:0] peer: [3:7587241236950364491:0] socket: 286 qp: -1
2025-12-24T02:11:01.023003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241240169815967:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:01.023020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.026845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241235874848642:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(28087233ns) {HandleHandshakeStatus:329(28086464ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(336ns) SwitchToState:179(228ns) ProcessPendingSessionEvents:495(25778121ns) {ForwardSessionEventToSession:612(25767228ns) ForwardSessionEventToSession:612(1693ns) ForwardSessionEventToSession:612(981ns) ForwardSessionEventToSession:612(500ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(862ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(1051ns) ForwardSessionEventToSession:612(478ns)}}}
2025-12-24T02:11:01.032212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241241019694210:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.032315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241236724726828:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.030807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241236257573637:2077] [node 5] ICP35 event processing took too much time PendingConnection:279(38620976ns) {HandleHandshakeStatus:329(38620382ns) {DropIncomingHandshake:442(185ns) DropOutgoingHandshake:457(1058ns) SwitchToState:179(118ns) ProcessPendingSessionEvents:495(29735211ns) {ForwardSessionEventToSession:612(29723188ns) ForwardSessionEventToSession:612(1752ns) ForwardSessionEventToSession:612(859ns) ForwardSessionEventToSession:612(684ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(527ns) ForwardSessionEventToSession:612(479ns) ForwardSessionEventToSession:612(678ns)}}}
2025-12-24T02:11:01.030886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241236950364493:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.030896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241236950364493:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.035625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241236724726826:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(52883878ns) {HandleHandshakeStatus:329(52883365ns) {DropIncomingHandshake:442(873ns) DropOutgoingHandshake:457(112ns) SwitchToState:179(141ns) ProcessPendingSessionEvents:495(188ns)}}
2025-12-24T02:11:01.038803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848652:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.038823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241235874848652:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.035460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241236211255711:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(40659373ns) {HandleHandshakeStatus:329(40658700ns) {DropIncomingHandshake:442(527ns) DropOutgoingHandshake:457(184ns) SwitchToState:179(291ns) ProcessPendingSessionEvents:495(153ns)}}
2025-12-24T02:11:01.040592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241245331801:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.040605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241245331801:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.041001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241245331810:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19984
2025-12-24T02:11:01.038728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541652:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.038750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241236818541652:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.038948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241236818541660:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.038955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241236818541660:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.043628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241245331801:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.046858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241245331810:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.046877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241245331799:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.046889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.046896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241245331799:2114] [node 5] ICP22 created new session: [3:7587241241245331812:2114]
2025-12-24T02:11:01.046902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241245331812:2114] [node 5] ICS09 handshake done sender: [3:7587241241245331801:2119] self: [3:7587241241245331800:0] peer: [5:7587241241019694214:0] socket: 283 qp: -1
2025-12-24T02:11:01.046910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241245331812:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:01.046920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:01.047096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241241245331799:2114] [node 5] ICP18 handshake (actor [3:7587241241245331810:2121]) is too old
2025-12-24T02:11:01.047391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241241113508967:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37673
2025-12-24T02:11:01.051007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241236950364493:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.051114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241236950364490:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.051127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.051135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241236950364490:2111] [node 4] ICP22 created new session: [3:7587241241245331815:2111]
2025-12-24T02:11:01.051141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241245331815:2111] [node 4] ICS09 handshake done sender: [3:7587241236950364493:2112] self: [3:7587241236950364492:0] peer: [4:7587241240506223026:0] socket: 309 qp: -1
2025-12-24T02:11:01.051147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241245331815:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:01.051154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.051037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241236724726828:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241241019694210:2117] held: no
2025-12-24T02:11:01.053368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240552540986:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:01.053424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240552540987:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:01.054977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241236818541652:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.055113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241236818541575:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.055136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.055146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241236818541575:2048] [node 2] ICP22 created new session: [1:7587241241113508969:2048]
2025-12-24T02:11:01.055152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241113508969:2048] [node 2] ICS09 handshake done sender: [1:7587241236818541652:2113] self: [1:7587241236818541637:0] peer: [2:7587241240169815965:0] socket: 307 qp: -1
2025-12-24T02:11:01.055158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241113508969:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:01.055176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.056033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241236818541660:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.056120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241236818541636:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.056135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.056142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241236818541636:2107] [node 4] ICP22 created new session: [1:7587241241113508972:2107]
2025-12-24T02:11:01.056149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241113508972:2107] [node 4] ICS09 handshake done sender: [1:7587241236818541660:2117] self: [1:7587241236818541641:0] peer: [4:7587241240506223023:0] socket: 305 qp: -1
2025-12-24T02:11:01.056155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241113508972:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:01.056161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.056551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241245331797:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.057206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241245331804:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.057214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241245331804:2120] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.057894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241245331804:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.057009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240552540987:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.057039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240552540986:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.057066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236257573599:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.057076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241236257573598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.057804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240552540990:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:01.058047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240552540987:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.058108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236257573599:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.058116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.058121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236257573599:2050] [node 3] ICP22 created new session: [6:7587241240552540991:2050]
2025-12-24T02:11:01.058126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240552540991:2050] [node 3] ICS09 handshake done sender: [6:7587241240552540987:2120] self: [6:7587241240552540988:0] peer: [3:7587241236950364497:0] socket: 333 qp: -1
2025-12-24T02:11:01.058130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240552540991:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.058135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.056836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241235874848652:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.058896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241236950364496:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.058912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.058919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241236950364496:2113] [node 6] ICP22 created new session: [3:7587241241245331818:2113]
2025-12-24T02:11:01.058924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241245331818:2113] [node 6] ICS09 handshake done sender: [3:7587241241245331804:2120] self: [3:7587241236950364497:0] peer: [6:7587241240552540988:0] socket: 329 qp: -1
2025-12-24T02:11:01.058929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241245331818:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:01.058935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.063787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241236724726876:2095] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:01.062823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241235874848644:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.062840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.062847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241235874848644:2109] [node 4] ICP22 created new session: [2:7587241240169815970:2109]
2025-12-24T02:11:01.062852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241240169815970:2109] [node 4] ICS09 handshake done sender: [2:7587241235874848652:2113] self: [2:7587241235874848650:0] peer: [4:7587241240506223025:0] socket: 318 qp: -1
2025-12-24T02:11:01.062858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241240169815970:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:01.062865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.066933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241235874848660:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.066943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241235874848660:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.070856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241236950364490:2111] [node 4] ICP35 event processing took too much time PendingConnection:279(19705742ns) {HandleHandshakeStatus:329(19704988ns) {DropIncomingHandshake:442(108ns) DropOutgoingHandshake:457(132ns) SwitchToState:179(386ns) ProcessPendingSessionEvents:495(19622619ns) {ForwardSessionEventToSession:612(19609421ns) ForwardSessionEventToSession:612(3582ns) ForwardSessionEventToSession:612(865ns) ForwardSessionEventToSession:612(1900ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(525ns)}}}
2025-12-24T02:11:01.070913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241236950364490:2111] [node 4] ICP18 handshake (actor [3:7587241241245331797:2117]) is too old
2025-12-24T02:11:01.074918Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241236818541575:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19764013ns) {HandleHandshakeStatus:329(19763058ns) {DropIncomingHandshake:442(6583ns) DropOutgoingHandshake:457(219ns) SwitchToState:179(732ns) ProcessPendingSessionEvents:495(19660851ns) {ForwardSessionEventToSession:612(19647931ns) ForwardSessionEventToSession:612(3416ns) ForwardSessionEventToSession:612(823ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(706ns) ForwardSessionEventToSession:612(1400ns) ForwardSessionEventToSession:612(544ns) ForwardSessionEventToSession:612(537ns)}}}
2025-12-24T02:11:01.075420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241240169815972:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59930
2025-12-24T02:11:01.075496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241235874848580:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241240169815964:2117] held: no
2025-12-24T02:11:01.075328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241241019694215:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34656
2025-12-24T02:11:01.079800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241241019694210:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.079911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241236724726828:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.079925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.079935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241236724726828:2050] [node 3] ICP22 created new session: [5:7587241241019694216:2050]
2025-12-24T02:11:01.079941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241241019694216:2050] [node 3] ICS09 handshake done sender: [5:7587241241019694210:2117] self: [5:7587241241019694214:0] peer: [3:7587241241245331800:0] socket: 288 qp: -1
2025-12-24T02:11:01.079949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241241019694216:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.079957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.087476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240552540993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59663
2025-12-24T02:11:01.090516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241235874848660:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.093587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241235874848658:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.093613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.093621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241235874848658:2115] [node 6] ICP22 created new session: [2:7587241240169815974:2115]
2025-12-24T02:11:01.093628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241240169815974:2115] [node 6] ICS09 handshake done sender: [2:7587241235874848660:2116] self: [2:7587241235874848659:0] peer: [6:7587241240552540989:0] socket: 323 qp: -1
2025-12-24T02:11:01.093636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241240169815974:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:01.093646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.090742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240552540986:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.090802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241236257573598:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.090827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.090840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241236257573598:2049] [node 2] ICP22 created new session: [6:7587241240552540994:2049]
2025-12-24T02:11:01.090847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240552540994:2049] [node 2] ICS09 handshake done sender: [6:7587241240552540986:2119] self: [6:7587241240552540989:0] peer: [2:7587241235874848659:0] socket: 326 qp: -1
2025-12-24T02:11:01.090855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240552540994:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.090863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.102827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241236211255643:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241236211255715:2115] held: no
2025-12-24T02:11:01.122856Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241236257573598:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32004734ns) {HandleHandshakeStatus:329(32003427ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(4908ns) SwitchToState:179(367ns) ProcessPendingSessionEvents:495(31876480ns) {ForwardSessionEventToSession:612(31859588ns) ForwardSessionEventToSession:612(3963ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(2400ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(1086ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.123308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241240506223027:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:01.123404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241240506223028:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.124289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241236211255715:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.126865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241235874848658:2115] [node 6] ICP35 event processing took too much time PendingConnection:279(33235457ns) {HandleHandshakeStatus:329(33234368ns) {DropIncomingHandshake:442(5310ns) DropOutgoingHandshake:457(179ns) SwitchToState:179(569ns) ProcessPendingSessionEvents:495(33124590ns) {ForwardSessionEventToSession:612(33110418ns) ForwardSessionEventToSession:612(2334ns) ForwardSessionEventToSession:612(1047ns) ForwardSessionEventToSession:612(2448ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(981ns) ForwardSessionEventToSession:612(740ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(1305ns)}}}
2025-12-24T02:11:01.126962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241240169815976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59930
2025-12-24T02:11:01.129494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241240506223020:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.129596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241236211255643:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.129614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.129624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241236211255643:2050] [node 3] ICP22 created new session: [4:7587241240506223030:2050]
2025-12-24T02:11:01.129630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241240506223030:2050] [node 3] ICS09 handshake done sender: [4:7587241236211255715:2115] self: [4:7587241240506223026:0] peer: [3:7587241236950364492:0] socket: 312 qp: -1
2025-12-24T02:11:01.129638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241240506223030:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.129647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.129811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241236211255642:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.129821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.129826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241236211255642:2049] [node 2] ICP22 created new session: [4:7587241240506223032:2049]
2025-12-24T02:11:01.129831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241240506223032:2049] [node 2] ICS09 handshake done sender: [4:7587241240506223020:2118] self: [4:7587241240506223025:0] peer: [2:7587241235874848650:0] socket: 322 qp: -1
2025-12-24T02:11:01.129837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241240506223032:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.129842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.130764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241240169815964:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.134778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241235874848580:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.134799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.134808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241235874848580:2048] [node 1] ICP22 created new session: [2:7587241240169815977:2048]
2025-12-24T02:11:01.134816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241240169815977:2048] [node 1] ICS09 handshake done sender: [2:7587241240169815964:2117] self: [2:7587241240169815965:0] peer: [1:7587241236818541637:0] socket: 308 qp: -1
2025-12-24T02:11:01.134823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241240169815977:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.134831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.136598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241240506223029:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32859
2025-12-24T02:11:01.136728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241236211255714:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.164347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241236211255641:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.164379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.164395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241236211255641:2048] [node 1] ICP22 created new session: [4:7587241240506223034:2048]
2025-12-24T02:11:01.164404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241240506223034:2048] [node 1] ICS09 handshake done sender: [4:7587241236211255714:2114] self: [4:7587241240506223023:0] peer: [1:7587241236818541641:0] socket: 311 qp: -1
2025-12-24T02:11:01.164416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241240506223034:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.164427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.255590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:01.255689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241241245331802:2048] [node 1] ICS01 socket: 271 reason# EndOfStream
2025-12-24T02:11:01.255698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241236950364420:2048] [node 1] ICP30 unregister session Session# [3:7587241241245331802:2048] VirtualId# [3:7587241236950364488:0]
2025-12-24T02:11:01.255706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241241245331802:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.255743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.255766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.258775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:01.258880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241240169815977:2048] [node 1] ICS01 socket: 308 reason# EndOfStream
2025-12-24T02:11:01.258889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241235874848580:2048] [node 1] ICP30 unregister session Session# [2:7587241240169815977:2048] VirtualId# [2:7587241240169815965:0]
2025-12-24T02:11:01.258895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241240169815977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.258956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.258985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.283954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:01.284059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241241019694208:2048] [node 1] ICS07 socket disconnect 281 reason# EndOfStream
2025-12-24T02:11:01.284066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241241019694208:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.284112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.284134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241241019694208:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:01.286758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:01.286858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236257573686:2048] [node 1] ICS01 socket: 296 reason# EndOfStream
2025-12-24T02:11:01.286869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236257573597:2048] [node 1] ICP30 unregister session Session# [6:7587241236257573686:2048] VirtualId# [6:7587241236257573677:0]
2025-12-24T02:11:01.286876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236257573686:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.286934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.286959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.287466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:01.287565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241240506223034:2048] [node 1] ICS01 socket: 311 reason# EndOfStream
2025-12-24T02:11:01.287573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241236211255641:2048] [node 1] ICP30 unregister session Session# [4:7587241240506223034:2048] VirtualId# [4:7587241240506223023:0]
2025-12-24T02:11:01.287580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241240506223034:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.287643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.287671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.295458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241236724726826:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.295480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241241019694208:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.295487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241236724726826:2048] [node 1] ICP30 unregister session Session# [5:7587241241019694208:2048] VirtualId# [5:7587241236724726902:0]
2025-12-24T02:11:01.295508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.295514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241236724726826:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.295517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10643) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.307368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241236724726826:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:01.346658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:01.346969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241241245331806:2049] [node 2] ICS01 socket: 291 reason# EndOfStream
2025-12-24T02:11:01.346981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241236950364421:2049] [node 2] ICP30 unregister session Session# [3:7587241241245331806:2049] VirtualId# [3:7587241236950364491:0]
2025-12-24T02:11:01.346989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241241245331806:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.347052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.347080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.349349Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:01.349462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241236724726910:2049] [node 2] ICS07 socket disconnect 287 reason# EndOfStream
2025-12-24T02:11:01.349470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241236724726910:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.349506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.349526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241236724726910:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:01.350537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241236724726827:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6890) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.350548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241236724726910:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.350553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241236724726827:2049] [node 2] ICP30 unregister session Session# [5:7587241236724726910:2049] VirtualId# [5:7587241236724726903:0]
2025-12-24T02:11:01.350572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.350578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241236724726827:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.350581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6890) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.350870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:01.350952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241240506223032:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-12-24T02:11:01.350959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241236211255642:2049] [node 2] ICP30 unregister session Session# [4:7587241240506223032:2049] VirtualId# [4:7587241240506223025:0]
2025-12-24T02:11:01.350967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241240506223032:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.351005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.351029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.352240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:01.352339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240552540994:2049] [node 2] ICS01 socket: 326 reason# EndOfStream
2025-12-24T02:11:01.352346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236257573598:2049] [node 2] ICP30 unregister session Session# [6:7587241240552540994:2049] VirtualId# [6:7587241240552540989:0]
2025-12-24T02:11:01.352353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240552540994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.352409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.352436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.362116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241236724726827:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:01.374807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:01.374909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241240506223030:2050] [node 3] ICS01 socket: 312 reason# EndOfStream
2025-12-24T02:11:01.374919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241236211255643:2050] [node 3] ICP30 unregister session Session# [4:7587241240506223030:2050] VirtualId# [4:7587241240506223026:0]
2025-12-24T02:11:01.374926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241240506223030:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.374986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:01.375014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:01.375055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:01.375125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241241019694216:2050] [node 3] ICS01 socket: 288 reason# EndOfStream
2025-12-24T02:11:01.375131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241236724726828:2050] [node 3] ICP30 unregister session Session# [5:7587241241019694216:2050] VirtualId# [5:7587241241019694214:0]
2025-12-24T02:11:01.375135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241241019694216:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.375161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:01.375198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:01.378543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:01.378644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240552540991:2050] [node 3] ICS01 socket: 333 reason# EndOfStream
2025-12-24T02:11:01.378654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236257573599:2050] [node 3] ICP30 unregister session Session# [6:7587241240552540991:2050] VirtualId# [6:7587241240552540988:0]
2025-12-24T02:11:01.378662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240552540991:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.378821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:01.378862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:01.386039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:01.386131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241236257573680:2068] [node 4] ICS01 socket: 275 reason# EndOfStream
2025-12-24T02:11:01.386141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241236257573621:2068] [node 4] ICP30 unregister session Session# [6:7587241236257573680:2068] VirtualId# [6:7587241236257573632:0]
2025-12-24T02:11:01.386147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241236257573680:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.386215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:01.386241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:01.430146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241241039364721:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:01.430154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241241039364722:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:01.436312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241233741599:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.436321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241233741600:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.438261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238055494236:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.438280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238055494237:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:01.440729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238965316081:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.440737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238965316082:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.440742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238965316083:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.458142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241239837567052:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.458175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241239837567053:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.466706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241239837567051:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.502737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240942444615:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.507244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240942444617:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.507354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240942444616:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.510663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741659:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.510795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494296:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
2025-12-24T02:11:01.511344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741660:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.511023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240942444639:2063] [node 5] ICP01 ready to work
2025-12-24T02:11:01.511100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241241039364785:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.511027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238965316129:2094] [node 4] ICP01 ready to work
2025-12-24T02:11:01.516134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240942444655:2073] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.519530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567112:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.526934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238055494305:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:01.530781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741660:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.532428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241241039364794:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.534514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494310:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
2025-12-24T02:11:01.534267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241240942444655:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18076243ns) {Configure:104(18075653ns) {ProcessConfigured:135(17787635ns) {StartInitialHandshake:157(17787094ns) {DropHandshakes:470(422ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(171ns) PrepareNewSessionHandshake:477(904ns) {SwitchToState:179(21ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.534866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741665:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.535216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238055494308:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:01.535870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494296:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.535910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241238055494236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.534795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316147:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.538734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241241039364786:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17430
2025-12-24T02:11:01.539018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241241039364785:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27518239ns) {Configure:104(27517573ns) {ProcessConfigured:135(27314821ns) {StartInitialHandshake:157(27314466ns) {DropHandshakes:470(214ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(885ns) {SwitchToState:179(363ns)}}}}}
2025-12-24T02:11:01.543132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241238055494305:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16027960ns) {Configure:104(16027124ns) {ProcessConfigured:135(15696576ns) {StartInitialHandshake:157(15695949ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:477(1218ns) {SwitchToState:179(606ns)}}}}}
2025-12-24T02:11:01.543770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316147:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.546752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364782:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.546764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241241039364782:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.546835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241241039364791:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:01.546886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364786:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.547384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241241039364802:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17430
2025-12-24T02:11:01.547433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241241039364803:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17430
2025-12-24T02:11:01.547658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364803:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.550900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364802:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.551557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741665:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.551627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241241233741599:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.551647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567113:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.551809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364788:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.551815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241241039364788:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.552407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741667:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.552563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241241039364788:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.552739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241233741665:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.552845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241241233741599:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.552863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.552868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241233741599:2048] [node 1] ICP22 created new session: [3:7587241241233741668:2048]
2025-12-24T02:11:01.552874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241233741668:2048] [node 1] ICS09 handshake done sender: [3:7587241241233741665:2110] self: [3:7587241241233741666:0] peer: [1:7587241241039364787:0] socket: 293 qp: -1
2025-12-24T02:11:01.552880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241233741668:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.552889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.553790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241233741671:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:01.555322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238055494306:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:01.555182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238965316135:2097] [node 6] ICP01 ready to work
2025-12-24T02:11:01.556052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238965316081:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.556178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241241039364722:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.556199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.556205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241241039364722:2049] [node 3] ICP22 created new session: [1:7587241241039364807:2049]
2025-12-24T02:11:01.556211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241039364807:2049] [node 3] ICS09 handshake done sender: [1:7587241241039364788:2110] self: [1:7587241241039364787:0] peer: [3:7587241241233741666:0] socket: 290 qp: -1
2025-12-24T02:11:01.556216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241039364807:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:01.556224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.555759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741659:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.556830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241233741678:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:01.557110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241241233741678:2115] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.559280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494310:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.560970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241233741681:2116] [node 5] ICP01 ready to work
2025-12-24T02:11:01.558805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364796:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.558815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241241039364796:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.559157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241239837567115:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.562898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240942444631:2061] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.562913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241240942444631:2061] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.563369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444693:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.564000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316131:2095] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.564009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241238965316131:2095] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.564549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316152:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.563477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567112:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.563504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567113:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.563649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241239837567120:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:01.563864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241239837567120:2111] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.563454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240942444631:2061] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.563582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241240942444617:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.563593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.563601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240942444617:2050] [node 3] ICP22 created new session: [6:7587241240942444695:2050]
2025-12-24T02:11:01.563607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240942444695:2050] [node 3] ICS09 handshake done sender: [6:7587241240942444631:2061] self: [6:7587241240942444630:0] peer: [3:7587241241233741679:0] socket: 273 qp: -1
2025-12-24T02:11:01.563614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240942444695:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.563643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.565085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238965316131:2095] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.565175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241238965316129:2094] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.565184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.565189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238965316129:2094] [node 4] ICP22 created new session: [5:7587241238965316154:2094]
2025-12-24T02:11:01.565194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241238965316154:2094] [node 4] ICS09 handshake done sender: [5:7587241238965316131:2095] self: [5:7587241238965316130:0] peer: [4:7587241239837567121:0] socket: 281 qp: -1
2025-12-24T02:11:01.565199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241238965316154:2094] [node 4] ICS10 traffic start
2025-12-24T02:11:01.565206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.565460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316152:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.567162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.567738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241233741659:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.568132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241241233741678:2115] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.568142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.568147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241233741678:2115] [node 6] ICP22 created new session: [3:7587241241233741684:2115]
2025-12-24T02:11:01.568151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241233741684:2115] [node 6] ICS09 handshake done sender: [3:7587241241233741659:2107] self: [3:7587241241233741679:0] peer: [6:7587241240942444630:0] socket: 274 qp: -1
2025-12-24T02:11:01.568157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241233741684:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:01.568161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.567289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567122:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.567334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567123:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.567632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567122:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.567647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241239837567112:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:01.567826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241239837567120:2111] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.567835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.567840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241239837567120:2111] [node 5] ICP22 created new session: [4:7587241239837567124:2111]
2025-12-24T02:11:01.567844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241239837567124:2111] [node 5] ICS09 handshake done sender: [4:7587241239837567112:2108] self: [4:7587241239837567121:0] peer: [5:7587241238965316130:0] socket: 282 qp: -1
2025-12-24T02:11:01.567850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241239837567124:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:01.567866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.571805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494319:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
2025-12-24T02:11:01.573959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241241233741600:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36435436ns) {Configure:104(36434736ns) {ProcessConfigured:135(36136673ns) {StartInitialHandshake:157(36136352ns) {DropHandshakes:470(461ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:477(655ns) {SwitchToState:179(507ns)}}}}}
2025-12-24T02:11:01.574009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241241233741600:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.571935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241239837567052:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.574742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241241039364796:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:01.574892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494319:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.574931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241241039364782:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.576068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241241039364794:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.576083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.576091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241241039364794:2112] [node 5] ICP22 created new session: [1:7587241241039364812:2112]
2025-12-24T02:11:01.576097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241039364812:2112] [node 5] ICS09 handshake done sender: [1:7587241241039364796:2113] self: [1:7587241241039364795:0] peer: [5:7587241238965316151:0] socket: 291 qp: -1
2025-12-24T02:11:01.576104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241039364812:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:01.576127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:01.578681Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241241233741681:2116] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17665863ns) {Configure:104(17665246ns) {ProcessConfigured:135(17401786ns) {StartInitialHandshake:157(17401518ns) {DropHandshakes:470(438ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(115ns) PrepareNewSessionHandshake:477(790ns) {SwitchToState:179(339ns)}}}}}
2025-12-24T02:11:01.579099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241241039364722:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(22898105ns) {HandleHandshakeStatus:329(22897638ns) {DropIncomingHandshake:442(3490ns) DropOutgoingHandshake:457(191ns) SwitchToState:179(195ns) ProcessPendingSessionEvents:495(16265354ns) {ForwardSessionEventToSession:612(16260131ns) ForwardSessionEventToSession:612(1555ns) ForwardSessionEventToSession:612(814ns) ForwardSessionEventToSession:612(577ns)}}}
2025-12-24T02:11:01.579124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241241039364721:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.579142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.579149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241241039364721:2048] [node 2] ICP22 created new session: [1:7587241241039364814:2048]
2025-12-24T02:11:01.579154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241039364814:2048] [node 2] ICS09 handshake done sender: [1:7587241241039364782:2107] self: [1:7587241241039364781:0] peer: [2:7587241238055494312:0] socket: 270 qp: -1
2025-12-24T02:11:01.579161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241039364814:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:01.579183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.578495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241239837567053:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18070278ns) {Configure:104(18067704ns) {ProcessConfigured:135(17814441ns) {StartInitialHandshake:157(17813666ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(954ns) {SwitchToState:179(598ns)}}}}}
2025-12-24T02:11:01.578532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241239837567053:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.578858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241239837567115:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17398459ns) {Configure:104(17396650ns) {ProcessConfigured:135(17142880ns) {StartInitialHandshake:157(17142395ns) {DropHandshakes:470(394ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(534ns) {SwitchToState:179(272ns)}}}}}
2025-12-24T02:11:01.582768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494313:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.582791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238055494313:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.582850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494298:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.582858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238055494298:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.582917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241238055494236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241238055494296:2107] held: no
2025-12-24T02:11:01.582818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741673:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.582838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241233741673:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.587157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241238965316083:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16273025ns) {Configure:104(16272144ns) {ProcessConfigured:135(15979419ns) {StartInitialHandshake:157(15978629ns) {DropHandshakes:470(848ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(1498ns) {SwitchToState:179(139ns)}}}}}
2025-12-24T02:11:01.587499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.587552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316166:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.589147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238965316147:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.589267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238965316081:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.589281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.589291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238965316081:2048] [node 1] ICP22 created new session: [5:7587241238965316167:2048]
2025-12-24T02:11:01.589298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241238965316167:2048] [node 1] ICS09 handshake done sender: [5:7587241238965316147:2109] self: [5:7587241238965316151:0] peer: [1:7587241241039364795:0] socket: 292 qp: -1
2025-12-24T02:11:01.589305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241238965316167:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.589341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.594957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
2025-12-24T02:11:01.595219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.595944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238055494321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1866
2025-12-24T02:11:01.596015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238055494296:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.598838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240942444693:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.598885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240942444615:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.603373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567135:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.607360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494321:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.607391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241238055494236:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.607406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.607413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238055494236:2048] [node 1] ICP22 created new session: [2:7587241238055494324:2048]
2025-12-24T02:11:01.607419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241238055494324:2048] [node 1] ICS09 handshake done sender: [2:7587241238055494296:2107] self: [2:7587241238055494312:0] peer: [1:7587241241039364781:0] socket: 272 qp: -1
2025-12-24T02:11:01.607426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241238055494324:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.607444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.607595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494323:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.610829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241241039364794:2112] [node 5] ICP35 event processing took too much time PendingConnection:279(34725429ns) {HandleHandshakeStatus:329(34724374ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(72ns) ProcessPendingSessionEvents:495(27940064ns) {ForwardSessionEventToSession:612(27933172ns) ForwardSessionEventToSession:612(1948ns) ForwardSessionEventToSession:612(962ns) ForwardSessionEventToSession:612(583ns)}}}
2025-12-24T02:11:01.610848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567135:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.610998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567136:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.614893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238055494313:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.615486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238055494305:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.615505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.615513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238055494305:2111] [node 4] ICP22 created new session: [2:7587241238055494329:2111]
2025-12-24T02:11:01.615520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241238055494329:2111] [node 4] ICS09 handshake done sender: [2:7587241238055494313:2114] self: [2:7587241238055494309:0] peer: [4:7587241239837567127:0] socket: 294 qp: -1
2025-12-24T02:11:01.615528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241238055494329:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:01.615537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.614872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316166:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.614929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238965316083:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.618857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241239837567113:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.619411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.619433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741691:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.622858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241233741673:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.623898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241239837567052:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.623925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.623935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241239837567052:2049] [node 2] ICP22 created new session: [4:7587241239837567137:2049]
2025-12-24T02:11:01.623944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241239837567137:2049] [node 2] ICS09 handshake done sender: [4:7587241239837567113:2109] self: [4:7587241239837567127:0] peer: [2:7587241238055494309:0] socket: 295 qp: -1
2025-12-24T02:11:01.623951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241239837567137:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.623961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.626747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741687:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.626767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241233741687:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.626807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741691:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.626832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241233741671:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.626850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.626862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241233741671:2112] [node 4] ICP22 created new session: [3:7587241241233741692:2112]
2025-12-24T02:11:01.626787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364798:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.626816Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241241039364798:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.626874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241233741692:2112] [node 4] ICS09 handshake done sender: [3:7587241241233741673:2113] self: [3:7587241241233741672:0] peer: [4:7587241239837567131:0] socket: 299 qp: -1
2025-12-24T02:11:01.626899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241233741692:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:01.626974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238055494298:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.626910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.626880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444698:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.630382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238055494237:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.630409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.630415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238055494237:2049] [node 3] ICP22 created new session: [2:7587241238055494333:2049]
2025-12-24T02:11:01.630422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241238055494333:2049] [node 3] ICS09 handshake done sender: [2:7587241238055494298:2108] self: [2:7587241238055494297:0] peer: [3:7587241241233741686:0] socket: 275 qp: -1
2025-12-24T02:11:01.630430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241238055494333:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:01.630437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.633476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241241233741600:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241241233741660:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.633039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240942444698:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.633101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240942444639:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.635232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241233741690:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.634796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241239837567051:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36280777ns) {Configure:104(36277798ns) {ProcessConfigured:135(35987708ns) {StartInitialHandshake:157(35986716ns) {DropHandshakes:470(1051ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(698ns)}}}}}
2025-12-24T02:11:01.639677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241239837567051:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.639703Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241239837567052:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.642062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364801:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.642078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241241039364801:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.642614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241241039364817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17430
2025-12-24T02:11:01.642911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.646723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316156:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.646744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241238965316156:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.647423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241233741695:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61166
2025-12-24T02:11:01.647364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241241039364801:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.650104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241233741687:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:01.650204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241233741681:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.650223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.650231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241233741681:2116] [node 5] ICP22 created new session: [3:7587241241233741697:2116]
2025-12-24T02:11:01.650238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241233741697:2116] [node 5] ICS09 handshake done sender: [3:7587241241233741687:2118] self: [3:7587241241233741682:0] peer: [5:7587241238965316169:0] socket: 284 qp: -1
2025-12-24T02:11:01.650250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241233741697:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:01.650259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:01.650276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241233741660:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.650346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241241039364817:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.650406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241241039364798:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.650471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241241233741600:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.650482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.650492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241233741600:2049] [node 2] ICP22 created new session: [3:7587241241233741699:2049]
2025-12-24T02:11:01.650499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241241233741699:2049] [node 2] ICS09 handshake done sender: [3:7587241241233741660:2108] self: [3:7587241241233741686:0] peer: [2:7587241238055494297:0] socket: 280 qp: -1
2025-12-24T02:11:01.650505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241241233741699:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.650515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.647243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.650752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241239837567122:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.650507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241241039364791:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.650522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.650529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241241039364791:2111] [node 4] ICP22 created new session: [1:7587241241039364820:2111]
2025-12-24T02:11:01.650530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241241039364785:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.650536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241039364820:2111] [node 4] ICS09 handshake done sender: [1:7587241241039364801:2115] self: [1:7587241241039364800:0] peer: [4:7587241239837567141:0] socket: 320 qp: -1
2025-12-24T02:11:01.650543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241039364820:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:01.650546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.650549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.650554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241241039364785:2108] [node 6] ICP22 created new session: [1:7587241241039364821:2108]
2025-12-24T02:11:01.650562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241241039364821:2108] [node 6] ICS09 handshake done sender: [1:7587241241039364798:2114] self: [1:7587241241039364790:0] peer: [6:7587241240942444699:0] socket: 296 qp: -1
2025-12-24T02:11:01.650568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241241039364821:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:01.650575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.650765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241241039364791:2111] [node 4] ICP18 handshake (actor [1:7587241241039364817:2118]) is too old
2025-12-24T02:11:01.650788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241241233741671:2112] [node 4] ICP35 event processing took too much time PendingConnection:279(23925115ns) {HandleHandshakeStatus:329(23924205ns) {DropIncomingHandshake:442(487ns) DropOutgoingHandshake:457(370ns) SwitchToState:179(273ns) ProcessPendingSessionEvents:495(23778853ns) {ForwardSessionEventToSession:612(23770033ns) ForwardSessionEventToSession:612(1366ns) ForwardSessionEventToSession:612(671ns) ForwardSessionEventToSession:612(1203ns) ForwardSessionEventToSession:612(524ns) ForwardSessionEventToSession:612(536ns) ForwardSessionEventToSession:612(766ns)}}}
2025-12-24T02:11:01.650842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241241233741671:2112] [node 4] ICP18 handshake (actor [3:7587241241233741690:2119]) is too old
2025-12-24T02:11:01.651050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240942444615:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241240942444693:2115] held: no
2025-12-24T02:11:01.654772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241239837567053:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.654795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.654802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241239837567053:2050] [node 3] ICP22 created new session: [4:7587241239837567144:2050]
2025-12-24T02:11:01.654810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241239837567144:2050] [node 3] ICS09 handshake done sender: [4:7587241239837567122:2112] self: [4:7587241239837567131:0] peer: [3:7587241241233741672:0] socket: 301 qp: -1
2025-12-24T02:11:01.654818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241239837567144:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.654825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.655226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444701:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.655289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.655330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444703:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.658807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240942444702:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.657612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316170:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.657674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316171:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.658947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238965316156:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.659073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238965316170:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.659107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238965316082:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.659279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241238965316135:2097] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.659293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.659300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238965316135:2097] [node 6] ICP22 created new session: [5:7587241238965316174:2097]
2025-12-24T02:11:01.659306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241238965316174:2097] [node 6] ICS09 handshake done sender: [5:7587241238965316156:2113] self: [5:7587241238965316150:0] peer: [6:7587241240942444700:0] socket: 309 qp: -1
2025-12-24T02:11:01.659312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241238965316174:2097] [node 6] ICS10 traffic start
2025-12-24T02:11:01.659318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.659071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240942444693:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.659665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238965316166:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:01.667794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241239837567051:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241239837567135:2117] held: no
2025-12-24T02:11:01.667844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241239837567135:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:01.667944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241239837567051:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.667958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.667964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241239837567051:2048] [node 1] ICP22 created new session: [4:7587241239837567146:2048]
2025-12-24T02:11:01.667969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241239837567146:2048] [node 1] ICS09 handshake done sender: [4:7587241239837567135:2117] self: [4:7587241239837567141:0] peer: [1:7587241241039364800:0] socket: 322 qp: -1
2025-12-24T02:11:01.667977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241239837567146:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.667982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.674883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238965316083:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.674906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:01.674915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238965316083:2050] [node 3] ICP22 created new session: [5:7587241238965316176:2050]
2025-12-24T02:11:01.674921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241238965316176:2050] [node 3] ICS09 handshake done sender: [5:7587241238965316166:2117] self: [5:7587241238965316169:0] peer: [3:7587241241233741682:0] socket: 324 qp: -1
2025-12-24T02:11:01.674929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241238965316176:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:01.674946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:01.674735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494315:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.674762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238055494315:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.674838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238055494318:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.674846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238055494318:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:01.678451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238055494318:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:01.678556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238055494306:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.678573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.678579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238055494306:2110] [node 5] ICP22 created new session: [2:7587241238055494337:2110]
2025-12-24T02:11:01.678585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241238055494337:2110] [node 5] ICS09 handshake done sender: [2:7587241238055494318:2116] self: [2:7587241238055494316:0] peer: [5:7587241238965316173:0] socket: 286 qp: -1
2025-12-24T02:11:01.678590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241238055494337:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:01.678597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:01.682812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240942444615:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.682836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:01.682846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240942444615:2048] [node 1] ICP22 created new session: [6:7587241240942444706:2048]
2025-12-24T02:11:01.682853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240942444706:2048] [node 1] ICS09 handshake done sender: [6:7587241240942444693:2115] self: [6:7587241240942444699:0] peer: [1:7587241241039364790:0] socket: 302 qp: -1
2025-12-24T02:11:01.682861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240942444706:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:01.682871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:01.683071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241240942444701:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.690999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240942444639:2063] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9897) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241240942444698:2116] held: no
2025-12-24T02:11:01.691100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240942444655:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.694438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241238965316135:2097] [node 6] ICP35 event processing took too much time PendingConnection:279(35115935ns) {HandleHandshakeStatus:329(35115362ns) {DropIncomingHandshake:442(2271ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(39ns) ProcessPendingSessionEvents:495(35042958ns) {ForwardSessionEventToSession:612(35027050ns) ForwardSessionEventToSession:612(3336ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(1012ns) ForwardSessionEventToSession:612(851ns) ForwardSessionEventToSession:612(966ns) ForwardSessionEventToSession:612(840ns) ForwardSessionEventToSession:612(1034ns) ForwardSessionEventToSession:612(494ns)}}}
2025-12-24T02:11:01.694469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241238965316082:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241238965316170:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.695410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238965316178:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43302
2025-12-24T02:11:01.699195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.699197Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241239837567053:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44377055ns) {HandleHandshakeStatus:329(44376558ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(2280ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(44274579ns) {ForwardSessionEventToSession:612(44262842ns) ForwardSessionEventToSession:612(2653ns) ForwardSessionEventToSession:612(907ns) ForwardSessionEventToSession:612(1115ns) ForwardSessionEventToSession:612(940ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(1183ns)}}}
2025-12-24T02:11:01.699838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240942444616:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.699869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241240942444616:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(41004364ns) {IncomingHandshake:261(41003940ns) {DropIncomingHandshake:442(59ns) PrepareNewSessionHandshake:477(1023ns) {SwitchToState:179(93ns)} GenerateSessionVirtualId:988(247ns) IssueIncomingHandshakeReply:193(40997226ns) {SwitchToState:179(916ns)}}}
2025-12-24T02:11:01.706142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240942444698:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:01.711656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238965316170:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.713548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238965316082:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.713564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.713573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238965316082:2049] [node 2] ICP22 created new session: [5:7587241238965316179:2049]
2025-12-24T02:11:01.713579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241238965316179:2049] [node 2] ICS09 handshake done sender: [5:7587241238965316170:2118] self: [5:7587241238965316173:0] peer: [2:7587241238055494316:0] socket: 300 qp: -1
2025-12-24T02:11:01.713588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241238965316179:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.713597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.717161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238055494315:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.717323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238055494308:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.717342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.717349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238055494308:2112] [node 6] ICP22 created new session: [2:7587241238055494340:2112]
2025-12-24T02:11:01.717356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241238055494340:2112] [node 6] ICS09 handshake done sender: [2:7587241238055494315:2115] self: [2:7587241238055494311:0] peer: [6:7587241240942444704:0] socket: 332 qp: -1
2025-12-24T02:11:01.717361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241238055494340:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:01.717382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:01.715501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240942444639:2063] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.715522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:01.715758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241238965316083:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(40842648ns) {HandleHandshakeStatus:329(40841563ns) {DropIncomingHandshake:442(793ns) DropOutgoingHandshake:457(1082ns) SwitchToState:179(128ns) ProcessPendingSessionEvents:495(40731539ns) {ForwardSessionEventToSession:612(40720157ns) ForwardSessionEventToSession:612(1281ns) ForwardSessionEventToSession:612(1504ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(666ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(872ns) ForwardSessionEventToSession:612(833ns) ForwardSessionEventToSession:612(572ns) ForwardSessionEventToSession:612(812ns)}}}
2025-12-24T02:11:01.715828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241238965316083:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:01.715533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240942444639:2063] [node 5] ICP22 created new session: [6:7587241240942444710:2063]
2025-12-24T02:11:01.715539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240942444710:2063] [node 5] ICS09 handshake done sender: [6:7587241240942444698:2116] self: [6:7587241240942444700:0] peer: [5:7587241238965316150:0] socket: 310 qp: -1
2025-12-24T02:11:01.715545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240942444710:2063] [node 5] ICS10 traffic start
2025-12-24T02:11:01.715553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.720034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240942444616:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241240942444702:2118] held: no
2025-12-24T02:11:01.720118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444712:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.722791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240942444702:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:01.722733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567132:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.722756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241239837567132:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.723426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241239837567149:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57625
2025-12-24T02:11:01.723594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241239837567132:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:01.723681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241239837567115:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:01.723697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:01.723705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241239837567115:2110] [node 6] ICP22 created new session: [4:7587241239837567150:2110]
2025-12-24T02:11:01.723711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241239837567150:2110] [node 6] ICS09 handshake done sender: [4:7587241239837567132:2116] self: [4:7587241239837567118:0] peer: [6:7587241240942444708:0] socket: 319 qp: -1
2025-12-24T02:11:01.723718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241239837567150:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:01.723726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.727061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241240942444713:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30048
2025-12-24T02:11:01.727307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241240942444701:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:01.730735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240942444616:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.730765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:01.730775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240942444616:2049] [node 2] ICP22 created new session: [6:7587241240942444714:2049]
2025-12-24T02:11:01.730782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240942444714:2049] [node 2] ICS09 handshake done sender: [6:7587241240942444702:2118] self: [6:7587241240942444704:0] peer: [2:7587241238055494311:0] socket: 276 qp: -1
2025-12-24T02:11:01.730782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240942444655:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:01.730792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240942444714:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:01.730803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:01.730803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:01.730811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240942444655:2073] [node 4] ICP22 created new session: [6:7587241240942444716:2073]
2025-12-24T02:11:01.730817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241240942444716:2073] [node 4] ICS09 handshake done sender: [6:7587241240942444701:2117] self: [6:7587241240942444708:0] peer: [4:7587241239837567118:0] socket: 327 qp: -1
2025-12-24T02:11:01.730824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241240942444716:2073] [node 4] ICS10 traffic start
2025-12-24T02:11:01.730854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:01.730879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241239837567149:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.730937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241239837567115:2110] [node 6] ICP18 handshake (actor [4:7587241239837567149:2122]) is too old
2025-12-24T02:11:01.742597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:01.742690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241238055494324:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-12-24T02:11:01.742713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241238055494236:2048] [node 1] ICP30 unregister session Session# [2:7587241238055494324:2048] VirtualId# [2:7587241238055494312:0]
2025-12-24T02:11:01.742721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241238055494324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.742765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.742790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.743656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:01.743737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241241233741668:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-12-24T02:11:01.743743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241241233741599:2048] [node 1] ICP30 unregister session Session# [3:7587241241233741668:2048] VirtualId# [3:7587241241233741666:0]
2025-12-24T02:11:01.743749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241241233741668:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.743795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.743823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.746597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:01.746670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241238965316167:2048] [node 1] ICS07 socket disconnect 292 reason# EndOfStream
2025-12-24T02:11:01.746676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241238965316167:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.746754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.746778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241238965316167:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:01.746763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:01.746844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241239837567146:2048] [node 1] ICS01 socket: 322 reason# EndOfStream
2025-12-24T02:11:01.746852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241239837567051:2048] [node 1] ICP30 unregister session Session# [4:7587241239837567146:2048] VirtualId# [4:7587241239837567141:0]
2025-12-24T02:11:01.746857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241239837567146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.746911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.746937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.746902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:01.746978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240942444706:2048] [node 1] ICS01 socket: 302 reason# EndOfStream
2025-12-24T02:11:01.746985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240942444615:2048] [node 1] ICP30 unregister session Session# [6:7587241240942444706:2048] VirtualId# [6:7587241240942444699:0]
2025-12-24T02:11:01.746991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240942444706:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.747041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:01.747076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.747611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241238965316081:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.747624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241238965316167:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.747629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238965316081:2048] [node 1] ICP30 unregister session Session# [5:7587241238965316167:2048] VirtualId# [5:7587241238965316151:0]
2025-12-24T02:11:01.747642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:01.747647Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241238965316081:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.747650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5700) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.758795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:01.758890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241241233741699:2049] [node 2] ICS07 socket disconnect 280 reason# EndOfStream
2025-12-24T02:11:01.758898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241241233741699:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.758946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.758964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241241233741699:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:01.762748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241238965316081:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:01.771427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241241233741600:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:01.771443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241241233741699:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:01.771874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241241233741600:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.771884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241241233741699:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.771891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241241233741600:2049] [node 2] ICP30 unregister session Session# [3:7587241241233741699:2049] VirtualId# [3:7587241241233741686:0]
2025-12-24T02:11:01.771907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.771913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241241233741600:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.771915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.782755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:01.782797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:01.782902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241238965316179:2049] [node 2] ICS01 socket: 300 reason# EndOfStream
2025-12-24T02:11:01.782911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238965316082:2049] [node 2] ICP30 unregister session Session# [5:7587241238965316179:2049] VirtualId# [5:7587241238965316173:0]
2025-12-24T02:11:01.782925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241238965316179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.782996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.783024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.783070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:01.783148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240942444714:2049] [node 2] ICS01 socket: 276 reason# EndOfStream
2025-12-24T02:11:01.783155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240942444616:2049] [node 2] ICP30 unregister session Session# [6:7587241240942444714:2049] VirtualId# [6:7587241240942444704:0]
2025-12-24T02:11:01.783160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240942444714:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.783208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.783236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.783424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241239837567137:2049] [node 2] ICS07 socket disconnect 295 reason# EndOfStream
2025-12-24T02:11:01.783435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241239837567137:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.783471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:01.783492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241239837567137:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:01.786665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241239837567052:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.786677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241239837567137:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.786683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241239837567052:2049] [node 2] ICP30 unregister session Session# [4:7587241239837567137:2049] VirtualId# [4:7587241239837567127:0]
2025-12-24T02:11:01.786712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:01.786719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241239837567052:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.786721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.793250Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:01.793351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240942444695:2050] [node 3] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:01.793361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240942444617:2050] [node 3] ICP30 unregister session Session# [6:7587241240942444695:2050] VirtualId# [6:7587241240942444630:0]
2025-12-24T02:11:01.793369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240942444695:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.793430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:01.793456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:01.793493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:01.793556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241238965316176:2050] [node 3] ICS07 socket disconnect 324 reason# EndOfStream
2025-12-24T02:11:01.793560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241238965316176:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.793587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:01.793609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241238965316176:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:01.794471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241238965316083:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61166) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:01.794483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241238965316176:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:01.794488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238965316083:2050] [node 3] ICP30 unregister session Session# [5:7587241238965316176:2050] VirtualId# [5:7587241238965316169:0]
2025-12-24T02:11:01.794500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:01.794506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241238965316083:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.794509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61166) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:01.806743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241238965316083:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:01.826786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:01.826887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241240942444716:2073] [node 4] ICS01 socket: 327 reason# EndOfStream
2025-12-24T02:11:01.826896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240942444655:2073] [node 4] ICP30 unregister session Session# [6:7587241240942444716:2073] VirtualId# [6:7587241240942444708:0]
2025-12-24T02:11:01.826903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241240942444716:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:01.826958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:01.826982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:01.870903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241238741153420:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:01.870912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241238741153421:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:01.869148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238445414125:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.869163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238445414126:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:01.872928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241894140317:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.872936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241894140318:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.886903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241241156342219:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.886921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241241156342220:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.886925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241241156342221:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.897352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240415558265:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.897371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240415558266:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.897376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240415558267:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.888581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238613459472:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:01.888591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238613459473:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:01.888596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238613459474:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:01.925557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241241156342266:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:01.925643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241241156342267:2090] [node 6] ICP01 ready to work
2025-12-24T02:11:01.926576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238445414190:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:01.926672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238445414191:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:01.926758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241238445414192:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:01.926863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241238741153481:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:01.932333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238613459516:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:01.935883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240415558322:2061] [node 5] ICP01 ready to work
2025-12-24T02:11:01.935993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241240415558323:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:01.945351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241241894140317:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20336789ns) {Configure:104(20335062ns) {ProcessConfigured:135(19885191ns) {StartInitialHandshake:157(19884722ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(94ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(1750ns) {SwitchToState:179(581ns)}}}}}
2025-12-24T02:11:01.945557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241894140382:2091] [node 4] ICP01 ready to work
2025-12-24T02:11:01.945646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241894140383:2093] [node 5] ICP01 ready to work
2025-12-24T02:11:01.945717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241241894140384:2095] [node 6] ICP01 ready to work
2025-12-24T02:11:01.954792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241241156342266:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29083896ns) {Configure:104(29083300ns) {ProcessConfigured:135(28843479ns) {StartInitialHandshake:157(28842875ns) {DropHandshakes:470(390ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:477(1759ns) {SwitchToState:179(1297ns)}}}}}
2025-12-24T02:11:01.962806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241238741153421:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35862430ns) {Configure:104(35861335ns) {ProcessConfigured:135(35327589ns) {StartInitialHandshake:157(35326884ns) {DropHandshakes:470(343ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(2187ns) {SwitchToState:179(1019ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.963540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241238741153490:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:01.963687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241238741153492:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:01.964147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241238741153491:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57700
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.964728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241241894140383:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18498499ns) {Configure:104(18498003ns) {ProcessConfigured:135(16843236ns) {StartInitialHandshake:157(16841833ns) {DropHandshakes:470(853ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(240ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(736ns) {SwitchToState:179(516ns)}}}}}
2025-12-24T02:11:01.965298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241894140385:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:01.965562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241894140395:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:01.972677Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241238445414191:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37378697ns) {Configure:104(37376847ns) {ProcessConfigured:135(37058759ns) {StartInitialHandshake:157(37058229ns) {DropHandshakes:470(602ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(1314ns) {SwitchToState:179(53ns)}}}}}
2025-12-24T02:11:01.973473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241241156342219:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18627241ns) {Configure:104(18626205ns) {ProcessConfigured:135(18285045ns) {StartInitialHandshake:157(18284825ns) {DropHandshakes:470(938ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(126ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(923ns) {SwitchToState:179(31ns)}}}}}
2025-12-24T02:11:01.978782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241240415558265:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42317563ns) {Configure:104(42317024ns) {ProcessConfigured:135(41481451ns) {StartInitialHandshake:157(41480917ns) {DropHandshakes:470(212ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(40ns)} GenerateSessionVirtualId:988(441ns) PrepareNewSessionHandshake:477(1640ns) {SwitchToState:179(683ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.982878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238445414200:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10563
2025-12-24T02:11:01.982907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238445414201:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10563
2025-12-24T02:11:01.982935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241238613459547:2088] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.989422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241241894140398:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:01.988949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241238741153500:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57700
2025-12-24T02:11:01.989008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241238741153501:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57700
2025-12-24T02:11:01.989029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241238741153502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57700
2025-12-24T02:11:01.989040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241238741153503:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57700
2025-12-24T02:11:01.987232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241241156342295:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:01.990830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153500:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.992886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238613459548:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:01.992928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238613459549:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:01.991298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140395:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.992940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238613459550:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:01.991407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241241894140317:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.994133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241238445414206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10563
2025-12-24T02:11:01.994366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241241156342296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:01.994393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414200:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.994479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238613459550:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.994503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238613459473:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.994573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414201:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.994793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238613459548:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.994816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238613459474:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:01.995133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140385:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.995185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140389:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:01.995189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241894140389:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.001781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241238613459555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.002543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241894140389:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.003445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241241156342297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:02.003559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241241156342296:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.003587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241241156342221:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.004458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153485:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.004476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241238741153485:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.004524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414196:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.004740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238445414196:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.005193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241238445414125:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.005340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238445414196:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.005417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238445414191:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.005432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.005438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238445414191:2107] [node 5] ICP22 created new session: [2:7587241242740381506:2107]
2025-12-24T02:11:02.005443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242740381506:2107] [node 5] ICS09 handshake done sender: [2:7587241238445414196:2112] self: [2:7587241238445414195:0] peer: [5:7587241238613459553:0] socket: 302 qp: -1
2025-12-24T02:11:02.005450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242740381506:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:02.005456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.004697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238613459548:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.005163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241894140383:2093] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.005186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.005193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241894140383:2093] [node 5] ICP22 created new session: [3:7587241246189107698:2093]
2025-12-24T02:11:02.005202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241246189107698:2093] [node 5] ICS09 handshake done sender: [3:7587241241894140389:2113] self: [3:7587241241894140388:0] peer: [5:7587241238613459554:0] socket: 289 qp: -1
2025-12-24T02:11:02.005209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241246189107698:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:02.005218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.005378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140387:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.005382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241894140387:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.006627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525639:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.006775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238613459474:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.006791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.006796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238613459474:2050] [node 3] ICP22 created new session: [5:7587241242908426853:2050]
2025-12-24T02:11:02.006802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242908426853:2050] [node 3] ICS09 handshake done sender: [5:7587241238613459548:2114] self: [5:7587241238613459554:0] peer: [3:7587241241894140388:0] socket: 301 qp: -1
2025-12-24T02:11:02.006806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242908426853:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.006812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.011038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140398:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.011349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241894140387:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.011437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241894140382:2091] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.011449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.011456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241894140382:2091] [node 4] ICP22 created new session: [3:7587241246189107703:2091]
2025-12-24T02:11:02.011461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241246189107703:2091] [node 4] ICS09 handshake done sender: [3:7587241241894140387:2112] self: [3:7587241241894140386:0] peer: [4:7587241245451309596:0] socket: 288 qp: -1
2025-12-24T02:11:02.011472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241246189107703:2091] [node 4] ICS10 traffic start
2025-12-24T02:11:02.011478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.014410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242740381509:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10563
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.015072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525640:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.015126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525641:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.016508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153491:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.016534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153501:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.016610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153489:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.016613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241238741153489:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.017544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241238741153485:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.017663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241238741153489:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.017852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241238741153421:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.017869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.017874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241238741153421:2049] [node 3] ICP22 created new session: [1:7587241243036120805:2049]
2025-12-24T02:11:02.017879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241243036120805:2049] [node 3] ICS09 handshake done sender: [1:7587241238741153485:2110] self: [1:7587241238741153483:0] peer: [3:7587241241894140399:0] socket: 282 qp: -1
2025-12-24T02:11:02.017884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241243036120805:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.017891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.017984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241238741153420:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.017991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.017996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241238741153420:2048] [node 2] ICP22 created new session: [1:7587241243036120807:2048]
2025-12-24T02:11:02.018000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241243036120807:2048] [node 2] ICS09 handshake done sender: [1:7587241238741153489:2112] self: [1:7587241238741153488:0] peer: [2:7587241242740381505:0] socket: 284 qp: -1
2025-12-24T02:11:02.018003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241243036120807:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:02.018006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.014825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238613459549:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.014924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242908426852:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.015858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241238613459473:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241238613459550:2116] held: no
2025-12-24T02:11:02.015954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238613459550:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.016044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238613459473:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.016054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.016058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238613459473:2049] [node 2] ICP22 created new session: [5:7587241242908426857:2049]
2025-12-24T02:11:02.016064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242908426857:2049] [node 2] ICS09 handshake done sender: [5:7587241238613459550:2116] self: [5:7587241238613459553:0] peer: [2:7587241238445414195:0] socket: 304 qp: -1
2025-12-24T02:11:02.016070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242908426857:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.016075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.018996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241246189107706:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:02.018887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244710525641:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.018914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240415558266:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.018971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525642:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.019074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244710525640:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.019092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240415558267:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.022870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241241894140391:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.022881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241241894140391:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.031050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241241156342295:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.031119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241241156342219:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.031166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241241156342297:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.031244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245451309597:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:02.031328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242740381509:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.031357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414198:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.031360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238445414198:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.031840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242740381510:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10563
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.032086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245451309598:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:02.032103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241241156342220:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.032148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241241156342296:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.032277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414206:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.032318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238445414198:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.032410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414194:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.032417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238445414194:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.032893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238445414192:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.032904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.032910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238445414192:2110] [node 6] ICP22 created new session: [2:7587241242740381514:2110]
2025-12-24T02:11:02.032915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242740381514:2110] [node 6] ICS09 handshake done sender: [2:7587241238445414198:2113] self: [2:7587241238445414197:0] peer: [6:7587241244710525643:0] socket: 291 qp: -1
2025-12-24T02:11:02.032921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242740381514:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:02.032930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.033069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238445414201:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.033167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241241156342221:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.033180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.033183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241241156342221:2050] [node 3] ICP22 created new session: [4:7587241245451309601:2050]
2025-12-24T02:11:02.033188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245451309601:2050] [node 3] ICS09 handshake done sender: [4:7587241241156342296:2116] self: [4:7587241245451309596:0] peer: [3:7587241241894140386:0] socket: 307 qp: -1
2025-12-24T02:11:02.033193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245451309601:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.033198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.034546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241238445414125:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.034559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.034564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238445414125:2048] [node 1] ICP22 created new session: [2:7587241242740381516:2048]
2025-12-24T02:11:02.034569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242740381516:2048] [node 1] ICS09 handshake done sender: [2:7587241238445414201:2115] self: [2:7587241242740381505:0] peer: [1:7587241238741153488:0] socket: 292 qp: -1
2025-12-24T02:11:02.034574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242740381516:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.034581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.039109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153502:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.039135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153503:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.039164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153487:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.039167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241238741153487:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.042764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241894140395:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.043221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525645:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.044236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241238741153487:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.043594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242908426860:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.043615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242908426861:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.046600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241241894140317:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25825) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241241894140395:2116] held: no
2025-12-24T02:11:02.046635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241241894140317:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.046646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.046651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241894140317:2048] [node 1] ICP22 created new session: [3:7587241246189107708:2048]
2025-12-24T02:11:02.046656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241246189107708:2048] [node 1] ICS09 handshake done sender: [3:7587241241894140395:2116] self: [3:7587241241894140399:0] peer: [1:7587241238741153483:0] socket: 286 qp: -1
2025-12-24T02:11:02.046663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241246189107708:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.046672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.044502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238445414194:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.047089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245451309604:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.044588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241238741153481:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.044609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.044616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241238741153481:2107] [node 4] ICP22 created new session: [1:7587241243036120814:2107]
2025-12-24T02:11:02.044623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241243036120814:2107] [node 4] ICS09 handshake done sender: [1:7587241238741153487:2111] self: [1:7587241238741153486:0] peer: [4:7587241245451309599:0] socket: 283 qp: -1
2025-12-24T02:11:02.044629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241243036120814:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:02.044639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.047422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245451309605:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:02.058752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241246189107710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:02.058866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241241894140391:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.062879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238445414190:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.062909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.062919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238445414190:2106] [node 4] ICP22 created new session: [2:7587241242740381520:2106]
2025-12-24T02:11:02.062925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242740381520:2106] [node 4] ICS09 handshake done sender: [2:7587241238445414194:2111] self: [2:7587241238445414193:0] peer: [4:7587241245451309600:0] socket: 300 qp: -1
2025-12-24T02:11:02.062932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242740381520:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:02.062940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.062890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241241894140384:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.062916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.062925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241894140384:2095] [node 6] ICP22 created new session: [3:7587241246189107711:2095]
2025-12-24T02:11:02.062931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241246189107711:2095] [node 6] ICS09 handshake done sender: [3:7587241241894140391:2114] self: [3:7587241241894140390:0] peer: [6:7587241244710525644:0] socket: 279 qp: -1
2025-12-24T02:11:02.062938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241246189107711:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:02.062946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.063461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244710525642:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.063489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244710525639:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.063558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244710525641:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.066997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242908426861:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.067033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238613459472:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.067032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240415558323:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.067127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525646:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.067168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525647:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.067146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153494:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.067158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241238741153494:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.068119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241238741153494:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.068573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241246189107710:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.068602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241241894140318:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.069530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241238445414203:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.069541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241238445414203:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.070505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241238445414203:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.067993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242908426863:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.068279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242908426861:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.068324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238613459472:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.068336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.068342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238613459472:2048] [node 1] ICP22 created new session: [5:7587241242908426864:2048]
2025-12-24T02:11:02.068348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242908426864:2048] [node 1] ICS09 handshake done sender: [5:7587241242908426861:2121] self: [5:7587241242908426862:0] peer: [1:7587241238741153493:0] socket: 340 qp: -1
2025-12-24T02:11:02.070194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241238741153490:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.070211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.070217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241238741153490:2108] [node 5] ICP22 created new session: [1:7587241243036120817:2108]
2025-12-24T02:11:02.070222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241243036120817:2108] [node 5] ICS09 handshake done sender: [1:7587241238741153494:2114] self: [1:7587241238741153493:0] peer: [5:7587241242908426862:0] socket: 295 qp: -1
2025-12-24T02:11:02.070227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241243036120817:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.070234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.070586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241238445414126:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.070600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.070605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241238445414126:2049] [node 3] ICP22 created new session: [2:7587241242740381523:2049]
2025-12-24T02:11:02.070610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242740381523:2049] [node 3] ICS09 handshake done sender: [2:7587241238445414203:2116] self: [2:7587241238445414202:0] peer: [3:7587241246189107713:0] socket: 293 qp: -1
2025-12-24T02:11:02.070615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242740381523:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.070621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.068354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242908426864:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.068361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.072071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244710525646:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.072089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244710525640:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.072167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240415558267:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3083) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241244710525640:2116] held: no
2025-12-24T02:11:02.072189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240415558267:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.072198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.072203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240415558267:2050] [node 3] ICP22 created new session: [6:7587241244710525649:2050]
2025-12-24T02:11:02.072207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244710525649:2050] [node 3] ICS09 handshake done sender: [6:7587241244710525640:2116] self: [6:7587241244710525644:0] peer: [3:7587241241894140390:0] socket: 324 qp: -1
2025-12-24T02:11:02.072213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244710525649:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.072220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.072331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240415558266:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.072338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.072342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240415558266:2049] [node 2] ICP22 created new session: [6:7587241244710525651:2049]
2025-12-24T02:11:02.072347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244710525651:2049] [node 2] ICS09 handshake done sender: [6:7587241244710525641:2117] self: [6:7587241244710525643:0] peer: [2:7587241238445414197:0] socket: 325 qp: -1
2025-12-24T02:11:02.072352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244710525651:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.072356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.072488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240415558322:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.079082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241240415558265:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.079535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241241894140318:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241246189107710:2119] held: no
2025-12-24T02:11:02.079604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241246189107714:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2828
2025-12-24T02:11:02.080611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241241156342291:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.080625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241241156342291:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.081419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245451309606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32328
2025-12-24T02:11:02.081544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241246189107710:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.081677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245451309598:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.081694Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241241156342220:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241241156342297:2117] held: no
2025-12-24T02:11:02.081715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241241156342219:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25825) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241241156342295:2115] held: no
2025-12-24T02:11:02.081775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241241156342295:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.083634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241238741153496:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.083644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241238741153496:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.084341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241238741153496:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.084502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525657:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.084530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525658:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.084541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244710525659:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34077
2025-12-24T02:11:02.084422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241238741153492:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.084440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.084447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241238741153492:2109] [node 6] ICP22 created new session: [1:7587241243036120820:2109]
2025-12-24T02:11:02.084458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241243036120820:2109] [node 6] ICS09 handshake done sender: [1:7587241238741153496:2115] self: [1:7587241238741153495:0] peer: [6:7587241244710525655:0] socket: 336 qp: -1
2025-12-24T02:11:02.084464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241243036120820:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:02.084476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.082787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241238613459552:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.082800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241238613459552:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.083385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242908426860:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.083497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241238613459516:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.084699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244710525646:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.084799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241241894140318:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.084813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.084824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241241894140318:2049] [node 2] ICP22 created new session: [3:7587241246189107715:2049]
2025-12-24T02:11:02.084831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241246189107715:2049] [node 2] ICS09 handshake done sender: [3:7587241246189107710:2119] self: [3:7587241246189107713:0] peer: [2:7587241238445414202:0] socket: 343 qp: -1
2025-12-24T02:11:02.084837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241246189107715:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.084845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.084758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241238613459552:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.084831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241238613459547:2088] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.084843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.084847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238613459547:2088] [node 6] ICP22 created new session: [5:7587241242908426868:2088]
2025-12-24T02:11:02.084851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242908426868:2088] [node 6] ICS09 handshake done sender: [5:7587241238613459552:2117] self: [5:7587241238613459551:0] peer: [6:7587241244710525653:0] socket: 326 qp: -1
2025-12-24T02:11:02.084854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242908426868:2088] [node 6] ICS10 traffic start
2025-12-24T02:11:02.084860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.085176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240415558265:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.085189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.085194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240415558265:2048] [node 1] ICP22 created new session: [6:7587241244710525660:2048]
2025-12-24T02:11:02.085199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244710525660:2048] [node 1] ICS09 handshake done sender: [6:7587241244710525646:2120] self: [6:7587241244710525655:0] peer: [1:7587241238741153495:0] socket: 338 qp: -1
2025-12-24T02:11:02.085205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244710525660:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.085211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.089409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244710525642:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.089500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240415558322:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9678) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241244710525642:2118] held: no
2025-12-24T02:11:02.089517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240415558322:2061] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.089527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.089532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240415558322:2061] [node 5] ICP22 created new session: [6:7587241244710525662:2061]
2025-12-24T02:11:02.089538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244710525662:2061] [node 5] ICS09 handshake done sender: [6:7587241244710525642:2118] self: [6:7587241244710525653:0] peer: [5:7587241238613459551:0] socket: 327 qp: -1
2025-12-24T02:11:02.089544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244710525662:2061] [node 5] ICS10 traffic start
2025-12-24T02:11:02.089550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.090841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241241156342291:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.091087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244710525639:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.091166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241240415558323:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.091193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.091199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241240415558323:2070] [node 4] ICP22 created new session: [6:7587241244710525664:2070]
2025-12-24T02:11:02.091204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244710525664:2070] [node 4] ICS09 handshake done sender: [6:7587241244710525639:2115] self: [6:7587241244710525648:0] peer: [4:7587241241156342290:0] socket: 320 qp: -1
2025-12-24T02:11:02.091210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244710525664:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:02.091219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.093874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241241156342297:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.094007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241241156342287:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.094016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241241156342287:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.094854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245451309606:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.094861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241241156342219:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.094872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.094878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241241156342219:2048] [node 1] ICP22 created new session: [4:7587241245451309608:2048]
2025-12-24T02:11:02.094883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245451309608:2048] [node 1] ICS09 handshake done sender: [4:7587241241156342295:2115] self: [4:7587241245451309599:0] peer: [1:7587241238741153486:0] socket: 306 qp: -1
2025-12-24T02:11:02.094889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245451309608:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.094895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.094992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241241156342267:2090] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.094997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.095001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241241156342267:2090] [node 6] ICP22 created new session: [4:7587241245451309610:2090]
2025-12-24T02:11:02.095005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245451309610:2090] [node 6] ICS09 handshake done sender: [4:7587241241156342291:2113] self: [4:7587241241156342290:0] peer: [6:7587241244710525648:0] socket: 311 qp: -1
2025-12-24T02:11:02.095008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245451309610:2090] [node 6] ICS10 traffic start
2025-12-24T02:11:02.095014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.095217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241241156342220:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.095223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.095227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241241156342220:2049] [node 2] ICP22 created new session: [4:7587241245451309613:2049]
2025-12-24T02:11:02.095231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245451309613:2049] [node 2] ICS09 handshake done sender: [4:7587241241156342297:2117] self: [4:7587241245451309600:0] peer: [2:7587241238445414193:0] socket: 308 qp: -1
2025-12-24T02:11:02.095235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245451309613:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.095239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.095469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241241156342267:2090] [node 6] ICP18 handshake (actor [4:7587241245451309606:2122]) is too old
2025-12-24T02:11:02.095950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241241156342287:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.096011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241241156342266:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.096020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.096025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241241156342266:2086] [node 5] ICP22 created new session: [4:7587241245451309616:2086]
2025-12-24T02:11:02.096030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245451309616:2086] [node 5] ICS09 handshake done sender: [4:7587241241156342287:2111] self: [4:7587241241156342268:0] peer: [5:7587241242908426867:0] socket: 334 qp: -1
2025-12-24T02:11:02.096034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245451309616:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:02.096040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.097706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241238613459516:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241242908426860:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.097768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242908426870:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52773
2025-12-24T02:11:02.097898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242908426860:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.097942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241238613459516:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.097949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.097953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241238613459516:2084] [node 4] ICP22 created new session: [5:7587241242908426871:2084]
2025-12-24T02:11:02.097959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242908426871:2084] [node 4] ICS09 handshake done sender: [5:7587241242908426860:2120] self: [5:7587241242908426867:0] peer: [4:7587241241156342268:0] socket: 339 qp: -1
2025-12-24T02:11:02.097964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242908426871:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:02.097970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.100826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:02.100898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241242740381516:2048] [node 1] ICS01 socket: 292 reason# EndOfStream
2025-12-24T02:11:02.100904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241238445414125:2048] [node 1] ICP30 unregister session Session# [2:7587241242740381516:2048] VirtualId# [2:7587241242740381505:0]
2025-12-24T02:11:02.100911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241242740381516:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.100940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.100958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.102099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:02.102172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241246189107708:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-12-24T02:11:02.102177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241241894140317:2048] [node 1] ICP30 unregister session Session# [3:7587241246189107708:2048] VirtualId# [3:7587241241894140399:0]
2025-12-24T02:11:02.102183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241246189107708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.102229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.102255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.103847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:02.103913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241244710525660:2048] [node 1] ICS07 socket disconnect 338 reason# EndOfStream
2025-12-24T02:11:02.103918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244710525660:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.103953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.103970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241244710525660:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:02.104284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241240415558265:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25825) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:02.104294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244710525660:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:02.104297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240415558265:2048] [node 1] ICP30 unregister session Session# [6:7587241244710525660:2048] VirtualId# [6:7587241244710525655:0]
2025-12-24T02:11:02.104310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.104315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241240415558265:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25825) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:02.104317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25825) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:02.104066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:02.104143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241245451309608:2048] [node 1] ICS01 socket: 306 reason# EndOfStream
2025-12-24T02:11:02.104149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241241156342219:2048] [node 1] ICP30 unregister session Session# [4:7587241245451309608:2048] VirtualId# [4:7587241245451309599:0]
2025-12-24T02:11:02.104154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241245451309608:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.104197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.104219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.104245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:02.104309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242908426864:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-12-24T02:11:02.104315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238613459472:2048] [node 1] ICP30 unregister session Session# [5:7587241242908426864:2048] VirtualId# [5:7587241242908426862:0]
2025-12-24T02:11:02.104319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242908426864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.104347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.104366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.114640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:02.114759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241240415558265:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:02.114771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241246189107715:2049] [node 2] ICS01 socket: 343 reason# EndOfStream
2025-12-24T02:11:02.114782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241241894140318:2049] [node 2] ICP30 unregister session Session# [3:7587241246189107715:2049] VirtualId# [3:7587241246189107713:0]
2025-12-24T02:11:02.114789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241246189107715:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.114846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.114872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.122784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:02.122785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:02.122872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241245451309613:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-12-24T02:11:02.122880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241241156342220:2049] [node 2] ICP30 unregister session Session# [4:7587241245451309613:2049] VirtualId# [4:7587241245451309600:0]
2025-12-24T02:11:02.122888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241245451309613:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.122945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.122967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.123001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:02.123068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242908426857:2049] [node 2] ICS01 socket: 304 reason# EndOfStream
2025-12-24T02:11:02.123072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238613459473:2049] [node 2] ICP30 unregister session Session# [5:7587241242908426857:2049] VirtualId# [5:7587241238613459553:0]
2025-12-24T02:11:02.123076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242908426857:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.123111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.123131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.123296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244710525651:2049] [node 2] ICS01 socket: 325 reason# EndOfStream
2025-12-24T02:11:02.123307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240415558266:2049] [node 2] ICP30 unregister session Session# [6:7587241244710525651:2049] VirtualId# [6:7587241244710525643:0]
2025-12-24T02:11:02.123311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244710525651:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.123336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.123355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.129718Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:02.129776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244710525649:2050] [node 3] ICS01 socket: 324 reason# EndOfStream
2025-12-24T02:11:02.129780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240415558267:2050] [node 3] ICP30 unregister session Session# [6:7587241244710525649:2050] VirtualId# [6:7587241244710525644:0]
2025-12-24T02:11:02.129785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244710525649:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.129808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.129826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.129346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:02.129455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241245451309601:2050] [node 3] ICS01 socket: 307 reason# EndOfStream
2025-12-24T02:11:02.129464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241241156342221:2050] [node 3] ICP30 unregister session Session# [4:7587241245451309601:2050] VirtualId# [4:7587241245451309596:0]
2025-12-24T02:11:02.129471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241245451309601:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.129522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.129546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.129578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:02.129641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242908426853:2050] [node 3] ICS01 socket: 301 reason# EndOfStream
2025-12-24T02:11:02.129646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241238613459474:2050] [node 3] ICP30 unregister session Session# [5:7587241242908426853:2050] VirtualId# [5:7587241238613459554:0]
2025-12-24T02:11:02.129650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242908426853:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.129674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.129691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.140171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:02.140237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244710525664:2070] [node 4] ICS01 socket: 320 reason# EndOfStream
2025-12-24T02:11:02.140242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240415558323:2070] [node 4] ICP30 unregister session Session# [6:7587241244710525664:2070] VirtualId# [6:7587241244710525648:0]
2025-12-24T02:11:02.140245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244710525664:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.140276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:02.140293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:02.148361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:02.148438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244710525662:2061] [node 5] ICS01 socket: 327 reason# EndOfStream
2025-12-24T02:11:02.148445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241240415558322:2061] [node 5] ICP30 unregister session Session# [6:7587241244710525662:2061] VirtualId# [6:7587241244710525653:0]
2025-12-24T02:11:02.148450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244710525662:2061] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.148487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:02.148506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:02.163824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241246424491251:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.163842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241246424491252:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.166269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241242378385823:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.166282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241242378385824:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.174174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241242538988519:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:02.174197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241242538988520:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.167884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241242959948741:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.167892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241242959948742:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.167896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241242959948743:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.171148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241244429948286:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.171156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241244429948287:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.171162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241244429948288:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.169424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241242847768771:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.169430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241242847768772:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.169435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241242847768773:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.201427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491311:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
2025-12-24T02:11:02.202906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241244429948348:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.203547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385883:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.208253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491312:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.208805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385884:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.211164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768841:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.228178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241242378385891:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:02.228268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241242378385892:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:02.228666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241242959948805:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:02.231591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241246424491321:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:02.231690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241246424491322:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:02.233484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241242378385889:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:02.234002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241242959948806:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:02.240058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241242959948741:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16150965ns) {Configure:104(16149845ns) {ProcessConfigured:135(15776559ns) {StartInitialHandshake:157(15776372ns) {DropHandshakes:470(472ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(657ns) {SwitchToState:179(293ns)}}}}}
2025-12-24T02:11:02.240025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241246424491323:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.243720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385896:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.244808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241242847768844:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:02.244915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241242847768845:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:02.245372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768846:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.245386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768847:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.247799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241242538988588:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.251295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768841:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.252209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385883:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.252519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491312:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.252676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385906:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
2025-12-24T02:11:02.250801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241242538988520:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16308122ns) {Configure:104(16307295ns) {ProcessConfigured:135(15967297ns) {StartInitialHandshake:157(15966502ns) {DropHandshakes:470(512ns) {DropIncomingHandshake:442(132ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(1076ns) {SwitchToState:179(738ns)}}}}}
2025-12-24T02:11:02.250976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241242538988590:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:02.251067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241242538988591:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:02.251222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241242538988585:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20827
2025-12-24T02:11:02.251982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241242538988592:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20827
2025-12-24T02:11:02.252014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241242538988593:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20827
2025-12-24T02:11:02.252059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241242538988594:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20827
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.255073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385884:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.255105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241242378385824:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.255149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491333:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
2025-12-24T02:11:02.255252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491331:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
2025-12-24T02:11:02.255265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491332:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.256505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948361:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.257362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988593:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.257399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988585:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.259626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491314:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.259640Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241246424491314:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.259822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491311:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.259879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241246424491251:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.260111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491333:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.259124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241244429948362:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:02.260025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244429948361:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.260043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241244429948287:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.263534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768846:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.263578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241242847768844:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.267101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491328:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.267115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241246424491328:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.267008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385896:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.269318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988594:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.269353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988580:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.269356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241242538988580:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.270014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385906:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.270040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241242378385823:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.270148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241242538988580:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.270240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988587:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.270246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241242538988587:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.272052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241242538988519:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.272090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.272099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241242538988519:2048] [node 2] ICP22 created new session: [1:7587241242538988605:2048]
2025-12-24T02:11:02.272105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241242538988605:2048] [node 2] ICS09 handshake done sender: [1:7587241242538988580:2107] self: [1:7587241242538988579:0] peer: [2:7587241246424491336:0] socket: 282 qp: -1
2025-12-24T02:11:02.272113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241242538988605:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:02.272121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.272740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
2025-12-24T02:11:02.272837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241242538988587:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.272979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241242378385906:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.273026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241242378385823:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.273038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.273042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241242378385823:2048] [node 1] ICP22 created new session: [3:7587241242378385911:2048]
2025-12-24T02:11:02.273048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241242378385911:2048] [node 1] ICS09 handshake done sender: [3:7587241242378385906:2118] self: [3:7587241242378385909:0] peer: [1:7587241242538988586:0] socket: 318 qp: -1
2025-12-24T02:11:02.273052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241242378385911:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.273069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.272751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948365:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.273599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241242538988520:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.273613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.273617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241242538988520:2049] [node 3] ICP22 created new session: [1:7587241242538988609:2049]
2025-12-24T02:11:02.273622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241242538988609:2049] [node 3] ICS09 handshake done sender: [1:7587241242538988587:2112] self: [1:7587241242538988586:0] peer: [3:7587241242378385909:0] socket: 316 qp: -1
2025-12-24T02:11:02.273627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241242538988609:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.273632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.274850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948819:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.276442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768854:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.276774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241242959948812:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.276790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241242959948812:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.278155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241246424491251:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241246424491311:2107] held: no
2025-12-24T02:11:02.278256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246424491338:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.278495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768854:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.278530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768855:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.279107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241242847768771:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.279125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242847768846:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.279246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241242847768844:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.279262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.279266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241242847768844:2112] [node 4] ICP22 created new session: [5:7587241242847768857:2112]
2025-12-24T02:11:02.279271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242847768857:2112] [node 4] ICS09 handshake done sender: [5:7587241242847768846:2114] self: [5:7587241242847768853:0] peer: [4:7587241242959948811:0] socket: 308 qp: -1
2025-12-24T02:11:02.279275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242847768857:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:02.279282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.278391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241242959948812:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.278785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241242959948805:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.278794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.278799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241242959948805:2108] [node 5] ICP22 created new session: [4:7587241242959948823:2108]
2025-12-24T02:11:02.278804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241242959948823:2108] [node 5] ICS09 handshake done sender: [4:7587241242959948812:2112] self: [4:7587241242959948811:0] peer: [5:7587241242847768853:0] socket: 299 qp: -1
2025-12-24T02:11:02.278810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241242959948823:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.278827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.278946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.277504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241242538988588:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29637417ns) {Configure:104(29636379ns) {ProcessConfigured:135(29246728ns) {StartInitialHandshake:157(29245499ns) {DropHandshakes:470(418ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(115ns) PrepareNewSessionHandshake:477(649ns) {SwitchToState:179(421ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.281460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385914:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
2025-12-24T02:11:02.281576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988596:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.281582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241242538988596:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.281310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948367:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.281365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948368:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.282669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241242538988596:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.282897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768860:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.283147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241246424491314:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.282980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241242538988590:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.283000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.283007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241242538988590:2109] [node 5] ICP22 created new session: [1:7587241242538988614:2109]
2025-12-24T02:11:02.283013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241242538988614:2109] [node 5] ICS09 handshake done sender: [1:7587241242538988596:2116] self: [1:7587241242538988595:0] peer: [5:7587241242847768856:0] socket: 317 qp: -1
2025-12-24T02:11:02.283020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241242538988614:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:02.283039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.285777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244429948367:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.285816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241244429948362:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.285888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244429948365:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.285933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241244429948286:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.285648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241242847768771:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241242847768854:2118] held: no
2025-12-24T02:11:02.285684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242847768854:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.288418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948371:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.286830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241242959948814:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.286851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241242959948814:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.288280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241242378385884:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.288515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241242378385824:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.288526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.288532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241242378385824:2049] [node 2] ICP22 created new session: [3:7587241242378385915:2049]
2025-12-24T02:11:02.288537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241242378385915:2049] [node 2] ICS09 handshake done sender: [3:7587241242378385884:2108] self: [3:7587241242378385907:0] peer: [2:7587241246424491313:0] socket: 296 qp: -1
2025-12-24T02:11:02.288542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241242378385915:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.288548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.288950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988598:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.288960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241242538988598:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.289521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241242538988598:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.289631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241242538988591:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.289643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.289648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241242538988591:2110] [node 6] ICP22 created new session: [1:7587241242538988617:2110]
2025-12-24T02:11:02.289652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241242538988617:2110] [node 6] ICS09 handshake done sender: [1:7587241242538988598:2117] self: [1:7587241242538988597:0] peer: [6:7587241244429948370:0] socket: 325 qp: -1
2025-12-24T02:11:02.289656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241242538988617:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:02.289664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.287825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241246424491252:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.287849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.287858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241246424491252:2049] [node 3] ICP22 created new session: [2:7587241246424491341:2049]
2025-12-24T02:11:02.287863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246424491341:2049] [node 3] ICS09 handshake done sender: [2:7587241246424491314:2109] self: [2:7587241246424491313:0] peer: [3:7587241242378385907:0] socket: 295 qp: -1
2025-12-24T02:11:02.287871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246424491341:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.287881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.288174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491331:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.288231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241246424491311:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.288866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241242847768771:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.288880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.288885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241242847768771:2048] [node 1] ICP22 created new session: [5:7587241242847768861:2048]
2025-12-24T02:11:02.288889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242847768861:2048] [node 1] ICS09 handshake done sender: [5:7587241242847768854:2118] self: [5:7587241242847768856:0] peer: [1:7587241242538988595:0] socket: 320 qp: -1
2025-12-24T02:11:02.288892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242847768861:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.288897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.291024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241242959948822:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.291085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241242959948741:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.291890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768851:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.291904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241242847768851:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.291418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241246424491328:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.292081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241242959948814:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.289093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244429948368:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.289126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241244429948348:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.291398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948373:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.295274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241242538988611:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.295284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241242538988611:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.296996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241242538988611:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.295015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948374:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.295037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244429948361:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.296139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244429948365:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.296201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241244429948287:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.296214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.296221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241244429948287:2049] [node 2] ICP22 created new session: [6:7587241244429948376:2049]
2025-12-24T02:11:02.296227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244429948376:2049] [node 2] ICS09 handshake done sender: [6:7587241244429948361:2114] self: [6:7587241244429948364:0] peer: [2:7587241246424491325:0] socket: 323 qp: -1
2025-12-24T02:11:02.296235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244429948376:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.296242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.296344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241244429948286:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.296351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.296354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241244429948286:2048] [node 1] ICP22 created new session: [6:7587241244429948378:2048]
2025-12-24T02:11:02.296357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244429948378:2048] [node 1] ICS09 handshake done sender: [6:7587241244429948365:2115] self: [6:7587241244429948370:0] peer: [1:7587241242538988597:0] socket: 326 qp: -1
2025-12-24T02:11:02.296361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244429948378:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.296366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.296491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948375:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.299155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241246424491251:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.299174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.299192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241246424491251:2048] [node 1] ICP22 created new session: [2:7587241246424491345:2048]
2025-12-24T02:11:02.299198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246424491345:2048] [node 1] ICS09 handshake done sender: [2:7587241246424491311:2107] self: [2:7587241246424491336:0] peer: [1:7587241242538988579:0] socket: 283 qp: -1
2025-12-24T02:11:02.299205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246424491345:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.299211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.300092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242847768851:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.300225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241242847768845:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.300240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.300246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241242847768845:2113] [node 6] ICP22 created new session: [5:7587241242847768864:2113]
2025-12-24T02:11:02.300252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242847768864:2113] [node 6] ICS09 handshake done sender: [5:7587241242847768851:2117] self: [5:7587241242847768850:0] peer: [6:7587241244429948372:0] socket: 334 qp: -1
2025-12-24T02:11:02.300256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242847768864:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:02.300262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.302859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241246424491322:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.302880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.302886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241246424491322:2112] [node 6] ICP22 created new session: [2:7587241246424491348:2112]
2025-12-24T02:11:02.302892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246424491348:2112] [node 6] ICS09 handshake done sender: [2:7587241246424491328:2115] self: [2:7587241246424491325:0] peer: [6:7587241244429948364:0] socket: 322 qp: -1
2025-12-24T02:11:02.302898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246424491348:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:02.302905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.302769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241242959948806:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.302790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.302800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241242959948806:2109] [node 6] ICP22 created new session: [4:7587241242959948827:2109]
2025-12-24T02:11:02.302806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241242959948827:2109] [node 6] ICS09 handshake done sender: [4:7587241242959948814:2113] self: [4:7587241242959948813:0] peer: [6:7587241244429948369:0] socket: 332 qp: -1
2025-12-24T02:11:02.302814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241242959948827:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:02.302824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.303275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948830:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
2025-12-24T02:11:02.303292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948831:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
2025-12-24T02:11:02.304970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768847:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.305017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241242847768773:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.305072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241242959948819:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.305100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241242959948822:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.305181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385898:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.305192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241242378385898:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.306971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241242538988588:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.306989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.306996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241242538988588:2108] [node 4] ICP22 created new session: [1:7587241242538988620:2108]
2025-12-24T02:11:02.307003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241242538988620:2108] [node 4] ICS09 handshake done sender: [1:7587241242538988611:2118] self: [1:7587241242538988589:0] peer: [4:7587241242959948825:0] socket: 329 qp: -1
2025-12-24T02:11:02.307009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241242538988620:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:02.307016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.307285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241242538988622:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20827
2025-12-24T02:11:02.307381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768867:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.307597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241244429948348:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241244429948368:2118] held: no
2025-12-24T02:11:02.316957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241242847768867:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.317019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241242847768772:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.319238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241242959948741:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.319263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.319273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241242959948741:2048] [node 1] ICP22 created new session: [4:7587241242959948832:2048]
2025-12-24T02:11:02.319280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241242959948832:2048] [node 1] ICS09 handshake done sender: [4:7587241242959948822:2116] self: [4:7587241242959948825:0] peer: [1:7587241242538988589:0] socket: 333 qp: -1
2025-12-24T02:11:02.319289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241242959948832:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.319298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.319549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241242959948743:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.319898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241242378385898:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.320050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241242378385891:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.320066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.320071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241242378385891:2110] [node 5] ICP22 created new session: [3:7587241242378385919:2110]
2025-12-24T02:11:02.320076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241242378385919:2110] [node 5] ICS09 handshake done sender: [3:7587241242378385898:2115] self: [3:7587241242378385897:0] peer: [5:7587241242847768866:0] socket: 305 qp: -1
2025-12-24T02:11:02.320081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241242378385919:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:02.320099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.321332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241242847768773:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8597) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241242847768847:2115] held: no
2025-12-24T02:11:02.324170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491326:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.324188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241246424491326:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.324885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385901:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.324899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241242378385901:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.325764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241242378385901:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.325856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241242378385889:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.325877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.325886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241242378385889:2109] [node 4] ICP22 created new session: [3:7587241242378385922:2109]
2025-12-24T02:11:02.325893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241242378385922:2109] [node 4] ICS09 handshake done sender: [3:7587241242378385901:2116] self: [3:7587241242378385900:0] peer: [4:7587241242959948834:0] socket: 303 qp: -1
2025-12-24T02:11:02.325899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241242378385922:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:02.325907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.326842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241242959948830:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.326911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241242959948742:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.327097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768869:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.327271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241246424491326:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.327337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242847768847:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.327434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241246424491330:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.327447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241246424491330:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.327482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241246424491321:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.327498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.327504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241246424491321:2111] [node 5] ICP22 created new session: [2:7587241246424491351:2111]
2025-12-24T02:11:02.327510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246424491351:2111] [node 5] ICS09 handshake done sender: [2:7587241246424491326:2114] self: [2:7587241246424491324:0] peer: [5:7587241242847768868:0] socket: 344 qp: -1
2025-12-24T02:11:02.327515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246424491351:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:02.327547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.328432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241246424491330:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.326935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948380:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.326972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244429948367:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.328815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241246424491323:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.328831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.328836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241246424491323:2110] [node 4] ICP22 created new session: [2:7587241246424491354:2110]
2025-12-24T02:11:02.328841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246424491354:2110] [node 4] ICS09 handshake done sender: [2:7587241246424491330:2116] self: [2:7587241246424491329:0] peer: [4:7587241242959948835:0] socket: 345 qp: -1
2025-12-24T02:11:02.328846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246424491354:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:02.328852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.332150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241242538988588:2108] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.332284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241242378385924:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38177
2025-12-24T02:11:02.332380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385904:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.332386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241242378385904:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.333022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241242378385904:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.333124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241242378385892:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.333141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.333148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241242378385892:2111] [node 6] ICP22 created new session: [3:7587241242378385926:2111]
2025-12-24T02:11:02.333154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241242378385926:2111] [node 6] ICS09 handshake done sender: [3:7587241242378385904:2117] self: [3:7587241242378385894:0] peer: [6:7587241244429948385:0] socket: 341 qp: -1
2025-12-24T02:11:02.333160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241242378385926:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:02.333168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.330807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241242847768773:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.330829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.330835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241242847768773:2050] [node 3] ICP22 created new session: [5:7587241242847768871:2050]
2025-12-24T02:11:02.330841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242847768871:2050] [node 3] ICS09 handshake done sender: [5:7587241242847768847:2115] self: [5:7587241242847768866:0] peer: [3:7587241242378385897:0] socket: 309 qp: -1
2025-12-24T02:11:02.330847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242847768871:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.330854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.330966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241242847768772:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241242847768867:2121] held: no
2025-12-24T02:11:02.331058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241242847768870:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8461
2025-12-24T02:11:02.331171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241242959948743:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8597) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241242959948819:2115] held: no
2025-12-24T02:11:02.331288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948836:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.331457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241244429948374:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.331493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244429948368:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.331664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241244429948362:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.334581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241242378385924:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.331674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.334613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241242378385892:2111] [node 6] ICP18 handshake (actor [3:7587241242378385924:2121]) is too old
2025-12-24T02:11:02.335691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241242847768867:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.331681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241244429948362:2113] [node 4] ICP22 created new session: [6:7587241244429948383:2113]
2025-12-24T02:11:02.331685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244429948383:2113] [node 4] ICS09 handshake done sender: [6:7587241244429948367:2117] self: [6:7587241244429948369:0] peer: [4:7587241242959948813:0] socket: 335 qp: -1
2025-12-24T02:11:02.331690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244429948383:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:02.331696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.331841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241244429948288:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.332319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241244429948348:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.332326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.332331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241244429948348:2108] [node 5] ICP22 created new session: [6:7587241244429948386:2108]
2025-12-24T02:11:02.332336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244429948386:2108] [node 5] ICS09 handshake done sender: [6:7587241244429948368:2118] self: [6:7587241244429948372:0] peer: [5:7587241242847768850:0] socket: 336 qp: -1
2025-12-24T02:11:02.332340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244429948386:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.332346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.333768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241244429948388:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59920
2025-12-24T02:11:02.333941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241244429948374:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.334927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241244429948288:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.334936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.334943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241244429948288:2050] [node 3] ICP22 created new session: [6:7587241244429948389:2050]
2025-12-24T02:11:02.334948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241244429948389:2050] [node 3] ICS09 handshake done sender: [6:7587241244429948374:2121] self: [6:7587241244429948385:0] peer: [3:7587241242378385894:0] socket: 343 qp: -1
2025-12-24T02:11:02.334954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241244429948389:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.334961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.335745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241244429948288:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:02.338838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241242847768772:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.338859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.338868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241242847768772:2049] [node 2] ICP22 created new session: [5:7587241242847768873:2049]
2025-12-24T02:11:02.338874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241242847768873:2049] [node 2] ICS09 handshake done sender: [5:7587241242847768867:2121] self: [5:7587241242847768868:0] peer: [2:7587241246424491324:0] socket: 349 qp: -1
2025-12-24T02:11:02.338881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241242847768873:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.338890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.342582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241242959948742:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16381) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241242959948830:2117] held: no
2025-12-24T02:11:02.342640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948838:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.342905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241242959948837:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13352
2025-12-24T02:11:02.342957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241242959948819:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.343047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241242959948743:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.343067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.343074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241242959948743:2050] [node 3] ICP22 created new session: [4:7587241242959948839:2050]
2025-12-24T02:11:02.343076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241242959948830:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.343082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241242959948839:2050] [node 3] ICS09 handshake done sender: [4:7587241242959948819:2115] self: [4:7587241242959948834:0] peer: [3:7587241242378385900:0] socket: 313 qp: -1
2025-12-24T02:11:02.343089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241242959948839:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.343094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.343467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241242959948742:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.343480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.343486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241242959948742:2049] [node 2] ICP22 created new session: [4:7587241242959948841:2049]
2025-12-24T02:11:02.343492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241242959948841:2049] [node 2] ICS09 handshake done sender: [4:7587241242959948830:2117] self: [4:7587241242959948835:0] peer: [2:7587241246424491329:0] socket: 346 qp: -1
2025-12-24T02:11:02.343499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241242959948841:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.343505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.349996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:02.350083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241246424491345:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:11:02.350092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241246424491251:2048] [node 1] ICP30 unregister session Session# [2:7587241246424491345:2048] VirtualId# [2:7587241246424491336:0]
2025-12-24T02:11:02.350101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241246424491345:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.350140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.350163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.351054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:02.351128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241242378385911:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:02.351134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241242378385823:2048] [node 1] ICP30 unregister session Session# [3:7587241242378385911:2048] VirtualId# [3:7587241242378385909:0]
2025-12-24T02:11:02.351141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241242378385911:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.351176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.351208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.358775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:02.358874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241242959948832:2048] [node 1] ICS01 socket: 333 reason# EndOfStream
2025-12-24T02:11:02.358883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241242959948741:2048] [node 1] ICP30 unregister session Session# [4:7587241242959948832:2048] VirtualId# [4:7587241242959948825:0]
2025-12-24T02:11:02.358890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241242959948832:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.358945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.358982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.359107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:02.359173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244429948378:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-12-24T02:11:02.359178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241244429948286:2048] [node 1] ICP30 unregister session Session# [6:7587241244429948378:2048] VirtualId# [6:7587241244429948370:0]
2025-12-24T02:11:02.359197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244429948378:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.359225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.359245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.362674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:02.362790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242847768861:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-12-24T02:11:02.362799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241242847768771:2048] [node 1] ICP30 unregister session Session# [5:7587241242847768861:2048] VirtualId# [5:7587241242847768856:0]
2025-12-24T02:11:02.362806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242847768861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.362872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.362905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.371280Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:02.371365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241242378385915:2049] [node 2] ICS01 socket: 296 reason# EndOfStream
2025-12-24T02:11:02.371374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241242378385824:2049] [node 2] ICP30 unregister session Session# [3:7587241242378385915:2049] VirtualId# [3:7587241242378385907:0]
2025-12-24T02:11:02.371380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241242378385915:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.371417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.371438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.382916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:02.383007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241242959948841:2049] [node 2] ICS01 socket: 346 reason# EndOfStream
2025-12-24T02:11:02.383015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241242959948742:2049] [node 2] ICP30 unregister session Session# [4:7587241242959948841:2049] VirtualId# [4:7587241242959948835:0]
2025-12-24T02:11:02.383020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241242959948841:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.383074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.383103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.385054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:02.385112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242847768873:2049] [node 2] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:02.385116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241242847768772:2049] [node 2] ICP30 unregister session Session# [5:7587241242847768873:2049] VirtualId# [5:7587241242847768868:0]
2025-12-24T02:11:02.385120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242847768873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.385150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.385174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.384860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:02.384937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244429948376:2049] [node 2] ICS01 socket: 323 reason# EndOfStream
2025-12-24T02:11:02.384944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241244429948287:2049] [node 2] ICP30 unregister session Session# [6:7587241244429948376:2049] VirtualId# [6:7587241244429948364:0]
2025-12-24T02:11:02.384950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244429948376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.384995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.385020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.394795Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:02.394887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241242847768871:2050] [node 3] ICS01 socket: 309 reason# EndOfStream
2025-12-24T02:11:02.394895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241242847768773:2050] [node 3] ICP30 unregister session Session# [5:7587241242847768871:2050] VirtualId# [5:7587241242847768866:0]
2025-12-24T02:11:02.394902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241242847768871:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.394962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.394985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.395022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:02.395081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244429948389:2050] [node 3] ICS01 socket: 343 reason# EndOfStream
2025-12-24T02:11:02.395085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241244429948288:2050] [node 3] ICP30 unregister session Session# [6:7587241244429948389:2050] VirtualId# [6:7587241244429948385:0]
2025-12-24T02:11:02.395089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244429948389:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.395115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.395138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.410152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:02.410253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241244429948383:2113] [node 4] ICS01 socket: 335 reason# EndOfStream
2025-12-24T02:11:02.410261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241244429948362:2113] [node 4] ICP30 unregister session Session# [6:7587241244429948383:2113] VirtualId# [6:7587241244429948369:0]
2025-12-24T02:11:02.410268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241244429948383:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.410343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:02.410368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:02.445830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241244612603854:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:02.445840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241244612603855:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.441068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241243840724854:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.441078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241243840724855:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.445719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241245841216213:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.445733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241245841216214:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.455130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241245986208183:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.455139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241245986208184:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.455144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241245986208185:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.457097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241243354464771:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.457106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241243354464772:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.457110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241243354464773:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.478257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245757422446:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.478306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245757422447:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.478314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245757422448:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.505482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241244612603915:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:02.506895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245757422504:2104] [node 5] ICP01 ready to work
2025-12-24T02:11:02.511149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241243354464826:2101] [node 4] ICP01 ready to work
2025-12-24T02:11:02.510273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241244612603922:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.521181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464833:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.526935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241245986208250:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.531039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241244612603929:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15885
2025-12-24T02:11:02.534436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241243840724922:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:02.534526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241243840724923:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:02.534596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241243840724924:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:02.531442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241245841216276:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:02.532027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241245841216281:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:02.532109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241245841216282:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:02.534758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241244612603921:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.539110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216291:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
2025-12-24T02:11:02.542866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241245986208185:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(37122918ns) {Configure:104(37121408ns) {ProcessConfigured:135(35023877ns) {StartInitialHandshake:157(35022810ns) {DropHandshakes:470(477ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:477(1717ns) {SwitchToState:179(1146ns)}}}}}
2025-12-24T02:11:02.543050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241245986208252:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.548002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208260:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.546967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245757422514:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:02.547700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422516:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
2025-12-24T02:11:02.548080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208261:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.548115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208262:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.548132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.550857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241245986208184:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44228948ns) {Configure:104(44228002ns) {ProcessConfigured:135(42556200ns) {StartInitialHandshake:157(42555378ns) {DropHandshakes:470(555ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(308ns) PrepareNewSessionHandshake:477(1117ns) {SwitchToState:179(604ns)}}}}}
2025-12-24T02:11:02.550966Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241243840724923:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16337663ns) {Configure:104(16337440ns) {ProcessConfigured:135(16105686ns) {StartInitialHandshake:157(16105453ns) {DropHandshakes:470(459ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(703ns) {SwitchToState:179(535ns)}}}}}
2025-12-24T02:11:02.551085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241243840724922:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16496163ns) {Configure:104(16494976ns) {ProcessConfigured:135(16206844ns) {StartInitialHandshake:157(16206352ns) {DropHandshakes:470(409ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(826ns) {SwitchToState:179(323ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.551537Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241244612603921:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16675705ns) {Configure:104(16675104ns) {ProcessConfigured:135(16312374ns) {StartInitialHandshake:157(16312098ns) {DropHandshakes:470(552ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(664ns) {SwitchToState:179(57ns)}}}}}
2025-12-24T02:11:02.551716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241244612603934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15885
2025-12-24T02:11:02.551727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241244612603935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15885
2025-12-24T02:11:02.544562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241243354464840:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:02.551402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241245841216213:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(41919977ns) {Configure:104(41919406ns) {ProcessConfigured:135(41559603ns) {StartInitialHandshake:157(41559351ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(930ns) {SwitchToState:179(491ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.553819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603929:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.554003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216293:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
2025-12-24T02:11:02.555202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216293:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.555423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603935:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.555526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216291:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.555545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241245841216213:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.556072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603926:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.556084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241244612603926:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.556721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241244612603926:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.557119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216296:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
2025-12-24T02:11:02.556830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241244612603854:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.556847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.556854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241244612603854:2048] [node 2] ICP22 created new session: [1:7587241244612603940:2048]
2025-12-24T02:11:02.556860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241244612603940:2048] [node 2] ICS09 handshake done sender: [1:7587241244612603926:2112] self: [1:7587241244612603925:0] peer: [2:7587241245841216295:0] socket: 300 qp: -1
2025-12-24T02:11:02.556867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241244612603940:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:02.556876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.557329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241245841216291:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.557403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241245841216213:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.557414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.557419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241245841216213:2048] [node 1] ICP22 created new session: [2:7587241245841216297:2048]
2025-12-24T02:11:02.557423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241245841216297:2048] [node 1] ICS09 handshake done sender: [2:7587241245841216291:2114] self: [2:7587241245841216295:0] peer: [1:7587241244612603925:0] socket: 305 qp: -1
2025-12-24T02:11:02.557427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241245841216297:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.557442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.559489Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241245757422446:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20658308ns) {Configure:104(20657538ns) {ProcessConfigured:135(20363988ns) {StartInitialHandshake:157(20363744ns) {DropHandshakes:470(326ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(744ns) {SwitchToState:179(300ns)}}}}}
2025-12-24T02:11:02.560527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208260:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.560614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241245986208183:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.564427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208263:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.567857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724933:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.567929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.567940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.568152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724933:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.568184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241243840724854:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.568274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603924:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.568284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241244612603924:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.568850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241244612603924:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.568913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241244612603855:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.568927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.568933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241244612603855:2049] [node 3] ICP22 created new session: [1:7587241244612603943:2049]
2025-12-24T02:11:02.568939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241244612603943:2049] [node 3] ICS09 handshake done sender: [1:7587241244612603924:2111] self: [1:7587241244612603923:0] peer: [3:7587241243840724936:0] socket: 299 qp: -1
2025-12-24T02:11:02.568946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241244612603943:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.568952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.570740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241243840724854:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3390) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241243840724933:2115] held: no
2025-12-24T02:11:02.574984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241243354464771:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51770909ns) {Configure:104(51769649ns) {ProcessConfigured:135(51409895ns) {StartInitialHandshake:157(51409086ns) {DropHandshakes:470(758ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(1401ns) {SwitchToState:179(717ns)}}}}}
2025-12-24T02:11:02.575852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603917:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.575875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241244612603917:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.582473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422524:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.583519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724938:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.583705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241243840724933:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.583749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241243840724854:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.583762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.583377Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241245757422447:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20162741ns) {Configure:104(20161669ns) {ProcessConfigured:135(19699927ns) {StartInitialHandshake:157(19699280ns) {DropHandshakes:470(1053ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(204ns)} GenerateSessionVirtualId:988(374ns) PrepareNewSessionHandshake:477(1077ns) {SwitchToState:179(531ns)}}}}}
2025-12-24T02:11:02.583769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241243840724854:2048] [node 1] ICP22 created new session: [3:7587241243840724939:2048]
2025-12-24T02:11:02.583775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241243840724939:2048] [node 1] ICS09 handshake done sender: [3:7587241243840724933:2115] self: [3:7587241243840724936:0] peer: [1:7587241244612603923:0] socket: 311 qp: -1
2025-12-24T02:11:02.583782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241243840724939:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.583813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.590613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603928:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.590624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241244612603928:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.590411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245757422524:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.590469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245757422446:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.591012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208267:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.596226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241243354464840:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21028447ns) {Configure:104(21026770ns) {ProcessConfigured:135(20675447ns) {StartInitialHandshake:157(20674753ns) {DropHandshakes:470(617ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:477(1339ns) {SwitchToState:179(752ns)}}}}}
2025-12-24T02:11:02.596317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464833:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.598864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208268:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.600777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422527:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.602981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464851:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.603003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464852:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.603014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464853:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.602891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241244612603917:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.602965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241244612603947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15885
2025-12-24T02:11:02.606472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241245986208260:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.606676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464854:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.606826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241244612603928:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.606835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464853:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.606878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241243354464773:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.607020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464854:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.607037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241243354464771:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.606943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241244612603922:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.606961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.606967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241244612603922:2110] [node 6] ICP22 created new session: [1:7587241244612603949:2110]
2025-12-24T02:11:02.606974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241244612603949:2110] [node 6] ICS09 handshake done sender: [1:7587241244612603928:2113] self: [1:7587241244612603927:0] peer: [6:7587241245757422526:0] socket: 324 qp: -1
2025-12-24T02:11:02.606979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241244612603949:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:02.606988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.607146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241244612603933:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.607150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241244612603933:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.607247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241244612603915:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.607256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.607261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241244612603915:2107] [node 4] ICP22 created new session: [1:7587241244612603951:2107]
2025-12-24T02:11:02.607266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241244612603951:2107] [node 4] ICS09 handshake done sender: [1:7587241244612603917:2108] self: [1:7587241244612603916:0] peer: [4:7587241245986208265:0] socket: 288 qp: -1
2025-12-24T02:11:02.607272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241244612603951:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:02.607279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.606895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241245986208183:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.606913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.606925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241245986208183:2048] [node 1] ICP22 created new session: [4:7587241245986208269:2048]
2025-12-24T02:11:02.606931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245986208269:2048] [node 1] ICS09 handshake done sender: [4:7587241245986208260:2114] self: [4:7587241245986208265:0] peer: [1:7587241244612603916:0] socket: 298 qp: -1
2025-12-24T02:11:02.606939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245986208269:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.606960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.607408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208267:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.607531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241245986208185:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.607752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724929:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.607768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241243840724929:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.608601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241243840724929:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.608670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241243840724922:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.608678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.608684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241243840724922:2108] [node 4] ICP22 created new session: [3:7587241243840724942:2108]
2025-12-24T02:11:02.608688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241243840724942:2108] [node 4] ICS09 handshake done sender: [3:7587241243840724929:2113] self: [3:7587241243840724926:0] peer: [4:7587241245986208272:0] socket: 290 qp: -1
2025-12-24T02:11:02.608694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241243840724942:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:02.608701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.617166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464857:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.617373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241243354464853:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.617458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241243354464773:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.617474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.617479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241243354464773:2050] [node 3] ICP22 created new session: [5:7587241243354464858:2050]
2025-12-24T02:11:02.617486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241243354464858:2050] [node 3] ICS09 handshake done sender: [5:7587241243354464853:2118] self: [5:7587241243354464855:0] peer: [3:7587241243840724925:0] socket: 340 qp: -1
2025-12-24T02:11:02.617493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241243354464858:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.617511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.614778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724928:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.614806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241243840724928:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.615650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241243840724928:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.615735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241243840724923:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.615748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.615758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241243840724923:2110] [node 5] ICP22 created new session: [3:7587241243840724946:2110]
2025-12-24T02:11:02.615763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241243840724946:2110] [node 5] ICS09 handshake done sender: [3:7587241243840724928:2112] self: [3:7587241243840724925:0] peer: [5:7587241243354464855:0] socket: 319 qp: -1
2025-12-24T02:11:02.615769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241243840724946:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:02.615778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.615939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245757422446:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3390) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241245757422524:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.626915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422530:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
2025-12-24T02:11:02.626989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
2025-12-24T02:11:02.628272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422529:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
2025-12-24T02:11:02.628452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245757422524:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.628525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245757422446:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.628540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.628547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245757422446:2048] [node 1] ICP22 created new session: [6:7587241245757422531:2048]
2025-12-24T02:11:02.628553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245757422531:2048] [node 1] ICS09 handshake done sender: [6:7587241245757422524:2115] self: [6:7587241245757422526:0] peer: [1:7587241244612603927:0] socket: 327 qp: -1
2025-12-24T02:11:02.628560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245757422531:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.628569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.628202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208273:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.631292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464861:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.631567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724934:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.631609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241243840724922:2108] [node 4] ICP18 handshake (actor [3:7587241243840724934:2116]) is too old
2025-12-24T02:11:02.631848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464852:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.631872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241243354464826:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.633019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464863:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.633232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241243354464852:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.633280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241243354464826:2101] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.633294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.633299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241243354464826:2101] [node 4] ICP22 created new session: [5:7587241243354464864:2101]
2025-12-24T02:11:02.633304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241243354464864:2101] [node 4] ICS09 handshake done sender: [5:7587241243354464852:2117] self: [5:7587241243354464862:0] peer: [4:7587241245986208253:0] socket: 339 qp: -1
2025-12-24T02:11:02.633310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241243354464864:2101] [node 4] ICS10 traffic start
2025-12-24T02:11:02.633317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.632360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208254:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.632371Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241245986208254:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.632717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241245986208185:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25108) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241245986208267:2119] held: no
2025-12-24T02:11:02.633109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241245986208254:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.633520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241245986208250:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.633527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.633549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241245986208250:2101] [node 5] ICP22 created new session: [4:7587241245986208276:2101]
2025-12-24T02:11:02.633558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245986208276:2101] [node 5] ICS09 handshake done sender: [4:7587241245986208254:2112] self: [4:7587241245986208253:0] peer: [5:7587241243354464862:0] socket: 308 qp: -1
2025-12-24T02:11:02.633563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245986208276:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:02.633567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.637280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241243354464854:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.637352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241243354464771:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.637368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.637373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241243354464771:2048] [node 1] ICP22 created new session: [5:7587241243354464866:2048]
2025-12-24T02:11:02.637378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241243354464866:2048] [node 1] ICS09 handshake done sender: [5:7587241243354464854:2119] self: [5:7587241243354464856:0] peer: [1:7587241244612603932:0] socket: 341 qp: -1
2025-12-24T02:11:02.637387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241243354464866:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.637395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.637691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
2025-12-24T02:11:02.637750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216301:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
2025-12-24T02:11:02.636444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241244612603933:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.636571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241244612603921:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.636583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.636598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241244612603921:2109] [node 5] ICP22 created new session: [1:7587241244612603954:2109]
2025-12-24T02:11:02.636604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241244612603954:2109] [node 5] ICS09 handshake done sender: [1:7587241244612603933:2115] self: [1:7587241244612603932:0] peer: [5:7587241243354464856:0] socket: 291 qp: -1
2025-12-24T02:11:02.636611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241244612603954:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:02.636620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.636512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241245986208267:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.638326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241245986208185:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.638336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.638341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241245986208185:2050] [node 3] ICP22 created new session: [4:7587241245986208278:2050]
2025-12-24T02:11:02.638346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245986208278:2050] [node 3] ICS09 handshake done sender: [4:7587241245986208267:2119] self: [4:7587241245986208272:0] peer: [3:7587241243840724926:0] socket: 332 qp: -1
2025-12-24T02:11:02.638351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245986208278:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.638370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.638057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245757422530:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.638099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245757422448:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.641607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216301:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.641852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216300:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.641938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464851:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.641956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241243354464772:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.642293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208262:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.642319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241245986208184:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.639725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245757422529:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.640836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245757422514:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.642161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724935:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.642193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241243840724855:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.646796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216290:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.646808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241245841216290:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.646792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245757422516:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.646821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245757422527:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.646832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245757422447:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.646852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245757422504:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.654770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241243354464843:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.654793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241243354464843:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.654909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241245986208256:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.654933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241245986208256:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.654861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216284:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.654876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241245841216284:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.654899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216280:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.654905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241245841216280:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.655294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241245841216286:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.655296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241245841216286:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.656258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241245841216280:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.656592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241245841216286:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.656660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241245841216276:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.656676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.656682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241245841216276:2107] [node 4] ICP22 created new session: [2:7587241245841216307:2107]
2025-12-24T02:11:02.656687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241245841216307:2107] [node 4] ICS09 handshake done sender: [2:7587241245841216280:2110] self: [2:7587241245841216279:0] peer: [4:7587241245986208280:0] socket: 301 qp: -1
2025-12-24T02:11:02.656692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241245841216307:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:02.656712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.656820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241245841216282:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.656825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.656828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241245841216282:2109] [node 6] ICP22 created new session: [2:7587241245841216309:2109]
2025-12-24T02:11:02.656832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241245841216309:2109] [node 6] ICS09 handshake done sender: [2:7587241245841216286:2112] self: [2:7587241245841216285:0] peer: [6:7587241245757422536:0] socket: 302 qp: -1
2025-12-24T02:11:02.656835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241245841216309:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:02.656838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.657421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241245841216290:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.654799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724932:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.654819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241243840724932:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.656426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724949:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.657207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241243840724950:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5218
2025-12-24T02:11:02.657555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241243840724935:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.657591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241243840724855:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.657600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.657605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241243840724855:2049] [node 2] ICP22 created new session: [3:7587241243840724951:2049]
2025-12-24T02:11:02.657609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241243840724951:2049] [node 2] ICS09 handshake done sender: [3:7587241243840724935:2117] self: [3:7587241243840724948:0] peer: [2:7587241245841216274:0] socket: 329 qp: -1
2025-12-24T02:11:02.657614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241243840724951:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.657619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.658388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241245841216214:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.658406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.658411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241245841216214:2049] [node 3] ICP22 created new session: [2:7587241245841216311:2049]
2025-12-24T02:11:02.658415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241245841216311:2049] [node 3] ICS09 handshake done sender: [2:7587241245841216290:2113] self: [2:7587241245841216274:0] peer: [3:7587241243840724948:0] socket: 326 qp: -1
2025-12-24T02:11:02.658419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241245841216311:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.658426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.655990Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241245986208250:2101] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:02.658881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241244612603921:2109] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.661490Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241245986208184:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19737) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241245986208262:2116] held: no
2025-12-24T02:11:02.661554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241245986208281:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52082
2025-12-24T02:11:02.661715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241245986208262:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.661840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241245986208184:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.661855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.661860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241245986208184:2049] [node 2] ICP22 created new session: [4:7587241245986208282:2049]
2025-12-24T02:11:02.661866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245986208282:2049] [node 2] ICS09 handshake done sender: [4:7587241245986208262:2116] self: [4:7587241245986208280:0] peer: [2:7587241245841216279:0] socket: 316 qp: -1
2025-12-24T02:11:02.661871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245986208282:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.661878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.662891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241243840724949:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.665053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241243840724932:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.665582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241243840724924:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.665608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.665618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241243840724924:2111] [node 6] ICP22 created new session: [3:7587241243840724955:2111]
2025-12-24T02:11:02.665626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241243840724955:2111] [node 6] ICS09 handshake done sender: [3:7587241243840724932:2114] self: [3:7587241243840724931:0] peer: [6:7587241245757422534:0] socket: 321 qp: -1
2025-12-24T02:11:02.665638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241243840724955:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:02.665647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.665817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241243354464843:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.665898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241243354464870:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33036
2025-12-24T02:11:02.665909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241243354464840:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.665927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.665932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241243354464840:2104] [node 6] ICP22 created new session: [5:7587241243354464871:2104]
2025-12-24T02:11:02.665937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241243354464871:2104] [node 6] ICS09 handshake done sender: [5:7587241243354464843:2113] self: [5:7587241243354464842:0] peer: [6:7587241245757422537:0] socket: 334 qp: -1
2025-12-24T02:11:02.665942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241243354464871:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:02.665949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.666789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241245841216284:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.666906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241245841216281:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.666920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.666927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241245841216281:2108] [node 5] ICP22 created new session: [2:7587241245841216314:2108]
2025-12-24T02:11:02.666933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241245841216314:2108] [node 5] ICS09 handshake done sender: [2:7587241245841216284:2111] self: [2:7587241245841216283:0] peer: [5:7587241243354464868:0] socket: 303 qp: -1
2025-12-24T02:11:02.666937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241245841216314:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.666943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.667496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241245986208256:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.667632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241245986208252:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.667650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.667658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241245986208252:2110] [node 6] ICP22 created new session: [4:7587241245986208285:2110]
2025-12-24T02:11:02.667663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241245986208285:2110] [node 6] ICS09 handshake done sender: [4:7587241245986208256:2113] self: [4:7587241245986208255:0] peer: [6:7587241245757422535:0] socket: 306 qp: -1
2025-12-24T02:11:02.667668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241245986208285:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:02.667675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.668326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241243354464851:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.668528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245757422504:2104] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241245757422527:2117] held: no
2025-12-24T02:11:02.668428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241243354464772:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.668441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.668446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241243354464772:2049] [node 2] ICP22 created new session: [5:7587241243354464873:2049]
2025-12-24T02:11:02.668452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241243354464873:2049] [node 2] ICS09 handshake done sender: [5:7587241243354464851:2116] self: [5:7587241243354464868:0] peer: [2:7587241245841216283:0] socket: 310 qp: -1
2025-12-24T02:11:02.668457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241243354464873:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.668465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.672616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245757422514:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241245757422529:2119] held: no
2025-12-24T02:11:02.672655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245757422448:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25108) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241245757422530:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.672883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422538:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.673028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241245841216316:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.673237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422539:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.673395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422540:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.673518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245757422541:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10011
2025-12-24T02:11:02.673656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245757422516:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.673705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245757422527:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.673752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245757422530:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.673798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245757422529:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.673851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245757422447:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.673869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.673879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245757422447:2049] [node 2] ICP22 created new session: [6:7587241245757422543:2049]
2025-12-24T02:11:02.673885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245757422543:2049] [node 2] ICS09 handshake done sender: [6:7587241245757422516:2112] self: [6:7587241245757422536:0] peer: [2:7587241245841216285:0] socket: 312 qp: -1
2025-12-24T02:11:02.673895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245757422543:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.673902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.674056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245757422504:2104] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.674068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.674073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245757422504:2104] [node 5] ICP22 created new session: [6:7587241245757422545:2104]
2025-12-24T02:11:02.674083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245757422545:2104] [node 5] ICS09 handshake done sender: [6:7587241245757422527:2117] self: [6:7587241245757422537:0] peer: [5:7587241243354464842:0] socket: 337 qp: -1
2025-12-24T02:11:02.674088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245757422545:2104] [node 5] ICS10 traffic start
2025-12-24T02:11:02.674094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.674160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245757422448:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.674172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.674176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245757422448:2050] [node 3] ICP22 created new session: [6:7587241245757422547:2050]
2025-12-24T02:11:02.674186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245757422547:2050] [node 3] ICS09 handshake done sender: [6:7587241245757422530:2120] self: [6:7587241245757422534:0] peer: [3:7587241243840724931:0] socket: 350 qp: -1
2025-12-24T02:11:02.674419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245757422547:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:02.674431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.674501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245757422514:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.674511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.674516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245757422514:2109] [node 4] ICP22 created new session: [6:7587241245757422549:2109]
2025-12-24T02:11:02.674520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245757422549:2109] [node 4] ICS09 handshake done sender: [6:7587241245757422529:2119] self: [6:7587241245757422535:0] peer: [4:7587241245986208255:0] socket: 349 qp: -1
2025-12-24T02:11:02.674524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245757422549:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:02.674529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.691117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:02.691226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241245841216297:2048] [node 1] ICS01 socket: 305 reason# EndOfStream
2025-12-24T02:11:02.691234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241245841216213:2048] [node 1] ICP30 unregister session Session# [2:7587241245841216297:2048] VirtualId# [2:7587241245841216295:0]
2025-12-24T02:11:02.691241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241245841216297:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.691274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.691292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.708967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:02.709078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241243840724939:2048] [node 1] ICS01 socket: 311 reason# EndOfStream
2025-12-24T02:11:02.709093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241243840724854:2048] [node 1] ICP30 unregister session Session# [3:7587241243840724939:2048] VirtualId# [3:7587241243840724936:0]
2025-12-24T02:11:02.709101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241243840724939:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.709159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.709189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.717872Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:02.717997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245757422531:2048] [node 1] ICS01 socket: 327 reason# EndOfStream
2025-12-24T02:11:02.718008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245757422446:2048] [node 1] ICP30 unregister session Session# [6:7587241245757422531:2048] VirtualId# [6:7587241245757422526:0]
2025-12-24T02:11:02.718016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245757422531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.718086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.718115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.718944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:02.719019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241245986208269:2048] [node 1] ICS01 socket: 298 reason# EndOfStream
2025-12-24T02:11:02.719026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241245986208183:2048] [node 1] ICP30 unregister session Session# [4:7587241245986208269:2048] VirtualId# [4:7587241245986208265:0]
2025-12-24T02:11:02.719033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241245986208269:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.719092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.719117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.721337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:02.721435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241243354464866:2048] [node 1] ICS07 socket disconnect 341 reason# EndOfStream
2025-12-24T02:11:02.721441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241243354464866:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.721499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:02.721519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241243354464866:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:02.726776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:02.726865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241243840724951:2049] [node 2] ICS01 socket: 329 reason# EndOfStream
2025-12-24T02:11:02.726875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241243840724855:2049] [node 2] ICP30 unregister session Session# [3:7587241243840724951:2049] VirtualId# [3:7587241243840724948:0]
2025-12-24T02:11:02.726883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241243840724951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.726943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.726968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.730499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:02.730574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241243354464873:2049] [node 2] ICS01 socket: 310 reason# EndOfStream
2025-12-24T02:11:02.730584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241243354464772:2049] [node 2] ICP30 unregister session Session# [5:7587241243354464873:2049] VirtualId# [5:7587241243354464868:0]
2025-12-24T02:11:02.730592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241243354464873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.730648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.730675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.730724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241243354464771:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3390) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:02.730734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241243354464866:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:02.730739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241243354464771:2048] [node 1] ICP30 unregister session Session# [5:7587241243354464866:2048] VirtualId# [5:7587241243354464856:0]
2025-12-24T02:11:02.730751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:02.730758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241243354464771:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:02.730760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3390) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:02.732663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:02.732770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245757422543:2049] [node 2] ICS01 socket: 312 reason# EndOfStream
2025-12-24T02:11:02.732779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245757422447:2049] [node 2] ICP30 unregister session Session# [6:7587241245757422543:2049] VirtualId# [6:7587241245757422536:0]
2025-12-24T02:11:02.732786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245757422543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.732853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.732885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.734759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:02.734852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241245986208282:2049] [node 2] ICS01 socket: 316 reason# EndOfStream
2025-12-24T02:11:02.734860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241245986208184:2049] [node 2] ICP30 unregister session Session# [4:7587241245986208282:2049] VirtualId# [4:7587241245986208280:0]
2025-12-24T02:11:02.734867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241245986208282:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.734935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:02.734963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:02.741880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241243354464771:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:02.753005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:02.753092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245757422547:2050] [node 3] ICS01 socket: 350 reason# EndOfStream
2025-12-24T02:11:02.753100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245757422448:2050] [node 3] ICP30 unregister session Session# [6:7587241245757422547:2050] VirtualId# [6:7587241245757422534:0]
2025-12-24T02:11:02.753106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245757422547:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.753177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.753203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.755971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:02.756062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241243354464858:2050] [node 3] ICS01 socket: 340 reason# EndOfStream
2025-12-24T02:11:02.756070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241243354464773:2050] [node 3] ICP30 unregister session Session# [5:7587241243354464858:2050] VirtualId# [5:7587241243354464855:0]
2025-12-24T02:11:02.756075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241243354464858:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.756132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:02.756170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:02.763344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:02.763448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245757422549:2109] [node 4] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:02.763459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245757422514:2109] [node 4] ICP30 unregister session Session# [6:7587241245757422549:2109] VirtualId# [6:7587241245757422535:0]
2025-12-24T02:11:02.763466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245757422549:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.763535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:02.763562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:02.773805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:02.773907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245757422545:2104] [node 5] ICS01 socket: 337 reason# EndOfStream
2025-12-24T02:11:02.773917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245757422504:2104] [node 5] ICP30 unregister session Session# [6:7587241245757422545:2104] VirtualId# [6:7587241245757422537:0]
2025-12-24T02:11:02.773927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245757422545:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:02.773991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:02.774017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:02.826787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246321266933:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:02.826822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246321266934:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.830844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241242359253801:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.830859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241242359253802:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:02.834923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241244916788714:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.834932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241244916788715:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.838368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241246245555244:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.838378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241246245555245:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.838383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241246245555246:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.863134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241246549432408:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.863145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241246549432409:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.863150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241246549432410:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.870419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245827888603:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:02.870430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245827888604:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:02.870436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245827888605:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:02.890468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241246549432425:2059] [node 6] ICP01 ready to work
2025-12-24T02:11:02.893434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246321266994:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:02.894980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241246245555297:2097] [node 6] ICP01 ready to work
2025-12-24T02:11:02.895640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245827888632:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.900232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246321266997:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:02.907911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246321267001:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:02.909891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242359253861:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
2025-12-24T02:11:02.910069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241242359253868:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:02.914801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241242359253802:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19908211ns) {Configure:104(19906929ns) {ProcessConfigured:135(19454289ns) {StartInitialHandshake:157(19453676ns) {DropHandshakes:470(473ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1642ns) {SwitchToState:179(852ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.918892Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241245827888632:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23186137ns) {Configure:104(23185320ns) {ProcessConfigured:135(22920289ns) {StartInitialHandshake:157(22919568ns) {DropHandshakes:470(545ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:477(5485ns) {SwitchToState:179(766ns)}}}}}
2025-12-24T02:11:02.918899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241246321266933:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23719575ns) {Configure:104(23718914ns) {ProcessConfigured:135(23432560ns) {StartInitialHandshake:157(23431556ns) {DropHandshakes:470(490ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(820ns) {SwitchToState:179(713ns)}}}}}
2025-12-24T02:11:02.919259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246321267000:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16419
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.925624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888679:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.923445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241246245555246:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28518057ns) {Configure:104(28516414ns) {ProcessConfigured:135(27155110ns) {StartInitialHandshake:157(27154631ns) {DropHandshakes:470(234ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(973ns) {SwitchToState:179(230ns)}}}}}
2025-12-24T02:11:02.924047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555313:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:02.924060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555314:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:02.924280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241246245555297:2097] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29240828ns) {Configure:104(29239853ns) {ProcessConfigured:135(27914711ns) {StartInitialHandshake:157(27913998ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(421ns) PrepareNewSessionHandshake:477(553ns) {SwitchToState:179(443ns)}}}}}
2025-12-24T02:11:02.924885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241244916788786:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.928239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241244916788776:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:02.930798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241246549432452:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.934927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241244916788714:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27633479ns) {Configure:104(27631895ns) {ProcessConfigured:135(27294148ns) {StartInitialHandshake:157(27293299ns) {DropHandshakes:470(965ns) {DropIncomingHandshake:442(237ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(1466ns) {SwitchToState:179(612ns)}}}}}
2025-12-24T02:11:02.935341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788781:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
2025-12-24T02:11:02.935434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788782:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
2025-12-24T02:11:02.935488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788783:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.938438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555321:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:02.939106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241246549432425:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48557302ns) {Configure:104(48556249ns) {ProcessConfigured:135(45582684ns) {StartInitialHandshake:157(45581892ns) {DropHandshakes:470(429ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(925ns) {SwitchToState:179(465ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.939240Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241246245555245:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30511727ns) {Configure:104(30511189ns) {ProcessConfigured:135(30201660ns) {StartInitialHandshake:157(30201407ns) {DropHandshakes:470(917ns) {DropIncomingHandshake:442(108ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(382ns) {SwitchToState:179(257ns)}}}}}
2025-12-24T02:11:02.940396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241246245555314:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.940436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241246245555244:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.940618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788782:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.940708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241244916788714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.941762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555324:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:02.942058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241246245555314:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.942143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241246245555244:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.942160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.942166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241246245555244:2048] [node 1] ICP22 created new session: [4:7587241246245555325:2048]
2025-12-24T02:11:02.942171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241246245555325:2048] [node 1] ICS09 handshake done sender: [4:7587241246245555314:2112] self: [4:7587241246245555323:0] peer: [1:7587241246321267006:0] socket: 316 qp: -1
2025-12-24T02:11:02.942179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241246245555325:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.942199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.939106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241246321266997:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31110977ns) {Configure:104(31110506ns) {ProcessConfigured:135(30855065ns) {StartInitialHandshake:157(30854819ns) {DropHandshakes:470(363ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(976ns) {SwitchToState:179(502ns)}}}}}
2025-12-24T02:11:02.939619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246321267012:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16419
2025-12-24T02:11:02.939644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246321267015:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16419
2025-12-24T02:11:02.940577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267007:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.940581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246321267007:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.940965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267015:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.940992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267005:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.940994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246321267005:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.941887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246321267007:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.944833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241245827888662:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:02.944723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241242359253867:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:02.942815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246321267005:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.943009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267000:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.943018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246321267001:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.943027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.943034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246321267001:2109] [node 4] ICP22 created new session: [1:7587241246321267019:2109]
2025-12-24T02:11:02.943039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246321267019:2109] [node 4] ICS09 handshake done sender: [1:7587241246321267007:2113] self: [1:7587241246321267006:0] peer: [4:7587241246245555323:0] socket: 313 qp: -1
2025-12-24T02:11:02.943044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246321267019:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:02.943051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.943183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246321266934:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.943209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:02.943213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246321266934:2049] [node 3] ICP22 created new session: [1:7587241246321267021:2049]
2025-12-24T02:11:02.943217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246321267021:2049] [node 3] ICS09 handshake done sender: [1:7587241246321267005:2112] self: [1:7587241246321267004:0] peer: [3:7587241244916788794:0] socket: 312 qp: -1
2025-12-24T02:11:02.943222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246321267021:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.943226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.945314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246321267011:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16419
2025-12-24T02:11:02.945710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267011:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.945720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246321267013:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16419
2025-12-24T02:11:02.946007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241246245555321:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.946039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241246245555246:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.946818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241242359253866:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:02.946731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241246245555304:2100] [node 5] ICP01 ready to work
2025-12-24T02:11:02.945349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788780:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
2025-12-24T02:11:02.946736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241244916788778:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:02.946777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241244916788714:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241244916788782:2112] held: no
2025-12-24T02:11:02.946844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788781:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.946869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788780:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.949357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245827888679:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.950934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241246321266994:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(57421488ns) {Configure:104(57420648ns) {ProcessConfigured:135(57038214ns) {StartInitialHandshake:157(57037849ns) {DropHandshakes:470(594ns) {DropIncomingHandshake:442(101ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(778ns) {SwitchToState:179(387ns)}}}}}
2025-12-24T02:11:02.951337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432487:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.951411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432486:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.951612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432486:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.950910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241244916788786:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25962016ns) {Configure:104(25961550ns) {ProcessConfigured:135(25784176ns) {StartInitialHandshake:157(25783593ns) {DropHandshakes:470(138ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(825ns)}}}}}
2025-12-24T02:11:02.951342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241244916788715:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.955099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888681:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.955385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245827888681:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.955438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245827888603:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.955991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432487:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.956033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241246549432408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.956197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267025:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.956206Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246321267025:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.956773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432490:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.956932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246321267025:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.957125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241246549432487:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.957245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241246549432408:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.957264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.957270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241246549432408:2048] [node 1] ICP22 created new session: [5:7587241246549432491:2048]
2025-12-24T02:11:02.957276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241246549432491:2048] [node 1] ICS09 handshake done sender: [5:7587241246549432487:2116] self: [5:7587241246549432489:0] peer: [1:7587241246321266995:0] socket: 340 qp: -1
2025-12-24T02:11:02.957283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241246549432491:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.957290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.957536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246321266994:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.957553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.957558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246321266994:2107] [node 5] ICP22 created new session: [1:7587241246321267029:2107]
2025-12-24T02:11:02.957562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246321267029:2107] [node 5] ICS09 handshake done sender: [1:7587241246321267025:2119] self: [1:7587241246321266995:0] peer: [5:7587241246549432489:0] socket: 339 qp: -1
2025-12-24T02:11:02.957566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246321267029:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:02.957573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.958285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245827888662:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.958195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788800:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.957644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253861:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.957691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253869:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.957694Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241242359253869:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.958577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241242359253869:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.958474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241246245555313:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.958511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241246245555319:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.958515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241246245555319:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.958928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241246245555245:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.959107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241246245555319:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.959103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267014:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.959119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246321267014:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.959201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788790:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.959216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241244916788790:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.959294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267012:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.959660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788801:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
2025-12-24T02:11:02.960225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241246245555297:2097] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.960243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.960252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241246245555297:2097] [node 6] ICP22 created new session: [4:7587241246245555332:2097]
2025-12-24T02:11:02.960258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241246245555332:2097] [node 6] ICS09 handshake done sender: [4:7587241246245555319:2114] self: [4:7587241246245555300:0] peer: [6:7587241245827888684:0] socket: 319 qp: -1
2025-12-24T02:11:02.960265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241246245555332:2097] [node 6] ICS10 traffic start
2025-12-24T02:11:02.960284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.960224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241242359253802:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:02.960242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
::ffff:127.1.0.0
2025-12-24T02:11:02.960250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241242359253802:2049] [node 3] ICP22 created new session: [2:7587241242359253882:2049]
2025-12-24T02:11:02.960256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242359253882:2049] [node 3] ICS09 handshake done sender: [2:7587241242359253869:2111] self: [2:7587241242359253862:0] peer: [3:7587241244916788799:0] socket: 306 qp: -1
2025-12-24T02:11:02.960266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242359253882:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:02.960299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:02.960464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253873:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.960468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241242359253873:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.961041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242359253880:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
2025-12-24T02:11:02.961178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555335:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.961418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241242359253873:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.961318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.962644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.961617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253880:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.961632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241242359253868:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.961641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.961649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241242359253868:2110] [node 4] ICP22 created new session: [2:7587241242359253886:2110]
2025-12-24T02:11:02.961654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242359253886:2110] [node 4] ICS09 handshake done sender: [2:7587241242359253873:2113] self: [2:7587241242359253872:0] peer: [4:7587241246245555331:0] socket: 308 qp: -1
2025-12-24T02:11:02.961659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242359253886:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:02.961666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.962307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241242359253867:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17392132ns) {Configure:104(17390524ns) {ProcessConfigured:135(17052499ns) {StartInitialHandshake:157(17052259ns) {DropHandshakes:470(515ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(102ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(823ns) {SwitchToState:179(609ns)}}}}}
2025-12-24T02:11:02.962757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241246245555313:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.965169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241246245555245:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.965193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.965203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241246245555245:2049] [node 2] ICP22 created new session: [4:7587241246245555337:2049]
2025-12-24T02:11:02.965211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241246245555337:2049] [node 2] ICS09 handshake done sender: [4:7587241246245555313:2111] self: [4:7587241246245555331:0] peer: [2:7587241242359253872:0] socket: 315 qp: -1
2025-12-24T02:11:02.965220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241246245555337:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.965227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.965242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246321267014:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.965323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246321266997:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.965338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.965345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246321266997:2108] [node 6] ICP22 created new session: [1:7587241246321267033:2108]
2025-12-24T02:11:02.965350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246321267033:2108] [node 6] ICS09 handshake done sender: [1:7587241246321267014:2117] self: [1:7587241246321267002:0] peer: [6:7587241245827888682:0] socket: 318 qp: -1
2025-12-24T02:11:02.965354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246321267033:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:02.965360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.965987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241244916788782:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.966091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241244916788781:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.966243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241244916788714:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.966258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.966265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241244916788714:2048] [node 1] ICP22 created new session: [3:7587241244916788803:2048]
2025-12-24T02:11:02.966272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241244916788803:2048] [node 1] ICS09 handshake done sender: [3:7587241244916788782:2112] self: [3:7587241244916788794:0] peer: [1:7587241246321267004:0] socket: 321 qp: -1
2025-12-24T02:11:02.966283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241244916788803:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.966302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.966478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241244916788715:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.966493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.966498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241244916788715:2049] [node 2] ICP22 created new session: [3:7587241244916788805:2049]
2025-12-24T02:11:02.966505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241244916788805:2049] [node 2] ICS09 handshake done sender: [3:7587241244916788781:2111] self: [3:7587241244916788799:0] peer: [2:7587241242359253862:0] socket: 320 qp: -1
2025-12-24T02:11:02.966514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241244916788805:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.966519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.968851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888685:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.968890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888686:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.968905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888687:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.969237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245827888681:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:02.978792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432493:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.978870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241246549432409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.979470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253881:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.979485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241242359253881:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.980314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241242359253881:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.980889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241242359253866:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.980917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.980925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241242359253866:2108] [node 5] ICP22 created new session: [2:7587241242359253893:2108]
2025-12-24T02:11:02.980939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241242359253893:2108] [node 5] ICS09 handshake done sender: [2:7587241242359253881:2115] self: [2:7587241242359253878:0] peer: [5:7587241246549432496:0] socket: 350 qp: -1
2025-12-24T02:11:02.980948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241242359253893:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.980957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.981124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245827888679:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.981371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245827888686:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.981402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245827888662:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.981414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.981419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245827888662:2075] [node 4] ICP22 created new session: [6:7587241245827888689:2075]
2025-12-24T02:11:02.981424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245827888689:2075] [node 4] ICS09 handshake done sender: [6:7587241245827888679:2114] self: [6:7587241245827888684:0] peer: [4:7587241246245555300:0] socket: 325 qp: -1
2025-12-24T02:11:02.981662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245827888689:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:02.981687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.981710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245827888603:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.981729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:02.981743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245827888603:2048] [node 1] ICP22 created new session: [6:7587241245827888691:2048]
2025-12-24T02:11:02.981750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245827888691:2048] [node 1] ICS09 handshake done sender: [6:7587241245827888681:2115] self: [6:7587241245827888682:0] peer: [1:7587241246321267002:0] socket: 333 qp: -1
2025-12-24T02:11:02.981756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245827888691:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:02.981763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:02.981760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241246549432409:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241246549432493:2118] held: no
2025-12-24T02:11:02.975961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241244916788790:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.976562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241244916788776:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.976574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.976581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241244916788776:2107] [node 4] ICP22 created new session: [3:7587241244916788809:2107]
2025-12-24T02:11:02.976588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241244916788809:2107] [node 4] ICS09 handshake done sender: [3:7587241244916788790:2116] self: [3:7587241244916788789:0] peer: [4:7587241246245555328:0] socket: 330 qp: -1
2025-12-24T02:11:02.976593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241244916788809:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:02.976602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.977352Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788783:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.978195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241244916788776:2107] [node 4] ICP18 handshake (actor [3:7587241244916788783:2113]) is too old
2025-12-24T02:11:02.983082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432494:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.983117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241246549432452:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.982867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241246245555246:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6952) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241246245555321:2115] held: no
2025-12-24T02:11:02.982939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241246245555245:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17733254ns) {HandleHandshakeStatus:329(17732549ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(2781ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(13143266ns) {ForwardSessionEventToSession:612(13132892ns) ForwardSessionEventToSession:612(1464ns) ForwardSessionEventToSession:612(891ns) ForwardSessionEventToSession:612(490ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(1271ns) ForwardSessionEventToSession:612(820ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(521ns) ForwardSessionEventToSession:612(714ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.983408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241246245555334:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.983416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241246245555334:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.984727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241246245555334:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.984823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241246245555304:2100] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.984847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.984857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241246245555304:2100] [node 5] ICP22 created new session: [4:7587241246245555340:2100]
2025-12-24T02:11:02.984864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241246245555340:2100] [node 5] ICS09 handshake done sender: [4:7587241246245555334:2118] self: [4:7587241246245555329:0] peer: [5:7587241246549432497:0] socket: 349 qp: -1
2025-12-24T02:11:02.984872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241246245555340:2100] [node 5] ICS10 traffic start
2025-12-24T02:11:02.984901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.986971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432498:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.987027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432500:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.987152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432499:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
2025-12-24T02:11:02.987238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241246549432494:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:02.987275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241246549432493:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:02.987322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241246549432409:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.987338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:02.987344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241246549432409:2049] [node 2] ICP22 created new session: [5:7587241246549432501:2049]
2025-12-24T02:11:02.987355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241246549432501:2049] [node 2] ICS09 handshake done sender: [5:7587241246549432493:2118] self: [5:7587241246549432496:0] peer: [2:7587241242359253878:0] socket: 351 qp: -1
2025-12-24T02:11:02.987362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241246549432501:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:02.987386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:02.987713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432498:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.987731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241246549432452:2072] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:02.987740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:02.987745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241246549432452:2072] [node 4] ICP22 created new session: [5:7587241246549432503:2072]
2025-12-24T02:11:02.987749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241246549432503:2072] [node 4] ICS09 handshake done sender: [5:7587241246549432494:2119] self: [5:7587241246549432497:0] peer: [4:7587241246245555329:0] socket: 352 qp: -1
2025-12-24T02:11:02.987754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241246549432503:2072] [node 4] ICS10 traffic start
2025-12-24T02:11:02.987759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:02.988497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241246549432410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.989986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241246549432507:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62313
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.990254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242359253895:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
2025-12-24T02:11:02.990451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241246549432498:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.988626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788797:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.988641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241244916788797:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.989537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241244916788797:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:02.989624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241244916788778:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.989650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:02.989658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241244916788778:2108] [node 5] ICP22 created new session: [3:7587241244916788812:2108]
2025-12-24T02:11:02.989664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241244916788812:2108] [node 5] ICS09 handshake done sender: [3:7587241244916788797:2118] self: [3:7587241244916788796:0] peer: [5:7587241246549432505:0] socket: 357 qp: -1
2025-12-24T02:11:02.989672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241244916788812:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:02.989684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:02.995227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253895:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.995274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241242359253801:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:02.997505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241244916788814:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10267
2025-12-24T02:11:02.997894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241244916788795:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.997897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241244916788795:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.998600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241244916788795:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:02.999215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246321267003:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.999230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246321267003:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:02.998836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241244916788786:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:02.998858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:02.998867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241244916788786:2109] [node 6] ICP22 created new session: [3:7587241244916788816:2109]
2025-12-24T02:11:02.998876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241244916788816:2109] [node 6] ICS09 handshake done sender: [3:7587241244916788795:2117] self: [3:7587241244916788787:0] peer: [6:7587241245827888696:0] socket: 329 qp: -1
2025-12-24T02:11:02.998884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241244916788816:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:02.998910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:02.997115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888688:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.997219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245827888604:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.997280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888693:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.997765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245827888688:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:02.997798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245827888605:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:02.998731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241245827888697:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:02.998982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245827888688:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:02.999364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241245827888603:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(30015311ns) {HandleHandshakeStatus:329(30014874ns) {DropIncomingHandshake:442(600ns) DropOutgoingHandshake:457(2889ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(9197482ns) {ForwardSessionEventToSession:612(9187121ns) ForwardSessionEventToSession:612(1163ns) ForwardSessionEventToSession:612(1115ns) ForwardSessionEventToSession:612(539ns) ForwardSessionEventToSession:612(514ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(1254ns)}}}
2025-12-24T02:11:03.000618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242359253898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
2025-12-24T02:11:03.000782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241242359253868:2110] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.002907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245827888605:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.002934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.002942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245827888605:2050] [node 3] ICP22 created new session: [6:7587241245827888698:2050]
2025-12-24T02:11:03.002949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241245827888698:2050] [node 3] ICS09 handshake done sender: [6:7587241245827888688:2120] self: [6:7587241245827888696:0] peer: [3:7587241244916788787:0] socket: 337 qp: -1
2025-12-24T02:11:03.002964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241245827888698:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.003000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.002728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241242359253896:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
2025-12-24T02:11:03.002975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241246549432410:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.002994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.003000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241246549432410:2050] [node 3] ICP22 created new session: [5:7587241250844399804:2050]
2025-12-24T02:11:03.003006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250844399804:2050] [node 3] ICS09 handshake done sender: [5:7587241246549432498:2120] self: [5:7587241246549432505:0] peer: [3:7587241244916788796:0] socket: 358 qp: -1
2025-12-24T02:11:03.003011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250844399804:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.003019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.003802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241242359253802:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.004448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241246549432410:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.006840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241245827888693:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.006890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241245827888632:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.008272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241246549432485:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.008287Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241246549432485:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.010904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241250540522639:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:03.011413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241242359253888:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.011430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241242359253888:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.012811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241246549432485:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.012078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241246245555304:2100] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.012151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241246245555342:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43067
2025-12-24T02:11:03.018879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241246549432425:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.018917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.018928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241246549432425:2059] [node 6] ICP22 created new session: [5:7587241250844399807:2059]
2025-12-24T02:11:03.018935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250844399807:2059] [node 6] ICS09 handshake done sender: [5:7587241246549432485:2114] self: [5:7587241246549432439:0] peer: [6:7587241250122855996:0] socket: 362 qp: -1
2025-12-24T02:11:03.018944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250844399807:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:03.018954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.019491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246321267003:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.019932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246321266933:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.019948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.019955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246321266933:2048] [node 2] ICP22 created new session: [1:7587241250616234332:2048]
2025-12-24T02:11:03.019961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241250616234332:2048] [node 2] ICS09 handshake done sender: [1:7587241246321267003:2111] self: [1:7587241246321266998:0] peer: [2:7587241242359253897:0] socket: 311 qp: -1
2025-12-24T02:11:03.019966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241250616234332:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:03.019974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.019884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241246654221195:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44272
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.021007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241242359253888:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.020337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241250122855997:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:03.020536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241250122855998:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59633
2025-12-24T02:11:03.020724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245827888632:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241245827888693:2121] held: no
2025-12-24T02:11:03.020759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245827888693:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.021462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245827888632:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.021477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.021483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245827888632:2066] [node 5] ICP22 created new session: [6:7587241250122855999:2066]
2025-12-24T02:11:03.021489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241250122855999:2066] [node 5] ICS09 handshake done sender: [6:7587241245827888693:2121] self: [6:7587241250122855996:0] peer: [5:7587241246549432439:0] socket: 297 qp: -1
2025-12-24T02:11:03.021495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241250122855999:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:03.021503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.022885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241246245555321:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.023051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241246245555246:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.023062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.023069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241246245555246:2050] [node 3] ICP22 created new session: [4:7587241250540522640:2050]
2025-12-24T02:11:03.023076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241250540522640:2050] [node 3] ICS09 handshake done sender: [4:7587241246245555321:2115] self: [4:7587241246245555328:0] peer: [3:7587241244916788789:0] socket: 332 qp: -1
2025-12-24T02:11:03.023081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241250540522640:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.023095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.024133Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241244916788786:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.022845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241245827888686:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.026771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241245827888604:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.023525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241242359253801:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241242359253895:2117] held: no
2025-12-24T02:11:03.023563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241242359253867:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.023575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.023580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241242359253867:2109] [node 6] ICP22 created new session: [2:7587241246654221197:2109]
2025-12-24T02:11:03.023585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246654221197:2109] [node 6] ICS09 handshake done sender: [2:7587241242359253888:2116] self: [2:7587241242359253877:0] peer: [6:7587241245827888694:0] socket: 353 qp: -1
2025-12-24T02:11:03.023590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246654221197:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:03.023596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.023732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241242359253895:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.024375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241242359253801:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.024383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.024388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241242359253801:2048] [node 1] ICP22 created new session: [2:7587241246654221199:2048]
2025-12-24T02:11:03.024392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241246654221199:2048] [node 1] ICS09 handshake done sender: [2:7587241242359253895:2117] self: [2:7587241242359253897:0] peer: [1:7587241246321266998:0] socket: 346 qp: -1
2025-12-24T02:11:03.024396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241246654221199:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.024402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.026784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.026791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241245827888604:2049] [node 2] ICP22 created new session: [6:7587241250122856001:2049]
2025-12-24T02:11:03.026797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241250122856001:2049] [node 2] ICS09 handshake done sender: [6:7587241245827888686:2118] self: [6:7587241245827888694:0] peer: [2:7587241242359253877:0] socket: 355 qp: -1
2025-12-24T02:11:03.026803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241250122856001:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.026810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.030989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241245827888604:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.046821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:03.046911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241246654221199:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-12-24T02:11:03.046920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241242359253801:2048] [node 1] ICP30 unregister session Session# [2:7587241246654221199:2048] VirtualId# [2:7587241242359253897:0]
2025-12-24T02:11:03.046927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241246654221199:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.046977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.046999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.047425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:03.047489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241244916788803:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:03.047495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241244916788714:2048] [node 1] ICP30 unregister session Session# [3:7587241244916788803:2048] VirtualId# [3:7587241244916788794:0]
2025-12-24T02:11:03.047499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241244916788803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.047526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.047545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.054754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:03.054855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241246549432491:2048] [node 1] ICS07 socket disconnect 340 reason# EndOfStream
2025-12-24T02:11:03.054863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241246549432491:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.054927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.054948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241246549432491:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:03.055371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241246549432408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.055385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241246549432491:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.055392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241246549432408:2048] [node 1] ICP30 unregister session Session# [5:7587241246549432491:2048] VirtualId# [5:7587241246549432489:0]
2025-12-24T02:11:03.055408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.055420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241246549432408:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.055423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.057423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:03.057523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241245827888691:2048] [node 1] ICS07 socket disconnect 333 reason# EndOfStream
2025-12-24T02:11:03.057537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245827888691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.057603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.057635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241245827888691:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:03.058360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:03.058445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241246245555325:2048] [node 1] ICS07 socket disconnect 316 reason# EndOfStream
2025-12-24T02:11:03.058451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241246245555325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.058670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245827888603:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.058688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245827888691:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.058709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245827888603:2048] [node 1] ICP30 unregister session Session# [6:7587241245827888691:2048] VirtualId# [6:7587241245827888682:0]
2025-12-24T02:11:03.058728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.058740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241245827888603:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.058748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.058499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.058529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241246245555325:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:03.065962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241246549432408:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:03.066002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:03.066101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241246549432501:2049] [node 2] ICS01 socket: 351 reason# EndOfStream
2025-12-24T02:11:03.066109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241246549432409:2049] [node 2] ICP30 unregister session Session# [5:7587241246549432501:2049] VirtualId# [5:7587241246549432496:0]
2025-12-24T02:11:03.066116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241246549432501:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.066176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.066203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.067366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:03.067443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241250122856001:2049] [node 2] ICS07 socket disconnect 355 reason# EndOfStream
2025-12-24T02:11:03.067454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241250122856001:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.067503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.066953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:03.067032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241246245555337:2049] [node 2] ICS07 socket disconnect 315 reason# EndOfStream
2025-12-24T02:11:03.067038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241246245555337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.067096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.067121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241246245555337:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:03.067531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241250122856001:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:03.068064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245827888604:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.068079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241250122856001:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.068086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245827888604:2049] [node 2] ICP30 unregister session Session# [6:7587241250122856001:2049] VirtualId# [6:7587241245827888694:0]
2025-12-24T02:11:03.068104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.068114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241245827888604:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241246245555245:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.068278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241246245555337:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.068283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241246245555245:2049] [node 2] ICP30 unregister session Session# [4:7587241246245555337:2049] VirtualId# [4:7587241246245555331:0]
2025-12-24T02:11:03.068294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.068298Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241246245555245:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61612) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241246245555244:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.068409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241246245555325:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.068414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241246245555244:2048] [node 1] ICP30 unregister session Session# [4:7587241246245555325:2048] VirtualId# [4:7587241246245555323:0]
2025-12-24T02:11:03.068428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.068434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241246245555244:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9024) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.068521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241245827888603:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:03.077353Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:03.077444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241250844399804:2050] [node 3] ICS01 socket: 358 reason# EndOfStream
2025-12-24T02:11:03.077454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241246549432410:2050] [node 3] ICP30 unregister session Session# [5:7587241250844399804:2050] VirtualId# [5:7587241246549432505:0]
2025-12-24T02:11:03.077461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241250844399804:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.077514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.077539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.078443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241245827888604:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:03.078469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:03.078529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241245827888698:2050] [node 3] ICS07 socket disconnect 337 reason# EndOfStream
2025-12-24T02:11:03.078533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245827888698:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.078566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.078586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241245827888698:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:03.079048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245827888605:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6952) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.079060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245827888698:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.079064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245827888605:2050] [node 3] ICP30 unregister session Session# [6:7587241245827888698:2050] VirtualId# [6:7587241245827888696:0]
2025-12-24T02:11:03.079077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.079081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241245827888605:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.079084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6952) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.090797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241245827888605:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:03.098166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:03.098262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241245827888689:2075] [node 4] ICS07 socket disconnect 325 reason# EndOfStream
2025-12-24T02:11:03.098269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241245827888689:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.098332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:03.098356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241245827888689:2075] [node 4] ICS15 start handshake
2025-12-24T02:11:03.098816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241245827888662:2075] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.098828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241245827888689:2075] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.098832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241245827888662:2075] [node 4] ICP30 unregister session Session# [6:7587241245827888689:2075] VirtualId# [6:7587241245827888684:0]
2025-12-24T02:11:03.098847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:03.098853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241245827888662:2075] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.098855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.110765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241245827888662:2075] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:03.146066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249662149071:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.146806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246857258029:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:03.150767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246857258030:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.156572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249662149070:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.161343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247565284971:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.161351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247565284972:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.161356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247565284973:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.146896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249112102684:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.146906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249112102685:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.164479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241250627544664:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.164488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241250627544665:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.164493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241250627544666:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.166517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241248769864165:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.166523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241248769864166:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.166528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241248769864167:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.195547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241248769864213:2093] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.198380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102744:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.197884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249662149130:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-12-24T02:11:03.196974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544725:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.198273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241248769864228:2100] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.201679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246857258093:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:03.199593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249662149131:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-12-24T02:11:03.203251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249112102746:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:03.207950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249112102749:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.209620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102753:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.209496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247565285034:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:03.207001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249662149133:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:03.209622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247565285035:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:03.206925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246857258097:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:03.209685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241250627544736:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:03.210574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241250627544740:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.215531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246857258101:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29961
2025-12-24T02:11:03.218106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149131:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.218170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241249662149070:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.218266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258090:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.218278Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246857258090:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.219018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246857258090:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.219117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246857258029:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.219133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.219141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246857258029:2048] [node 2] ICP22 created new session: [1:7587241246857258106:2048]
2025-12-24T02:11:03.219148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246857258106:2048] [node 2] ICS09 handshake done sender: [1:7587241246857258090:2107] self: [1:7587241246857258089:0] peer: [2:7587241249662149141:0] socket: 306 qp: -1
2025-12-24T02:11:03.219157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246857258106:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:03.219177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.227902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246857258110:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:03.226949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249662149144:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:03.227066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249662149145:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.233129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246857258113:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29961
2025-12-24T02:11:03.233235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241249112102746:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29514605ns) {Configure:104(22217127ns) {ProcessConfigured:135(21882128ns) {StartInitialHandshake:157(21881695ns) {DropHandshakes:470(519ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:477(1758ns) {SwitchToState:179(282ns)}}}}}
2025-12-24T02:11:03.232954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241248769864165:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22122300ns) {Configure:104(22121740ns) {ProcessConfigured:135(21791371ns) {StartInitialHandshake:157(21790764ns) {DropHandshakes:470(423ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1194ns) {SwitchToState:179(585ns)}}}}}
2025-12-24T02:11:03.233408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249112102757:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.234937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544744:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.234967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544745:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.235133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249662149150:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-12-24T02:11:03.235147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249662149151:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50233
2025-12-24T02:11:03.239075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258101:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.239125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241246857258029:2048] [node 2] ICP18 handshake (actor [1:7587241246857258101:2111]) is too old
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.239381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.239395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.239420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864246:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.239472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864243:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.240339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241248769864245:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.240391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241248769864166:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.241071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864248:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.241188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241248769864245:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.241233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241248769864166:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.241254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.241259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241248769864166:2049] [node 2] ICP22 created new session: [6:7587241248769864249:2049]
2025-12-24T02:11:03.241264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241248769864249:2049] [node 2] ICS09 handshake done sender: [6:7587241248769864245:2117] self: [6:7587241248769864247:0] peer: [2:7587241249662149147:0] socket: 342 qp: -1
2025-12-24T02:11:03.241269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241248769864249:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.241299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241248769864243:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.241305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.241341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241248769864165:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.241802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258099:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.241812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246857258099:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.242524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246857258099:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.242631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246857258097:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.242644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.242660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246857258097:2109] [node 6] ICP22 created new session: [1:7587241246857258115:2109]
2025-12-24T02:11:03.242668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246857258115:2109] [node 6] ICS09 handshake done sender: [1:7587241246857258099:2110] self: [1:7587241246857258098:0] peer: [6:7587241248769864251:0] socket: 310 qp: -1
2025-12-24T02:11:03.242674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246857258115:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:03.242679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.238795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241249662149071:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26459701ns) {Configure:104(26458544ns) {ProcessConfigured:135(26110779ns) {StartInitialHandshake:157(26109936ns) {DropHandshakes:470(451ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(854ns) {SwitchToState:179(370ns)}}}}}
2025-12-24T02:11:03.238983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249662149131:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.240278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241249662149070:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.240290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.240296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249662149070:2048] [node 1] ICP22 created new session: [2:7587241249662149153:2048]
2025-12-24T02:11:03.240302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249662149153:2048] [node 1] ICS09 handshake done sender: [2:7587241249662149131:2108] self: [2:7587241249662149141:0] peer: [1:7587241246857258089:0] socket: 307 qp: -1
2025-12-24T02:11:03.240309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249662149153:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.240317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.240588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149148:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.240593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249662149148:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.241049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249662149148:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.241200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249662149145:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.241385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.241393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249662149145:2113] [node 6] ICP22 created new session: [2:7587241249662149156:2113]
2025-12-24T02:11:03.241399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249662149156:2113] [node 6] ICS09 handshake done sender: [2:7587241249662149148:2114] self: [2:7587241249662149147:0] peer: [6:7587241248769864247:0] socket: 335 qp: -1
2025-12-24T02:11:03.241406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249662149156:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:03.241436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.246772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544743:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.250985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102763:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.251008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102764:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.251082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102765:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.254119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544746:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.254270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544743:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.251170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241249662149144:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24059536ns) {Configure:104(24058415ns) {ProcessConfigured:135(23592790ns) {StartInitialHandshake:157(23591604ns) {DropHandshakes:470(949ns) {DropIncomingHandshake:442(135ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(1646ns) {SwitchToState:179(1032ns)}}}}}
2025-12-24T02:11:03.254302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241249662149070:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.254302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241250627544665:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.269871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241247565284972:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54896106ns) {Configure:104(54894753ns) {ProcessConfigured:135(54480898ns) {StartInitialHandshake:157(54480154ns) {DropHandshakes:470(710ns) {DropIncomingHandshake:442(35ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:477(1307ns) {SwitchToState:179(661ns)}}}}}
2025-12-24T02:11:03.270947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544746:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.271000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241250627544664:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.271153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258112:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.271165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246857258112:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.272195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544749:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.272746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241250627544746:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.272226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149152:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.272239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249662149152:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.272547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246857258112:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.273131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249662149152:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.273228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249662149071:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.273250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.273259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249662149071:2049] [node 3] ICP22 created new session: [2:7587241249662149162:2049]
2025-12-24T02:11:03.273265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249662149162:2049] [node 3] ICS09 handshake done sender: [2:7587241249662149152:2117] self: [2:7587241249662149140:0] peer: [3:7587241249112102768:0] socket: 304 qp: -1
2025-12-24T02:11:03.272850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241250627544664:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.272871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.272879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241250627544664:2048] [node 1] ICP22 created new session: [5:7587241250627544750:2048]
2025-12-24T02:11:03.272885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250627544750:2048] [node 1] ICS09 handshake done sender: [5:7587241250627544746:2119] self: [5:7587241250627544748:0] peer: [1:7587241246857258111:0] socket: 347 qp: -1
2025-12-24T02:11:03.272892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250627544750:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.272910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.271138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241247565284971:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24331833ns) {Configure:104(24331126ns) {ProcessConfigured:135(24027071ns) {StartInitialHandshake:157(24026835ns) {DropHandshakes:470(524ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:477(534ns) {SwitchToState:179(232ns)}}}}}
2025-12-24T02:11:03.273745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246857258110:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.273758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.273764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246857258110:2114] [node 5] ICP22 created new session: [1:7587241246857258119:2114]
2025-12-24T02:11:03.273770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246857258119:2114] [node 5] ICS09 handshake done sender: [1:7587241246857258112:2115] self: [1:7587241246857258111:0] peer: [5:7587241250627544748:0] socket: 346 qp: -1
2025-12-24T02:11:03.273775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246857258119:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:03.273781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.270789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241249112102684:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(67458633ns) {Configure:104(67457885ns) {ProcessConfigured:135(67081873ns) {StartInitialHandshake:157(67081422ns) {DropHandshakes:470(564ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(114ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:477(1373ns) {SwitchToState:179(810ns)}}}}}
2025-12-24T02:11:03.270906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241249112102685:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26499656ns) {Configure:104(26499032ns) {ProcessConfigured:135(26224817ns) {StartInitialHandshake:157(26224516ns) {DropHandshakes:470(459ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(294ns) PrepareNewSessionHandshake:477(910ns) {SwitchToState:179(19ns)}}}}}
2025-12-24T02:11:03.270957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102765:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.270985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241249112102685:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.274890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102764:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.274939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241249112102684:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.273273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249662149162:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:03.273288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.275038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149135:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.275046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249662149135:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.279029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285049:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.279119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285050:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.279130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285051:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.279146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285052:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.279182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.282821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258102:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.282840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246857258102:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.282818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285049:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.282857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247565284971:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.282860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285052:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.282905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247565284972:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.286858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258104:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.286871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246857258104:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.287510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285056:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.286797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149159:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.286810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249662149159:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.289393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864253:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.291048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247565285052:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.291107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247565284972:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.291126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.291132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247565284972:2049] [node 2] ICP22 created new session: [4:7587241247565285057:2049]
2025-12-24T02:11:03.291140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241247565285057:2049] [node 2] ICS09 handshake done sender: [4:7587241247565285052:2118] self: [4:7587241247565285055:0] peer: [2:7587241249662149146:0] socket: 356 qp: -1
2025-12-24T02:11:03.291146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241247565285057:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.291177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.290835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249662149159:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.290967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249662149135:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.291209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249662149133:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.291223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.291228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249662149144:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.291232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249662149133:2109] [node 5] ICP22 created new session: [2:7587241249662149166:2109]
2025-12-24T02:11:03.291233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.291237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249662149144:2112] [node 4] ICP22 created new session: [2:7587241249662149167:2112]
2025-12-24T02:11:03.291239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249662149166:2109] [node 5] ICS09 handshake done sender: [2:7587241249662149135:2110] self: [2:7587241249662149134:0] peer: [5:7587241250627544747:0] socket: 311 qp: -1
2025-12-24T02:11:03.291242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249662149167:2112] [node 4] ICS09 handshake done sender: [2:7587241249662149159:2118] self: [2:7587241249662149146:0] peer: [4:7587241247565285055:0] socket: 312 qp: -1
2025-12-24T02:11:03.291246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249662149167:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:03.291247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249662149166:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:03.291252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.291256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.292254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246857258102:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.292371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246857258030:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.292384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.292404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246857258030:2049] [node 3] ICP22 created new session: [1:7587241246857258122:2049]
2025-12-24T02:11:03.292410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246857258122:2049] [node 3] ICS09 handshake done sender: [1:7587241246857258102:2112] self: [1:7587241246857258094:0] peer: [3:7587241249112102769:0] socket: 331 qp: -1
2025-12-24T02:11:03.292416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246857258122:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:03.292426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.293318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544744:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.292830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241248769864243:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.298757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241248769864165:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.298780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.298788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241248769864165:2048] [node 1] ICP22 created new session: [6:7587241248769864254:2048]
2025-12-24T02:11:03.298796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241248769864254:2048] [node 1] ICS09 handshake done sender: [6:7587241248769864243:2115] self: [6:7587241248769864251:0] peer: [1:7587241246857258098:0] socket: 323 qp: -1
2025-12-24T02:11:03.298806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241248769864254:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.298814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.303574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544754:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.304125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241250627544743:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.306603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241250627544665:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.306621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.306627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241250627544665:2049] [node 2] ICP22 created new session: [5:7587241250627544755:2049]
2025-12-24T02:11:03.306633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250627544755:2049] [node 2] ICS09 handshake done sender: [5:7587241250627544743:2116] self: [5:7587241250627544747:0] peer: [2:7587241249662149134:0] socket: 324 qp: -1
2025-12-24T02:11:03.306639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250627544755:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.306646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.306268Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285051:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.310592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249662149150:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.310802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241249662149144:2112] [node 4] ICP35 event processing took too much time PendingConnection:279(19542730ns) {HandleHandshakeStatus:329(19541898ns) {DropIncomingHandshake:442(36ns) DropOutgoingHandshake:457(213ns) SwitchToState:179(59ns) ProcessPendingSessionEvents:495(19492012ns) {ForwardSessionEventToSession:612(19480329ns) ForwardSessionEventToSession:612(1798ns) ForwardSessionEventToSession:612(1025ns) ForwardSessionEventToSession:612(485ns) ForwardSessionEventToSession:612(475ns) ForwardSessionEventToSession:612(978ns) ForwardSessionEventToSession:612(897ns) ForwardSessionEventToSession:612(451ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(619ns)}}}
2025-12-24T02:11:03.310818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241249662149133:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(19599283ns) {HandleHandshakeStatus:329(19598548ns) {DropIncomingHandshake:442(117ns) DropOutgoingHandshake:457(216ns) SwitchToState:179(209ns) ProcessPendingSessionEvents:495(19476746ns) {ForwardSessionEventToSession:612(19465940ns) ForwardSessionEventToSession:612(1256ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(1360ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(556ns)}}}
2025-12-24T02:11:03.310902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241249662149133:2109] [node 5] ICP18 handshake (actor [2:7587241249662149150:2115]) is too old
2025-12-24T02:11:03.311930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102763:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.313404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285041:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.313416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241247565285041:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.314130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247565285041:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.315252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102771:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.316185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544745:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.316283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241250627544736:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.318111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246857258104:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:03.313249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241248769864244:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:03.313289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241248769864228:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.314039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864257:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.314317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241248769864244:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.314367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241248769864228:2100] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.314378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.314385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241248769864228:2100] [node 4] ICP22 created new session: [6:7587241248769864258:2100]
2025-12-24T02:11:03.314392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241248769864258:2100] [node 4] ICS09 handshake done sender: [6:7587241248769864244:2116] self: [6:7587241248769864256:0] peer: [4:7587241247565285040:0] socket: 341 qp: -1
2025-12-24T02:11:03.314398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241248769864258:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:03.314406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.318973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102751:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.318992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249112102751:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.319636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544725:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.322836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285039:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.322849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241247565285039:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.323508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285062:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.323591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241247565284972:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(32459046ns) {HandleHandshakeStatus:329(32458413ns) {DropIncomingHandshake:442(300ns) DropOutgoingHandshake:457(5813ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(28663521ns) {ForwardSessionEventToSession:612(28652594ns) ForwardSessionEventToSession:612(1360ns) ForwardSessionEventToSession:612(694ns) ForwardSessionEventToSession:612(674ns) ForwardSessionEventToSession:612(504ns) ForwardSessionEventToSession:612(1219ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(498ns) ForwardSessionEventToSession:612(764ns) ForwardSessionEventToSession:612(1045ns)}}}
2025-12-24T02:11:03.323868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247565285039:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.324102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241247565285034:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.324114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.324120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247565285034:2108] [node 5] ICP22 created new session: [4:7587241247565285064:2108]
2025-12-24T02:11:03.324125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241247565285064:2108] [node 5] ICS09 handshake done sender: [4:7587241247565285039:2111] self: [4:7587241247565285038:0] peer: [5:7587241250627544757:0] socket: 321 qp: -1
2025-12-24T02:11:03.324130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241247565285064:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:03.324148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.324319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285053:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.324336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241247565285034:2108] [node 5] ICP18 handshake (actor [4:7587241247565285053:2119]) is too old
2025-12-24T02:11:03.324581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247565285049:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.324677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247565284971:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.324687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.324691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247565284971:2048] [node 1] ICP22 created new session: [4:7587241247565285066:2048]
2025-12-24T02:11:03.324695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241247565285066:2048] [node 1] ICS09 handshake done sender: [4:7587241247565285049:2115] self: [4:7587241247565285054:0] peer: [1:7587241246857258095:0] socket: 334 qp: -1
2025-12-24T02:11:03.324698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241247565285066:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.324703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.324348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544759:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.324596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241250627544745:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.324698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249112102765:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.329265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102744:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.329276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241249112102685:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.329290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.329299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249112102685:2049] [node 2] ICP22 created new session: [3:7587241249112102773:2049]
2025-12-24T02:11:03.329305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249112102773:2049] [node 2] ICS09 handshake done sender: [3:7587241249112102765:2117] self: [3:7587241249112102768:0] peer: [2:7587241249662149140:0] socket: 350 qp: -1
2025-12-24T02:11:03.329312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249112102773:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.329320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.330820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247565285050:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.330848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247565284973:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.330978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241250627544736:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.330997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.331003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241250627544736:2112] [node 4] ICP22 created new session: [5:7587241250627544760:2112]
2025-12-24T02:11:03.331008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250627544760:2112] [node 4] ICS09 handshake done sender: [5:7587241250627544745:2118] self: [5:7587241250627544757:0] peer: [4:7587241247565285038:0] socket: 339 qp: -1
2025-12-24T02:11:03.331015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250627544760:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:03.331031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.332288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246857258125:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29961
2025-12-24T02:11:03.332308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246857258126:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29961
2025-12-24T02:11:03.334868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102756:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.334882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249112102756:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.338882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102753:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.338940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249112102756:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.339278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249112102746:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.339296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.339304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249112102746:2108] [node 4] ICP22 created new session: [3:7587241249112102778:2108]
2025-12-24T02:11:03.339310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249112102778:2108] [node 4] ICS09 handshake done sender: [3:7587241249112102756:2113] self: [3:7587241249112102754:0] peer: [4:7587241247565285068:0] socket: 336 qp: -1
2025-12-24T02:11:03.339316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249112102778:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:03.339323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.339644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246857258126:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.339091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247565285069:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36967
2025-12-24T02:11:03.339292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247565285050:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.339369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247565284973:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.339380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.339385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247565284973:2050] [node 3] ICP22 created new session: [4:7587241247565285070:2050]
2025-12-24T02:11:03.339391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241247565285070:2050] [node 3] ICS09 handshake done sender: [4:7587241247565285050:2116] self: [4:7587241247565285068:0] peer: [3:7587241249112102754:0] socket: 326 qp: -1
2025-12-24T02:11:03.339397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241247565285070:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.339401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.347278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249112102772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10523
2025-12-24T02:11:03.348107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241250627544666:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.348157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241250627544666:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(54773492ns) {IncomingHandshake:261(54772926ns) {DropIncomingHandshake:442(233ns) PrepareNewSessionHandshake:477(544ns) {SwitchToState:179(253ns)} GenerateSessionVirtualId:988(277ns) IssueIncomingHandshakeReply:193(54767946ns) {SwitchToState:179(1181ns)}}}
2025-12-24T02:11:03.348264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241247565285035:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.350719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241246857258030:2049] [node 3] ICP18 handshake (actor [1:7587241246857258126:2118]) is too old
2025-12-24T02:11:03.348300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.348309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247565285035:2109] [node 6] ICP22 created new session: [4:7587241247565285072:2109]
2025-12-24T02:11:03.348317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241247565285072:2109] [node 6] ICS09 handshake done sender: [4:7587241247565285041:2112] self: [4:7587241247565285040:0] peer: [6:7587241248769864256:0] socket: 322 qp: -1
2025-12-24T02:11:03.348327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241247565285072:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:03.348360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.354980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249112102764:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.359744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249112102759:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.359753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249112102759:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.360119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249112102751:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.360298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249112102759:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.360353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249112102757:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.360369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.360377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249112102757:2112] [node 6] ICP22 created new session: [3:7587241249112102783:2112]
2025-12-24T02:11:03.360382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249112102783:2112] [node 6] ICS09 handshake done sender: [3:7587241249112102759:2114] self: [3:7587241249112102758:0] peer: [6:7587241248769864260:0] socket: 337 qp: -1
2025-12-24T02:11:03.360388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249112102783:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:03.360396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.360678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249112102749:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.360688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.360692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249112102749:2109] [node 5] ICP22 created new session: [3:7587241249112102785:2109]
2025-12-24T02:11:03.360697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249112102785:2109] [node 5] ICS09 handshake done sender: [3:7587241249112102751:2110] self: [3:7587241249112102750:0] peer: [5:7587241250627544753:0] socket: 314 qp: -1
2025-12-24T02:11:03.360700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249112102785:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:03.360705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.361804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241249112102684:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.361824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.361835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249112102684:2048] [node 1] ICP22 created new session: [3:7587241249112102787:2048]
2025-12-24T02:11:03.361840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249112102787:2048] [node 1] ICS09 handshake done sender: [3:7587241249112102764:2116] self: [3:7587241249112102769:0] peer: [1:7587241246857258094:0] socket: 349 qp: -1
2025-12-24T02:11:03.361846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249112102787:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.361853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.358811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241248769864246:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.358871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241248769864167:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.363134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241248769864167:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6953) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241248769864246:2118] held: no
2025-12-24T02:11:03.363593Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241250627544666:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6953) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241250627544744:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.366905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246857258093:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.366927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.366936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246857258093:2108] [node 4] ICP22 created new session: [1:7587241246857258127:2108]
2025-12-24T02:11:03.366942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246857258127:2108] [node 4] ICS09 handshake done sender: [1:7587241246857258104:2113] self: [1:7587241246857258095:0] peer: [4:7587241247565285054:0] socket: 333 qp: -1
2025-12-24T02:11:03.366950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246857258127:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:03.366977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.371365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864262:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.371061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241250627544763:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36403
2025-12-24T02:11:03.372975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241246857258093:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(54696739ns) {HandleHandshakeStatus:329(54696169ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(338ns) SwitchToState:179(125ns) ProcessPendingSessionEvents:495(772695ns) {ForwardSessionEventToSession:612(763260ns) ForwardSessionEventToSession:612(1313ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(453ns) ForwardSessionEventToSession:612(949ns) ForwardSessionEventToSession:612(692ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(820ns)}}}
2025-12-24T02:11:03.373005Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241246857258093:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.372568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864261:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.372742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241247565285035:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(57617117ns) {HandleHandshakeStatus:329(57616240ns) {DropIncomingHandshake:442(12224ns) DropOutgoingHandshake:457(205ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(5672913ns) {ForwardSessionEventToSession:612(5662109ns) ForwardSessionEventToSession:612(2008ns) ForwardSessionEventToSession:612(743ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(751ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(909ns) ForwardSessionEventToSession:612(447ns) ForwardSessionEventToSession:612(871ns)}}}
2025-12-24T02:11:03.375009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241248769864246:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.375040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241250627544744:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.375151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241250627544666:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.375164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.375172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241250627544666:2050] [node 3] ICP22 created new session: [5:7587241250627544764:2050]
2025-12-24T02:11:03.375178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250627544764:2050] [node 3] ICS09 handshake done sender: [5:7587241250627544744:2117] self: [5:7587241250627544753:0] peer: [3:7587241249112102750:0] socket: 338 qp: -1
2025-12-24T02:11:03.375187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250627544764:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.375214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.378830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241248769864261:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.378865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241248769864167:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.378878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.378886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241248769864167:2050] [node 3] ICP22 created new session: [6:7587241248769864263:2050]
2025-12-24T02:11:03.378894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241248769864263:2050] [node 3] ICS09 handshake done sender: [6:7587241248769864246:2118] self: [6:7587241248769864260:0] peer: [3:7587241249112102758:0] socket: 343 qp: -1
2025-12-24T02:11:03.378895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241248769864213:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.378902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241248769864263:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.378910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.382830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241250627544742:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.382847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241250627544742:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.383708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241250627544742:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.383798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241250627544740:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.383826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.383836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241250627544740:2114] [node 6] ICP22 created new session: [5:7587241250627544767:2114]
2025-12-24T02:11:03.383844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241250627544767:2114] [node 6] ICS09 handshake done sender: [5:7587241250627544742:2115] self: [5:7587241250627544741:0] peer: [6:7587241248769864264:0] socket: 354 qp: -1
2025-12-24T02:11:03.383851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241250627544767:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:03.383867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.386762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241248769864213:2093] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241248769864261:2122] held: no
2025-12-24T02:11:03.386877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241248769864266:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12631
2025-12-24T02:11:03.387067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241248769864261:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.394924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241248769864213:2093] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.394944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.394954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241248769864213:2093] [node 5] ICP22 created new session: [6:7587241248769864267:2093]
2025-12-24T02:11:03.394960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241248769864267:2093] [node 5] ICS09 handshake done sender: [6:7587241248769864261:2122] self: [6:7587241248769864264:0] peer: [5:7587241250627544741:0] socket: 373 qp: -1
2025-12-24T02:11:03.394969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241248769864267:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:03.394983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.422796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:03.422879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241249112102787:2048] [node 1] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:03.422888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241249112102684:2048] [node 1] ICP30 unregister session Session# [3:7587241249112102787:2048] VirtualId# [3:7587241249112102769:0]
2025-12-24T02:11:03.422894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241249112102787:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.422935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.422958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.422779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:03.422880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241249662149153:2048] [node 1] ICS07 socket disconnect 307 reason# EndOfStream
2025-12-24T02:11:03.422888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241249662149153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.422940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.422962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241249662149153:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:03.423326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241249662149070:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2421) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.423334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241249662149153:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.423339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241249662149070:2048] [node 1] ICP30 unregister session Session# [2:7587241249662149153:2048] VirtualId# [2:7587241249662149141:0]
2025-12-24T02:11:03.423354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.423359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241249662149070:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.423362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2421) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.430865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:03.430945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241248769864254:2048] [node 1] ICS01 socket: 323 reason# EndOfStream
2025-12-24T02:11:03.430956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241248769864165:2048] [node 1] ICP30 unregister session Session# [6:7587241248769864254:2048] VirtualId# [6:7587241248769864251:0]
2025-12-24T02:11:03.430961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241248769864254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.430774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:03.430869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241247565285066:2048] [node 1] ICS01 socket: 334 reason# EndOfStream
2025-12-24T02:11:03.430877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241247565284971:2048] [node 1] ICP30 unregister session Session# [4:7587241247565285066:2048] VirtualId# [4:7587241247565285054:0]
2025-12-24T02:11:03.430883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241247565285066:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.430936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.430960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.431015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.431043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.431098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:03.431181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241250627544750:2048] [node 1] ICS01 socket: 347 reason# EndOfStream
2025-12-24T02:11:03.431187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241250627544664:2048] [node 1] ICP30 unregister session Session# [5:7587241250627544750:2048] VirtualId# [5:7587241250627544748:0]
2025-12-24T02:11:03.431192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241250627544750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.431264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.431288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.446805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:03.446905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241249112102773:2049] [node 2] ICS01 socket: 350 reason# EndOfStream
2025-12-24T02:11:03.446914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241249112102685:2049] [node 2] ICP30 unregister session Session# [3:7587241249112102773:2049] VirtualId# [3:7587241249112102768:0]
2025-12-24T02:11:03.446922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241249112102773:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.446980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.447001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.474778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:03.474866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241247565285057:2049] [node 2] ICS07 socket disconnect 356 reason# EndOfStream
2025-12-24T02:11:03.474871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241247565285057:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.474926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.474945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241247565285057:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:03.475324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241247565284972:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.475337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241247565285057:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.475343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241247565284972:2049] [node 2] ICP30 unregister session Session# [4:7587241247565285057:2049] VirtualId# [4:7587241247565285055:0]
2025-12-24T02:11:03.475356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.475361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241247565284972:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.475363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.478776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:03.478885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241250627544755:2049] [node 2] ICS01 socket: 324 reason# EndOfStream
2025-12-24T02:11:03.478899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241250627544665:2049] [node 2] ICP30 unregister session Session# [5:7587241250627544755:2049] VirtualId# [5:7587241250627544747:0]
2025-12-24T02:11:03.478907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241250627544755:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.478986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.479024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.478852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:03.478956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241248769864249:2049] [node 2] ICS07 socket disconnect 342 reason# EndOfStream
2025-12-24T02:11:03.478963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241248769864249:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.479035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.479063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241248769864249:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:03.479552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241248769864166:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:03.479564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241248769864249:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:03.479571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241248769864166:2049] [node 2] ICP30 unregister session Session# [6:7587241248769864249:2049] VirtualId# [6:7587241248769864247:0]
2025-12-24T02:11:03.479584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.479591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241248769864166:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.479593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14788) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:03.486746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241247565284972:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:03.490756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241248769864166:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:03.508771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:03.508869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241248769864263:2050] [node 3] ICS01 socket: 343 reason# EndOfStream
2025-12-24T02:11:03.508881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241248769864167:2050] [node 3] ICP30 unregister session Session# [6:7587241248769864263:2050] VirtualId# [6:7587241248769864260:0]
2025-12-24T02:11:03.508890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241248769864263:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.508951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.508974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.509724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:03.509819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241250627544764:2050] [node 3] ICS01 socket: 338 reason# EndOfStream
2025-12-24T02:11:03.509827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241250627544666:2050] [node 3] ICP30 unregister session Session# [5:7587241250627544764:2050] VirtualId# [5:7587241250627544753:0]
2025-12-24T02:11:03.509834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241250627544764:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.509890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.509917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.526055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:03.526149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241248769864258:2100] [node 4] ICS01 socket: 341 reason# EndOfStream
2025-12-24T02:11:03.526158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241248769864228:2100] [node 4] ICP30 unregister session Session# [6:7587241248769864258:2100] VirtualId# [6:7587241248769864256:0]
2025-12-24T02:11:03.526164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241248769864258:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.526214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:03.526239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:03.573999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246760348358:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:03.574009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246760348359:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.569868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241249567564222:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.569878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241249567564223:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.569883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241249567564224:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.578842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249602679721:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.578860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249602679722:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.578291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249721169721:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.578313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249721169722:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.579340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241248399447014:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.579348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241248399447015:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.579352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241248399447016:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.590846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241249131984416:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.590862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241249131984417:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.590866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241249131984418:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.620611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246760348423:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:03.620705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246760348424:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:03.622752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246760348425:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16698
2025-12-24T02:11:03.622903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249602679782:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:03.622880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241249131984469:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:03.622967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241249131984471:2097] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.631896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447075:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.632753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249721169787:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:03.632846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249721169788:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:03.632915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249721169789:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:03.640847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241248399447080:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.643241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241246760348433:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.643815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169790:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
2025-12-24T02:11:03.643885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169791:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
2025-12-24T02:11:03.648176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169790:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.648210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241249721169721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.644867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246760348434:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16698
2025-12-24T02:11:03.644893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246760348435:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16698
2025-12-24T02:11:03.648157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348425:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.656957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169795:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.656968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249721169795:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.655247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241249567564288:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:03.655337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241249567564289:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:03.655401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564283:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.655416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564286:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.655428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564287:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.656842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564287:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.656868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241249567564223:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.658973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249602679794:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:03.659063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241249602679795:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:03.664082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249721169795:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.664226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249721169787:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.664242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.664248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249721169787:2108] [node 4] ICP22 created new session: [2:7587241249721169805:2108]
2025-12-24T02:11:03.664254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249721169805:2108] [node 4] ICS09 handshake done sender: [2:7587241249721169795:2114] self: [2:7587241249721169794:0] peer: [4:7587241249567564300:0] socket: 327 qp: -1
2025-12-24T02:11:03.664261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249721169805:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:03.664270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.664405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169804:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.664763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564304:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.666844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241248399447015:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25930344ns) {Configure:104(25929673ns) {ProcessConfigured:135(25679150ns) {StartInitialHandshake:157(25678658ns) {DropHandshakes:470(1314ns) {DropIncomingHandshake:442(139ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:477(1278ns) {SwitchToState:179(746ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.671256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348419:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.671277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246760348419:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.671924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246760348441:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16698
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.675692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984494:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.674846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246760348419:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.674966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246760348358:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.674988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.674998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246760348358:2048] [node 2] ICP22 created new session: [1:7587241246760348443:2048]
2025-12-24T02:11:03.675004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246760348443:2048] [node 2] ICS09 handshake done sender: [1:7587241246760348419:2107] self: [1:7587241246760348418:0] peer: [2:7587241249721169802:0] socket: 306 qp: -1
2025-12-24T02:11:03.675013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246760348443:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:03.675022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.676900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241246760348446:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16698
2025-12-24T02:11:03.677431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241248399447091:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:03.678602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447088:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.678639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447089:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.678653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447090:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.684430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447075:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.684532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447088:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.684567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241248399447014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.684329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241249131984494:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.684375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241249131984416:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.684504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348431:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.684515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246760348431:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.685101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348435:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.690818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348429:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.690833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246760348429:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.695092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679796:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.695311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679797:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.695385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679802:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.696099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241249567564287:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.696256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241249567564223:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.696274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.696280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241249567564223:2049] [node 2] ICP22 created new session: [4:7587241249567564305:2049]
2025-12-24T02:11:03.696287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241249567564305:2049] [node 2] ICS09 handshake done sender: [4:7587241249567564287:2112] self: [4:7587241249567564300:0] peer: [2:7587241249721169794:0] socket: 330 qp: -1
2025-12-24T02:11:03.696294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241249567564305:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.696312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.696492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564283:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.696518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564286:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.698849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679802:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.706890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984496:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.711064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246760348431:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.711362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241249567564224:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.714739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447089:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.714787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241248399447091:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.714880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679788:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.714897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249602679788:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.715514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348446:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.715594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348434:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.715653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246760348424:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.715666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.715673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246760348424:2109] [node 6] ICP22 created new session: [1:7587241246760348449:2109]
2025-12-24T02:11:03.715679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246760348449:2109] [node 6] ICS09 handshake done sender: [1:7587241246760348431:2113] self: [1:7587241246760348430:0] peer: [6:7587241249131984495:0] socket: 309 qp: -1
2025-12-24T02:11:03.715687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246760348449:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:03.715697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.715996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241246760348424:2109] [node 6] ICP18 handshake (actor [1:7587241246760348446:2119]) is too old
2025-12-24T02:11:03.728327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679797:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.728440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241249602679721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.727108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564295:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.727125Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241249567564295:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.729281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241249567564295:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.730936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447090:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.730975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241248399447015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.731094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447099:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.731108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447100:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.732889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249602679788:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.733061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679796:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.733078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249602679782:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.733089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.733101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249602679782:2107] [node 4] ICP22 created new session: [3:7587241249602679806:2107]
2025-12-24T02:11:03.733107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249602679806:2107] [node 4] ICS09 handshake done sender: [3:7587241249602679788:2110] self: [3:7587241249602679787:0] peer: [4:7587241249567564308:0] socket: 315 qp: -1
2025-12-24T02:11:03.733115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249602679806:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:03.733126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.733290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241249602679722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.731340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246760348429:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.732071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241249721169787:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.732129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241249721169721:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241249721169790:2111] held: no
2025-12-24T02:11:03.732192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169797:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.732196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249721169797:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.732793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.737736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169809:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
2025-12-24T02:11:03.737851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169791:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.737891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169793:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.737895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249721169793:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.738568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249721169790:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.743582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241249131984494:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.744367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679810:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.743717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241249131984416:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241249131984494:2115] held: no
2025-12-24T02:11:03.743741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241249131984416:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.743754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.743761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241249131984416:2048] [node 1] ICP22 created new session: [6:7587241249131984500:2048]
2025-12-24T02:11:03.743767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241249131984500:2048] [node 1] ICS09 handshake done sender: [6:7587241249131984494:2115] self: [6:7587241249131984495:0] peer: [1:7587241246760348430:0] socket: 344 qp: -1
2025-12-24T02:11:03.743774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241249131984500:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.743783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.743945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984497:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.743995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984498:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.744032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.743485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348441:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.746835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241249567564288:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.746858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.746869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241249567564288:2109] [node 5] ICP22 created new session: [4:7587241249567564313:2109]
2025-12-24T02:11:03.746876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241249567564313:2109] [node 5] ICS09 handshake done sender: [4:7587241249567564295:2115] self: [4:7587241249567564294:0] peer: [5:7587241248399447098:0] socket: 333 qp: -1
2025-12-24T02:11:03.746883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241249567564313:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:03.746894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.747276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.750929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241249131984498:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.750989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241249131984497:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.752303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241249131984418:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.752321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241249131984471:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.752687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679813:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.752718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679811:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.752738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241249602679812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60502
2025-12-24T02:11:03.751057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241249721169721:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.750934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348427:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.750953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246760348427:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.751478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246760348423:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.751083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.752528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564312:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.751091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249721169721:2048] [node 1] ICP22 created new session: [2:7587241249721169813:2048]
2025-12-24T02:11:03.752549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241249567564286:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.752879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564297:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.752885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241249567564297:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.753246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241249567564224:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.753256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.753262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241249567564224:2050] [node 3] ICP22 created new session: [4:7587241249567564316:2050]
2025-12-24T02:11:03.753268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241249567564316:2050] [node 3] ICS09 handshake done sender: [4:7587241249567564286:2111] self: [4:7587241249567564308:0] peer: [3:7587241249602679787:0] socket: 320 qp: -1
2025-12-24T02:11:03.753274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241249567564316:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.753283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.754112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241249567564297:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.754174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241249567564289:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.754187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.754193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241249567564289:2110] [node 6] ICP22 created new session: [4:7587241249567564319:2110]
2025-12-24T02:11:03.754198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241249567564319:2110] [node 6] ICS09 handshake done sender: [4:7587241249567564297:2116] self: [4:7587241249567564296:0] peer: [6:7587241249131984504:0] socket: 356 qp: -1
2025-12-24T02:11:03.754203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241249567564319:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:03.754209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.751493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.751499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246760348423:2108] [node 5] ICP22 created new session: [1:7587241246760348454:2108]
2025-12-24T02:11:03.751506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246760348454:2108] [node 5] ICS09 handshake done sender: [1:7587241246760348429:2112] self: [1:7587241246760348428:0] peer: [5:7587241248399447097:0] socket: 329 qp: -1
2025-12-24T02:11:03.751512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246760348454:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:03.751522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.754750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241248399447088:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.754818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241248399447014:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.754829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.754836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241248399447014:2048] [node 1] ICP22 created new session: [5:7587241248399447103:2048]
2025-12-24T02:11:03.754842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241248399447103:2048] [node 1] ICS09 handshake done sender: [5:7587241248399447088:2115] self: [5:7587241248399447097:0] peer: [1:7587241246760348428:0] socket: 347 qp: -1
2025-12-24T02:11:03.754849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241248399447103:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.754856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.751105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249721169813:2048] [node 1] ICS09 handshake done sender: [2:7587241249721169790:2111] self: [2:7587241249721169802:0] peer: [1:7587241246760348418:0] socket: 321 qp: -1
2025-12-24T02:11:03.751117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249721169813:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.751125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.751431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169809:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.751497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249721169797:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.751634Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249721169793:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.751703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249721169788:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.751711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.751718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249721169788:2109] [node 5] ICP22 created new session: [2:7587241249721169816:2109]
2025-12-24T02:11:03.751723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249721169816:2109] [node 5] ICS09 handshake done sender: [2:7587241249721169797:2115] self: [2:7587241249721169796:0] peer: [5:7587241248399447101:0] socket: 339 qp: -1
2025-12-24T02:11:03.751728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249721169816:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:03.751735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.751838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249721169722:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.751848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.751852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249721169722:2049] [node 3] ICP22 created new session: [2:7587241249721169818:2049]
2025-12-24T02:11:03.751856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249721169818:2049] [node 3] ICS09 handshake done sender: [2:7587241249721169793:2113] self: [2:7587241249721169792:0] peer: [3:7587241249602679809:0] socket: 326 qp: -1
2025-12-24T02:11:03.751859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249721169818:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:03.751863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.755667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241248399447089:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.755718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241248399447091:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.755728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.755733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241248399447091:2111] [node 4] ICP22 created new session: [5:7587241248399447106:2111]
2025-12-24T02:11:03.755738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241248399447106:2111] [node 4] ICS09 handshake done sender: [5:7587241248399447089:2116] self: [5:7587241248399447098:0] peer: [4:7587241249567564294:0] socket: 348 qp: -1
2025-12-24T02:11:03.755743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241248399447106:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:03.755749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.755986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447102:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.756211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241248399447014:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.756246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241248399447090:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.756294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241248399447015:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.756302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.756306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241248399447015:2049] [node 2] ICP22 created new session: [5:7587241248399447108:2049]
2025-12-24T02:11:03.756310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241248399447108:2049] [node 2] ICS09 handshake done sender: [5:7587241248399447090:2117] self: [5:7587241248399447101:0] peer: [2:7587241249721169796:0] socket: 349 qp: -1
2025-12-24T02:11:03.756315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241248399447108:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.756321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.760199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241249721169788:2109] [node 5] ICP18 handshake (actor [2:7587241249721169809:2119]) is too old
2025-12-24T02:11:03.759496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679799:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.759514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249602679799:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.761631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249602679796:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.763989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241249602679722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10229) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241249602679796:2113] held: no
2025-12-24T02:11:03.764021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241249602679722:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.764034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.764041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249602679722:2049] [node 2] ICP22 created new session: [3:7587241249602679815:2049]
2025-12-24T02:11:03.764048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249602679815:2049] [node 2] ICS09 handshake done sender: [3:7587241249602679796:2113] self: [3:7587241249602679809:0] peer: [2:7587241249721169792:0] socket: 336 qp: -1
2025-12-24T02:11:03.764060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249602679815:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.764067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.764647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249602679797:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.763761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246760348427:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.764813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241249602679721:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.764826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.764835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249602679721:2048] [node 1] ICP22 created new session: [3:7587241249602679818:2048]
2025-12-24T02:11:03.764841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249602679818:2048] [node 1] ICS09 handshake done sender: [3:7587241249602679797:2114] self: [3:7587241249602679805:0] peer: [1:7587241246760348426:0] socket: 337 qp: -1
2025-12-24T02:11:03.764847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249602679818:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.764854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.766317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447110:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.767134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249602679799:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.770902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249602679794:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.770930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.770940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249602679794:2111] [node 6] ICP22 created new session: [3:7587241249602679820:2111]
2025-12-24T02:11:03.770947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249602679820:2111] [node 6] ICS09 handshake done sender: [3:7587241249602679799:2115] self: [3:7587241249602679798:0] peer: [6:7587241249131984503:0] socket: 335 qp: -1
2025-12-24T02:11:03.770957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249602679820:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:03.770966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:03.770946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241249131984471:2097] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241249131984497:2117] held: no
2025-12-24T02:11:03.771019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241249131984499:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.771248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.771340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679810:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.771368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241249602679794:2111] [node 6] ICP18 handshake (actor [3:7587241249602679810:2118]) is too old
2025-12-24T02:11:03.771563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246760348359:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.771588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.771595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246760348359:2049] [node 3] ICP22 created new session: [1:7587241246760348458:2049]
2025-12-24T02:11:03.771607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246760348458:2049] [node 3] ICS09 handshake done sender: [1:7587241246760348427:2111] self: [1:7587241246760348426:0] peer: [3:7587241249602679805:0] socket: 328 qp: -1
2025-12-24T02:11:03.771616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246760348458:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:03.771625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.774539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447110:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.774622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241248399447016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.774847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241249602679801:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.774858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241249602679801:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.775640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241249602679801:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.775741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241249602679795:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.775760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.775767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241249602679795:2112] [node 5] ICP22 created new session: [3:7587241249602679823:2112]
2025-12-24T02:11:03.775772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241249602679823:2112] [node 5] ICS09 handshake done sender: [3:7587241249602679801:2116] self: [3:7587241249602679800:0] peer: [5:7587241248399447111:0] socket: 331 qp: -1
2025-12-24T02:11:03.775779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241249602679823:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:03.775787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.774752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241249567564312:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.774786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241249567564222:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.778921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984506:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.779012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241249131984417:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.779112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984507:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.779180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241246760348439:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.779192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241246760348439:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.782795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249721169799:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.782815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249721169799:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.783749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241249131984498:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.784120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249721169799:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.784201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249721169789:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.784210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.784216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249721169789:2110] [node 6] ICP22 created new session: [2:7587241249721169821:2110]
2025-12-24T02:11:03.784221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241249721169821:2110] [node 6] ICS09 handshake done sender: [2:7587241249721169799:2116] self: [2:7587241249721169798:0] peer: [6:7587241249131984509:0] socket: 363 qp: -1
2025-12-24T02:11:03.784227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241249721169821:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:03.784235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.783874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241246760348439:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.783978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241246760348433:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.783994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.784002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241246760348433:2114] [node 4] ICP22 created new session: [1:7587241246760348461:2114]
2025-12-24T02:11:03.784009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241246760348461:2114] [node 4] ICS09 handshake done sender: [1:7587241246760348439:2117] self: [1:7587241246760348438:0] peer: [4:7587241249567564321:0] socket: 361 qp: -1
2025-12-24T02:11:03.784017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241246760348461:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:03.784029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:03.785161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241248399447016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241248399447110:2122] held: no
2025-12-24T02:11:03.785282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241248399447112:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28769
2025-12-24T02:11:03.791010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241249131984506:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.791030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241249131984418:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.791045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.791054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241249131984418:2050] [node 3] ICP22 created new session: [6:7587241249131984511:2050]
2025-12-24T02:11:03.791061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241249131984511:2050] [node 3] ICS09 handshake done sender: [6:7587241249131984498:2118] self: [6:7587241249131984503:0] peer: [3:7587241249602679798:0] socket: 365 qp: -1
2025-12-24T02:11:03.791070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241249131984511:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.791080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.792312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241249721169823:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.793272Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241249567564222:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14913) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241249567564312:2120] held: no
2025-12-24T02:11:03.793407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241249567564322:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50459
2025-12-24T02:11:03.793782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241249567564312:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:03.793830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241249567564222:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.793837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:03.793842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241249567564222:2048] [node 1] ICP22 created new session: [4:7587241249567564323:2048]
2025-12-24T02:11:03.793847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241249567564323:2048] [node 1] ICS09 handshake done sender: [4:7587241249567564312:2120] self: [4:7587241249567564321:0] peer: [1:7587241246760348438:0] socket: 368 qp: -1
2025-12-24T02:11:03.793851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241249567564323:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:03.793857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:03.793132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241248399447110:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:03.793486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241248399447016:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.793498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:03.793504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241248399447016:2050] [node 3] ICP22 created new session: [5:7587241248399447113:2050]
2025-12-24T02:11:03.793511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241248399447113:2050] [node 3] ICS09 handshake done sender: [5:7587241248399447110:2122] self: [5:7587241248399447111:0] peer: [3:7587241249602679800:0] socket: 313 qp: -1
2025-12-24T02:11:03.793518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241248399447113:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:03.793526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:03.794983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241249131984497:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:03.802820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241249131984471:2097] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.802846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:03.802858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241249131984471:2097] [node 4] ICP22 created new session: [6:7587241249131984513:2097]
2025-12-24T02:11:03.802865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241249131984513:2097] [node 4] ICS09 handshake done sender: [6:7587241249131984497:2117] self: [6:7587241249131984504:0] peer: [4:7587241249567564296:0] socket: 345 qp: -1
2025-12-24T02:11:03.802878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241249131984513:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:03.802890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.810801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241249131984469:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:03.810896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984510:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.815048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241249131984499:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:03.822803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241249131984417:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.822833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:03.822844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241249131984417:2049] [node 2] ICP22 created new session: [6:7587241249131984516:2049]
2025-12-24T02:11:03.822852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241249131984516:2049] [node 2] ICS09 handshake done sender: [6:7587241249131984499:2119] self: [6:7587241249131984509:0] peer: [2:7587241249721169798:0] socket: 366 qp: -1
2025-12-24T02:11:03.822863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241249131984516:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:03.822874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:03.830882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241249131984418:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:03.835143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241248399447087:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:03.835159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241248399447087:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:03.835938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241248399447087:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:03.836026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241248399447080:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:03.836043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:03.836053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241248399447080:2110] [node 6] ICP22 created new session: [5:7587241248399447116:2110]
2025-12-24T02:11:03.836060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241248399447116:2110] [node 6] ICS09 handshake done sender: [5:7587241248399447087:2114] self: [5:7587241248399447086:0] peer: [6:7587241249131984515:0] socket: 317 qp: -1
2025-12-24T02:11:03.836067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241248399447116:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:03.836076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.839804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241249131984469:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241249131984506:2121] held: no
2025-12-24T02:11:03.839881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241249131984518:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49719
2025-12-24T02:11:03.840039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241249131984506:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:03.840097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241249131984469:2095] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:03.840111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:03.840118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241249131984469:2095] [node 5] ICP22 created new session: [6:7587241249131984519:2095]
2025-12-24T02:11:03.840125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241249131984519:2095] [node 5] ICS09 handshake done sender: [6:7587241249131984506:2121] self: [6:7587241249131984515:0] peer: [5:7587241248399447086:0] socket: 314 qp: -1
2025-12-24T02:11:03.840135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241249131984519:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:03.840146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:03.843125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:03.843229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241249721169813:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:03.843236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241249721169721:2048] [node 1] ICP30 unregister session Session# [2:7587241249721169813:2048] VirtualId# [2:7587241249721169802:0]
2025-12-24T02:11:03.843244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241249721169813:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.843288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.843311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.846661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:03.846289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:03.846747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241249602679818:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-12-24T02:11:03.846380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241248399447103:2048] [node 1] ICS01 socket: 347 reason# EndOfStream
2025-12-24T02:11:03.846388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241248399447014:2048] [node 1] ICP30 unregister session Session# [5:7587241248399447103:2048] VirtualId# [5:7587241248399447097:0]
2025-12-24T02:11:03.846395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241248399447103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.846754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241249602679721:2048] [node 1] ICP30 unregister session Session# [3:7587241249602679818:2048] VirtualId# [3:7587241249602679805:0]
2025-12-24T02:11:03.846462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.846760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241249602679818:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.846493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.846806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.846831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.848788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:03.848871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241249131984500:2048] [node 1] ICS01 socket: 344 reason# EndOfStream
2025-12-24T02:11:03.848878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241249131984416:2048] [node 1] ICP30 unregister session Session# [6:7587241249131984500:2048] VirtualId# [6:7587241249131984495:0]
2025-12-24T02:11:03.848884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241249131984500:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.848939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.848965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.850420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:03.850496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241249567564323:2048] [node 1] ICS01 socket: 368 reason# EndOfStream
2025-12-24T02:11:03.850502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241249567564222:2048] [node 1] ICP30 unregister session Session# [4:7587241249567564323:2048] VirtualId# [4:7587241249567564321:0]
2025-12-24T02:11:03.850507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241249567564323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.850561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:03.850585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:03.851003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:03.851083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241249602679815:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-12-24T02:11:03.851089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241249602679722:2049] [node 2] ICP30 unregister session Session# [3:7587241249602679815:2049] VirtualId# [3:7587241249602679809:0]
2025-12-24T02:11:03.851095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241249602679815:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.851148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.851170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.852780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:03.852855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241249131984516:2049] [node 2] ICS01 socket: 366 reason# EndOfStream
2025-12-24T02:11:03.852860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241249131984417:2049] [node 2] ICP30 unregister session Session# [6:7587241249131984516:2049] VirtualId# [6:7587241249131984509:0]
2025-12-24T02:11:03.852864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241249131984516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.852892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.852914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.853041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:03.853111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241249567564305:2049] [node 2] ICS01 socket: 330 reason# EndOfStream
2025-12-24T02:11:03.853115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241249567564223:2049] [node 2] ICP30 unregister session Session# [4:7587241249567564305:2049] VirtualId# [4:7587241249567564300:0]
2025-12-24T02:11:03.853119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241249567564305:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.852587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:03.852666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241248399447108:2049] [node 2] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:03.852673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241248399447015:2049] [node 2] ICP30 unregister session Session# [5:7587241248399447108:2049] VirtualId# [5:7587241248399447101:0]
2025-12-24T02:11:03.852678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241248399447108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.852726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.852750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.853150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:03.853169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:03.865579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:03.865667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241249131984511:2050] [node 3] ICS01 socket: 365 reason# EndOfStream
2025-12-24T02:11:03.865672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241249131984418:2050] [node 3] ICP30 unregister session Session# [6:7587241249131984511:2050] VirtualId# [6:7587241249131984503:0]
2025-12-24T02:11:03.865678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241249131984511:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.865710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.865733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.865321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:03.865423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241248399447113:2050] [node 3] ICS01 socket: 313 reason# EndOfStream
2025-12-24T02:11:03.865432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241248399447016:2050] [node 3] ICP30 unregister session Session# [5:7587241248399447113:2050] VirtualId# [5:7587241248399447111:0]
2025-12-24T02:11:03.865439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241248399447113:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.865508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:03.865534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:03.881381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:03.881484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241249131984513:2097] [node 4] ICS01 socket: 345 reason# EndOfStream
2025-12-24T02:11:03.881495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241249131984471:2097] [node 4] ICP30 unregister session Session# [6:7587241249131984513:2097] VirtualId# [6:7587241249131984504:0]
2025-12-24T02:11:03.881504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241249131984513:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:03.881578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:03.881605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:03.910139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241249309578222:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:03.910154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241249309578223:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.918498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241248912245335:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.918506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241248912245336:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.916787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249765931485:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.916803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249765931486:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:03.926800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241249446546382:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.926815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241249446546383:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.926820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241249446546384:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.922038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247377371656:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.922049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247377371657:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.922058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247377371658:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.929630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241247022933456:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:03.929640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241247022933457:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:03.929646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241247022933458:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:03.971540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241247022933501:2090] [node 5] ICP01 ready to work
2025-12-24T02:11:03.974938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241249446546444:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:03.974958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247377371718:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:03.984053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241248912245401:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:03.984156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241248912245402:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:03.986725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241247022933507:2094] [node 4] ICP01 ready to work
2025-12-24T02:11:03.986428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241248912245396:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:03.986987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241249309578286:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:03.987137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241249309578287:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:03.987238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241249309578288:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.984850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249765931551:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:03.984944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249765931552:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:03.985018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241249765931553:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:03.991080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241249446546384:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16079377ns) {Configure:104(16078353ns) {ProcessConfigured:135(15558988ns) {StartInitialHandshake:157(15558473ns) {DropHandshakes:470(592ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1247ns) {SwitchToState:179(625ns)}}}}}
2025-12-24T02:11:03.991223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241249446546449:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:03.992123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241249446546452:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:03.993614Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241249446546444:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17822276ns) {Configure:104(17821585ns) {ProcessConfigured:135(17558462ns) {StartInitialHandshake:157(17558122ns) {DropHandshakes:470(550ns) {DropIncomingHandshake:442(93ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(929ns) {SwitchToState:179(483ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:03.994788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241247377371658:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19771647ns) {Configure:104(19771055ns) {ProcessConfigured:135(15112161ns) {StartInitialHandshake:157(15111293ns) {DropHandshakes:470(688ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(129ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:477(1067ns) {SwitchToState:179(651ns)}}}}}
2025-12-24T02:11:03.994949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241247377371723:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:03.996111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241247377371718:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21081112ns) {Configure:104(21080015ns) {ProcessConfigured:135(16224831ns) {StartInitialHandshake:157(16224081ns) {DropHandshakes:470(409ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(38ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(678ns) {SwitchToState:179(41ns)}}}}}
2025-12-24T02:11:03.998152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241247377371724:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.002881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241248912245402:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16376387ns) {Configure:104(16375602ns) {ProcessConfigured:135(16064222ns) {StartInitialHandshake:157(16063672ns) {DropHandshakes:470(539ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:477(1714ns) {SwitchToState:179(734ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.003802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241247022933507:2094] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16999871ns) {Configure:104(16999172ns) {ProcessConfigured:135(16710864ns) {StartInitialHandshake:157(16710567ns) {DropHandshakes:470(463ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1476ns) {SwitchToState:179(249ns)}}}}}
2025-12-24T02:11:04.004087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900827:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.004102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.004288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339031:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.003592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212707:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
2025-12-24T02:11:04.003611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212708:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
2025-12-24T02:11:04.005681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212711:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.008555Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241248912245336:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24045172ns) {Configure:104(24044512ns) {ProcessConfigured:135(23846619ns) {StartInitialHandshake:157(23846365ns) {DropHandshakes:470(273ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(720ns) {SwitchToState:179(436ns)}}}}}
2025-12-24T02:11:04.006796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241249309578223:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19503192ns) {Configure:104(19502540ns) {ProcessConfigured:135(19216161ns) {StartInitialHandshake:157(19215340ns) {DropHandshakes:470(449ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(106ns) PrepareNewSessionHandshake:477(1253ns) {SwitchToState:179(692ns)}}}}}
2025-12-24T02:11:04.007385Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241249309578222:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(35932614ns) {Configure:104(35931481ns) {ProcessConfigured:135(35479959ns) {StartInitialHandshake:157(35479754ns) {DropHandshakes:470(853ns) {DropIncomingHandshake:442(116ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(700ns) {SwitchToState:179(199ns)}}}}}
2025-12-24T02:11:04.008263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241253604545591:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4100
2025-12-24T02:11:04.008277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241253604545592:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4100
2025-12-24T02:11:04.008289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241253604545593:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4100
2025-12-24T02:11:04.008301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241253604545594:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4100
2025-12-24T02:11:04.008363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241253604545595:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4100
2025-12-24T02:11:04.011769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545594:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.014062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251672339031:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.014104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247377371658:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.020415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513757:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.027157Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241249765931486:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(53047825ns) {Configure:104(53046568ns) {ProcessConfigured:135(52755456ns) {StartInitialHandshake:157(52754901ns) {DropHandshakes:470(431ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(2233ns) {SwitchToState:179(938ns)}}}}}
2025-12-24T02:11:04.029629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898860:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.029673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.033850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898860:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.032418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241248912245404:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.032443Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241248912245404:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.037128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247377371724:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.037181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247377371657:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.035546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241247022933457:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51891442ns) {Configure:104(51890928ns) {ProcessConfigured:135(51684668ns) {StartInitialHandshake:157(51683519ns) {DropHandshakes:470(210ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(138ns) PrepareNewSessionHandshake:477(1088ns) {SwitchToState:179(559ns)}}}}}
2025-12-24T02:11:04.036809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900832:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.036774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212713:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
2025-12-24T02:11:04.040399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241253741513757:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251317900827:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241247022933501:2090] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.040696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241249446546452:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241249446546383:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.040816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251317900828:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241247022933458:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.041147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249765931555:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.041158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249765931555:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.041251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249765931557:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.041258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249765931557:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.041277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241249446546454:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.041284Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241249446546454:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.040663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212708:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212707:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.040837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212711:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.041325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212703:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.041328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241253207212703:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.036289Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241249309578287:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29429498ns) {Configure:104(29428742ns) {ProcessConfigured:135(29091242ns) {StartInitialHandshake:157(29090889ns) {DropHandshakes:470(1390ns) {DropIncomingHandshake:442(582ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(1237ns) {SwitchToState:179(179ns)}}}}}
2025-12-24T02:11:04.042826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545593:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.042969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251317900832:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.043005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241247022933456:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.042952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545592:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.042986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545591:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.043829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898861:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.043942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241249765931485:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.044018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249765931555:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.044613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249765931557:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.044702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249765931551:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.044719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.044726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249765931551:2106] [node 4] ICP22 created new session: [2:7587241254060898866:2106]
2025-12-24T02:11:04.044731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241254060898866:2106] [node 4] ICS09 handshake done sender: [2:7587241249765931555:2111] self: [2:7587241249765931554:0] peer: [4:7587241251672339033:0] socket: 313 qp: -1
2025-12-24T02:11:04.044736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241254060898866:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:04.044744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.045032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241249446546454:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.045399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.045415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.045535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251317900828:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.045582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241247022933458:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.045598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.045608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241247022933458:2050] [node 3] ICP22 created new session: [6:7587241251317900838:2050]
2025-12-24T02:11:04.045614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251317900838:2050] [node 3] ICS09 handshake done sender: [6:7587241251317900828:2115] self: [6:7587241251317900834:0] peer: [3:7587241248912245406:0] socket: 332 qp: -1
2025-12-24T02:11:04.045624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251317900838:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.045641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.046443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251317900827:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.043782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241248912245404:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.045367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253207212703:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.046507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241248912245402:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.044890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545590:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.046519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.046524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241248912245402:2109] [node 6] ICP22 created new session: [3:7587241253207212719:2109]
2025-12-24T02:11:04.046529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253207212719:2109] [node 6] ICS09 handshake done sender: [3:7587241253207212703:2112] self: [3:7587241248912245406:0] peer: [6:7587241251317900834:0] socket: 330 qp: -1
2025-12-24T02:11:04.044902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241253604545590:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.046534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253207212719:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.046540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.043939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339034:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.046816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251672339031:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.046871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241247022933501:2090] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.046885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.046889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241247022933501:2090] [node 5] ICP22 created new session: [6:7587241251317900841:2090]
2025-12-24T02:11:04.046894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251317900841:2090] [node 5] ICS09 handshake done sender: [6:7587241251317900827:2114] self: [6:7587241251317900833:0] peer: [5:7587241249446546453:0] socket: 331 qp: -1
2025-12-24T02:11:04.046898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251317900841:2090] [node 5] ICS10 traffic start
2025-12-24T02:11:04.046903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.047189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241248912245401:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.047225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.047237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241248912245401:2108] [node 4] ICP22 created new session: [3:7587241253207212721:2108]
2025-12-24T02:11:04.047247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253207212721:2108] [node 4] ICS09 handshake done sender: [3:7587241248912245404:2111] self: [3:7587241248912245403:0] peer: [4:7587241251672339032:0] socket: 329 qp: -1
2025-12-24T02:11:04.047254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253207212721:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:04.047260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.050971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545595:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.051006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545588:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.051011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241253604545588:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.054839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241253604545588:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.059201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249765931552:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.059230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.059237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249765931552:2107] [node 5] ICP22 created new session: [2:7587241254060898868:2107]
2025-12-24T02:11:04.059244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241254060898868:2107] [node 5] ICS09 handshake done sender: [2:7587241249765931557:2112] self: [2:7587241249765931556:0] peer: [5:7587241253741513759:0] socket: 314 qp: -1
2025-12-24T02:11:04.059250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241254060898868:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:04.059258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.059467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241249446546449:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.059487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.059492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241249446546449:2109] [node 6] ICP22 created new session: [5:7587241253741513761:2109]
2025-12-24T02:11:04.059497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241253741513761:2109] [node 6] ICS09 handshake done sender: [5:7587241249446546454:2113] self: [5:7587241249446546453:0] peer: [6:7587241251317900833:0] socket: 319 qp: -1
2025-12-24T02:11:04.059501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241253741513761:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.059525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.059168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241249309578222:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.059195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.059211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241249309578222:2048] [node 2] ICP22 created new session: [1:7587241253604545608:2048]
2025-12-24T02:11:04.059219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241253604545608:2048] [node 2] ICS09 handshake done sender: [1:7587241253604545588:2111] self: [1:7587241249309578282:0] peer: [2:7587241254060898864:0] socket: 345 qp: -1
2025-12-24T02:11:04.059227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241253604545608:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:04.059235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.059889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241253604545590:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.059964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241249309578288:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.059978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.059990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241249309578288:2109] [node 6] ICP22 created new session: [1:7587241253604545612:2109]
2025-12-24T02:11:04.059997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241253604545612:2109] [node 6] ICS09 handshake done sender: [1:7587241253604545590:2112] self: [1:7587241253604545589:0] peer: [6:7587241251317900835:0] socket: 347 qp: -1
2025-12-24T02:11:04.060002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241253604545612:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.060009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.060655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900844:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.062884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247377371658:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.062909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.062917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247377371658:2050] [node 3] ICP22 created new session: [4:7587241251672339036:2050]
2025-12-24T02:11:04.062924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251672339036:2050] [node 3] ICS09 handshake done sender: [4:7587241251672339031:2116] self: [4:7587241251672339032:0] peer: [3:7587241248912245403:0] socket: 336 qp: -1
2025-12-24T02:11:04.062931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251672339036:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.062937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.066306Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241248912245401:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(19081904ns) {HandleHandshakeStatus:329(19081293ns) {DropIncomingHandshake:442(19065ns) DropOutgoingHandshake:457(177ns) SwitchToState:179(226ns) ProcessPendingSessionEvents:495(10976544ns) {ForwardSessionEventToSession:612(10963452ns) ForwardSessionEventToSession:612(2383ns) ForwardSessionEventToSession:612(815ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1167ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(526ns) ForwardSessionEventToSession:612(1424ns)}}}
2025-12-24T02:11:04.066492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212713:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.066531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241248912245335:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.066846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241247377371657:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30129) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241247377371724:2111] held: no
2025-12-24T02:11:04.066913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.066719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898872:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.066784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.066835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241249765931485:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241254060898861:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.067480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241247022933456:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241251317900832:2117] held: no
2025-12-24T02:11:04.067564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251317900832:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.067638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900845:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.067762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241247022933456:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.067782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.067789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241247022933456:2048] [node 1] ICP22 created new session: [6:7587241251317900846:2048]
2025-12-24T02:11:04.067794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251317900846:2048] [node 1] ICS09 handshake done sender: [6:7587241251317900832:2117] self: [6:7587241251317900835:0] peer: [1:7587241253604545589:0] socket: 353 qp: -1
2025-12-24T02:11:04.067801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251317900846:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.067807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.067520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247377371724:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.070879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513764:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.067671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898870:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.067705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241254060898871:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45429
2025-12-24T02:11:04.067794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898872:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.067815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898873:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.067833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241254060898861:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.071093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241249446546452:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.071152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241249446546383:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.071168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.071175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241249446546383:2049] [node 2] ICP22 created new session: [5:7587241253741513765:2049]
2025-12-24T02:11:04.071185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241253741513765:2049] [node 2] ICS09 handshake done sender: [5:7587241249446546452:2112] self: [5:7587241253741513759:0] peer: [2:7587241249765931556:0] socket: 316 qp: -1
2025-12-24T02:11:04.071192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241253741513765:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.071216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.070825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339039:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.070895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247377371657:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.070907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.070914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247377371657:2049] [node 2] ICP22 created new session: [4:7587241251672339040:2049]
2025-12-24T02:11:04.070921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251672339040:2049] [node 2] ICS09 handshake done sender: [4:7587241247377371724:2111] self: [4:7587241251672339033:0] peer: [2:7587241249765931554:0] socket: 321 qp: -1
2025-12-24T02:11:04.070928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251672339040:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.070937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.071170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.071672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251672339038:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.072073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900848:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.070764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.077514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513767:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.077569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513768:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.077598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513769:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.076448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241248912245335:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241253207212713:2118] held: no
2025-12-24T02:11:04.076531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212727:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
2025-12-24T02:11:04.076631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253207212713:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.076673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241248912245335:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.078565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251672339039:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.074726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545586:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.074742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241253604545586:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.075447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241253604545586:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.075512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241249309578223:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.075526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.075532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241249309578223:2049] [node 3] ICP22 created new session: [1:7587241253604545616:2049]
2025-12-24T02:11:04.075538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241253604545616:2049] [node 3] ICS09 handshake done sender: [1:7587241253604545586:2110] self: [1:7587241249309578289:0] peer: [3:7587241253207212725:0] socket: 346 qp: -1
2025-12-24T02:11:04.075543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241253604545616:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.075548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.078616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241247377371656:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.078857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898871:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.080136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241249765931485:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.080151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.080157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249765931485:2048] [node 1] ICP22 created new session: [2:7587241254060898877:2048]
2025-12-24T02:11:04.080164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241254060898877:2048] [node 1] ICS09 handshake done sender: [2:7587241254060898861:2116] self: [2:7587241254060898864:0] peer: [1:7587241249309578282:0] socket: 352 qp: -1
2025-12-24T02:11:04.080172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241254060898877:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.080181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.080300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241249765931552:2107] [node 5] ICP18 handshake (actor [2:7587241254060898872:2119]) is too old
2025-12-24T02:11:04.076680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.076686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241248912245335:2048] [node 1] ICP22 created new session: [3:7587241253207212728:2048]
2025-12-24T02:11:04.076691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253207212728:2048] [node 1] ICS09 handshake done sender: [3:7587241253207212713:2118] self: [3:7587241253207212725:0] peer: [1:7587241249309578289:0] socket: 348 qp: -1
2025-12-24T02:11:04.076696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253207212728:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.076700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.083098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241253741513767:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.083150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241249446546382:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.082806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545597:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.082819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241253604545597:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.083591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251317900848:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.083629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241247022933507:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.083745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241253604545601:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.083753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241253604545601:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.084172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241253604545597:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.084249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241249309578286:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.084262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.084268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241249309578286:2107] [node 4] ICP22 created new session: [1:7587241253604545620:2107]
2025-12-24T02:11:04.084274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241253604545620:2107] [node 4] ICS09 handshake done sender: [1:7587241253604545597:2118] self: [1:7587241253604545596:0] peer: [4:7587241251672339045:0] socket: 366 qp: -1
2025-12-24T02:11:04.084281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241253604545620:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.084288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.084691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241253604545601:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.084746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241249309578287:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.084754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.084759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241249309578287:2108] [node 5] ICP22 created new session: [1:7587241253604545622:2108]
2025-12-24T02:11:04.084763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241253604545622:2108] [node 5] ICS09 handshake done sender: [1:7587241253604545601:2119] self: [1:7587241253604545587:0] peer: [5:7587241253741513771:0] socket: 367 qp: -1
2025-12-24T02:11:04.084767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241253604545622:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:04.084773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.086308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241253741513768:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.086826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241249446546382:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241253741513767:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.087321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513772:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.090813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251317900845:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.090861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241247022933457:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.091063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241249765931559:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.091077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241249765931559:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.091881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241249446546384:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.092042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241253741513767:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.091735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241249765931559:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.091818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249765931553:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.091848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.091857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249765931553:2108] [node 6] ICP22 created new session: [2:7587241254060898881:2108]
2025-12-24T02:11:04.091862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241254060898881:2108] [node 6] ICS09 handshake done sender: [2:7587241249765931559:2113] self: [2:7587241249765931558:0] peer: [6:7587241251317900850:0] socket: 312 qp: -1
2025-12-24T02:11:04.091868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241254060898881:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:04.091877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.092424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241249446546382:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.092441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.092448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241249446546382:2048] [node 1] ICP22 created new session: [5:7587241253741513774:2048]
2025-12-24T02:11:04.092454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241253741513774:2048] [node 1] ICS09 handshake done sender: [5:7587241253741513767:2118] self: [5:7587241253741513771:0] peer: [1:7587241253604545587:0] socket: 372 qp: -1
2025-12-24T02:11:04.092460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241253741513774:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.092467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.090757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339046:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
2025-12-24T02:11:04.091685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247377371731:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.091693Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241247377371731:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.093645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241247377371656:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1040) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241251672339039:2120] held: no
2025-12-24T02:11:04.093866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251672339047:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20314
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.094793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241247022933457:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30129) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241251317900845:2121] held: no
2025-12-24T02:11:04.094891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900851:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.095605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241254060898859:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.095616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241254060898859:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.096305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241254060898859:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.096373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241249765931486:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.096389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.096394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241249765931486:2049] [node 3] ICP22 created new session: [2:7587241254060898884:2049]
2025-12-24T02:11:04.096399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241254060898884:2049] [node 3] ICS09 handshake done sender: [2:7587241254060898859:2114] self: [2:7587241249765931548:0] peer: [3:7587241253207212730:0] socket: 324 qp: -1
2025-12-24T02:11:04.096403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241254060898884:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.096410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.095052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212726:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.095093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253207212706:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.095095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241253207212706:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.095488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241248912245336:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.094893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247377371731:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.095697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251672339046:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.095725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241247377371718:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.095738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.095745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247377371718:2105] [node 6] ICP22 created new session: [4:7587241251672339049:2105]
2025-12-24T02:11:04.095750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251672339049:2105] [node 6] ICS09 handshake done sender: [4:7587241247377371731:2115] self: [4:7587241247377371720:0] peer: [6:7587241251317900849:0] socket: 374 qp: -1
2025-12-24T02:11:04.095757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251672339049:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:04.095764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.098037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251672339039:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.098784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241253741513769:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.098834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241249446546444:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.098846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251317900852:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57637
2025-12-24T02:11:04.098921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251317900845:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.100124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241247377371656:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.100144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.100153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247377371656:2048] [node 1] ICP22 created new session: [4:7587241251672339051:2048]
2025-12-24T02:11:04.100159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251672339051:2048] [node 1] ICS09 handshake done sender: [4:7587241251672339039:2120] self: [4:7587241251672339045:0] peer: [1:7587241253604545596:0] socket: 370 qp: -1
2025-12-24T02:11:04.100167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251672339051:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.100175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.100311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241247377371730:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.100315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241247377371730:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.101206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241247377371730:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.101263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241247377371723:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.101274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.101279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241247377371723:2109] [node 5] ICP22 created new session: [4:7587241251672339055:2109]
2025-12-24T02:11:04.101285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251672339055:2109] [node 5] ICS09 handshake done sender: [4:7587241247377371730:2114] self: [4:7587241247377371729:0] peer: [5:7587241253741513776:0] socket: 373 qp: -1
2025-12-24T02:11:04.101290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251672339055:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:04.101296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.101055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241248912245336:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30129) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241253207212726:2119] held: no
2025-12-24T02:11:04.101127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253207212731:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30482
2025-12-24T02:11:04.101594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253207212726:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.101638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241248912245336:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.101645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.101652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241248912245336:2049] [node 2] ICP22 created new session: [3:7587241253207212732:2049]
2025-12-24T02:11:04.101658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253207212732:2049] [node 2] ICS09 handshake done sender: [3:7587241253207212726:2119] self: [3:7587241253207212730:0] peer: [2:7587241249765931548:0] socket: 371 qp: -1
2025-12-24T02:11:04.101662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253207212732:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.101667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.103266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253207212706:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.106450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241248912245396:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.106475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.106482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241248912245396:2106] [node 5] ICP22 created new session: [3:7587241253207212735:2106]
2025-12-24T02:11:04.106488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253207212735:2106] [node 5] ICS09 handshake done sender: [3:7587241253207212706:2113] self: [3:7587241253207212705:0] peer: [5:7587241253741513773:0] socket: 327 qp: -1
2025-12-24T02:11:04.106495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253207212735:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:04.106504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.106766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241247022933457:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.106783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.106789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241247022933457:2049] [node 2] ICP22 created new session: [6:7587241251317900853:2049]
2025-12-24T02:11:04.106794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251317900853:2049] [node 2] ICS09 handshake done sender: [6:7587241251317900845:2121] self: [6:7587241251317900850:0] peer: [2:7587241249765931558:0] socket: 322 qp: -1
2025-12-24T02:11:04.106799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251317900853:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.106806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.107003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251317900848:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.107153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241249446546444:2107] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241253741513769:2120] held: no
2025-12-24T02:11:04.107101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241247022933507:2094] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241251317900848:2122] held: no
2025-12-24T02:11:04.107125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241247022933507:2094] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.107132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.107137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241247022933507:2094] [node 4] ICP22 created new session: [6:7587241251317900855:2094]
2025-12-24T02:11:04.107143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251317900855:2094] [node 4] ICS09 handshake done sender: [6:7587241251317900848:2122] self: [6:7587241251317900849:0] peer: [4:7587241247377371720:0] socket: 375 qp: -1
2025-12-24T02:11:04.107149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251317900855:2094] [node 4] ICS10 traffic start
2025-12-24T02:11:04.107155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.114817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241249446546384:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241253741513768:2119] held: no
2025-12-24T02:11:04.114952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513777:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.115125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241253741513778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41271
2025-12-24T02:11:04.115129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241253741513769:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.115195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241249446546444:2107] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.115221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.115227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241249446546444:2107] [node 4] ICP22 created new session: [5:7587241253741513779:2107]
2025-12-24T02:11:04.115234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241253741513779:2107] [node 4] ICS09 handshake done sender: [5:7587241253741513769:2120] self: [5:7587241253741513776:0] peer: [4:7587241247377371729:0] socket: 376 qp: -1
2025-12-24T02:11:04.115241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241253741513779:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.115250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.115290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241253741513768:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.115478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241249446546384:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.115487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.115493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241249446546384:2050] [node 3] ICP22 created new session: [5:7587241253741513781:2050]
2025-12-24T02:11:04.115498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241253741513781:2050] [node 3] ICS09 handshake done sender: [5:7587241253741513768:2119] self: [5:7587241253741513773:0] peer: [3:7587241253207212705:0] socket: 328 qp: -1
2025-12-24T02:11:04.115504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241253741513781:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.115512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.121288Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:04.121371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251317900846:2048] [node 1] ICS01 socket: 353 reason# EndOfStream
2025-12-24T02:11:04.121380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241247022933456:2048] [node 1] ICP30 unregister session Session# [6:7587241251317900846:2048] VirtualId# [6:7587241251317900835:0]
2025-12-24T02:11:04.121387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251317900846:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.121433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.121460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.122776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:04.122839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241253741513774:2048] [node 1] ICS01 socket: 372 reason# EndOfStream
2025-12-24T02:11:04.122844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241249446546382:2048] [node 1] ICP30 unregister session Session# [5:7587241253741513774:2048] VirtualId# [5:7587241253741513771:0]
2025-12-24T02:11:04.122847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241253741513774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.122881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.122899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.122774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:04.122848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241253207212728:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-12-24T02:11:04.122853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241248912245335:2048] [node 1] ICP30 unregister session Session# [3:7587241253207212728:2048] VirtualId# [3:7587241253207212725:0]
2025-12-24T02:11:04.122858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241253207212728:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.122911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.122939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.122736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:04.122807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241251672339051:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:04.122813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241247377371656:2048] [node 1] ICP30 unregister session Session# [4:7587241251672339051:2048] VirtualId# [4:7587241251672339045:0]
2025-12-24T02:11:04.122818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241251672339051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.122858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.122881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.130617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:04.130708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241251672339040:2049] [node 2] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:04.130718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241247377371657:2049] [node 2] ICP30 unregister session Session# [4:7587241251672339040:2049] VirtualId# [4:7587241251672339033:0]
2025-12-24T02:11:04.130726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241251672339040:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.130760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.131037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:04.131117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241253741513765:2049] [node 2] ICS01 socket: 316 reason# EndOfStream
2025-12-24T02:11:04.131123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241249446546383:2049] [node 2] ICP30 unregister session Session# [5:7587241253741513765:2049] VirtualId# [5:7587241253741513759:0]
2025-12-24T02:11:04.131127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241253741513765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.130788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.131590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:04.131648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251317900853:2049] [node 2] ICS01 socket: 322 reason# EndOfStream
2025-12-24T02:11:04.131176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.131198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.131653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241247022933457:2049] [node 2] ICP30 unregister session Session# [6:7587241251317900853:2049] VirtualId# [6:7587241251317900850:0]
2025-12-24T02:11:04.131658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251317900853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.131692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.131714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.144095Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:04.144184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251317900838:2050] [node 3] ICS01 socket: 332 reason# EndOfStream
2025-12-24T02:11:04.144191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241247022933458:2050] [node 3] ICP30 unregister session Session# [6:7587241251317900838:2050] VirtualId# [6:7587241251317900834:0]
2025-12-24T02:11:04.144198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251317900838:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.144256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.144278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.148363Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:04.148442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241253741513781:2050] [node 3] ICS01 socket: 328 reason# EndOfStream
2025-12-24T02:11:04.148448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241249446546384:2050] [node 3] ICP30 unregister session Session# [5:7587241253741513781:2050] VirtualId# [5:7587241253741513773:0]
2025-12-24T02:11:04.148454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241253741513781:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.148507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.148527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.152590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:04.152660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241253741513779:2107] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-12-24T02:11:04.152666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241249446546444:2107] [node 4] ICP30 unregister session Session# [5:7587241253741513779:2107] VirtualId# [5:7587241253741513776:0]
2025-12-24T02:11:04.152671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241253741513779:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.152697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:04.152718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:04.152439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:04.152525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251317900855:2094] [node 4] ICS01 socket: 375 reason# EndOfStream
2025-12-24T02:11:04.152532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241247022933507:2094] [node 4] ICP30 unregister session Session# [6:7587241251317900855:2094] VirtualId# [6:7587241251317900849:0]
2025-12-24T02:11:04.152538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251317900855:2094] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.152581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:04.152614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:04.176052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253827839538:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.176060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253827839539:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.177654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255086046638:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.177662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255086046639:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.173622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241252953390834:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:04.173628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241252953390835:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.183066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241254039322164:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.183074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241254039322165:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.183078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241254039322166:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.179506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241253645069185:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.179513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241253645069186:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.179517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241253645069187:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.181262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241251608629461:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.181271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241251608629462:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.181278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241251608629463:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.196431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241254039322226:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:04.196529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241254039322227:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.197484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839599:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.197887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839598:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.198619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069245:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.197711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241252953390898:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.199005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069246:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.201478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241253645069246:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.201662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241253645069248:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:04.202062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241253645069248:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.201745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255086046695:2100] [node 5] ICP01 ready to work
2025-12-24T02:11:04.203261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046694:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.203304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253827839604:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:04.203404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253827839605:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:04.203489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253827839606:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:04.203409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255086046698:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:04.203496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255086046699:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:04.204937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241254039322229:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.204952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241254039322229:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.206225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839598:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.203963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241252953390907:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:04.204060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241252953390908:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.211111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629520:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.215923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
2025-12-24T02:11:04.215619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241253645069255:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:04.215715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241253645069245:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.216110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241253645069185:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.217001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241254039322229:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.217585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241254039322226:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.217599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.217607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241254039322226:2106] [node 4] ICP22 created new session: [6:7587241254039322239:2106]
2025-12-24T02:11:04.217612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241254039322239:2106] [node 4] ICS09 handshake done sender: [6:7587241254039322229:2109] self: [6:7587241254039322228:0] peer: [4:7587241253645069249:0] socket: 326 qp: -1
2025-12-24T02:11:04.217619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241254039322239:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:04.217629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.217032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390902:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.217045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241252953390902:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.217490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241252953390913:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:79
2025-12-24T02:11:04.217707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241252953390902:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.217908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241252953390898:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.217918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.217925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241252953390898:2108] [node 4] ICP22 created new session: [1:7587241252953390916:2108]
2025-12-24T02:11:04.217931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241252953390916:2108] [node 4] ICS09 handshake done sender: [1:7587241252953390902:2110] self: [1:7587241252953390901:0] peer: [4:7587241253645069258:0] socket: 324 qp: -1
2025-12-24T02:11:04.217936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241252953390916:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:04.217946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.220151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241251608629529:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:04.220251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241251608629530:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.235043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046715:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.235071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046716:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.236003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322245:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.237851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046694:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.237895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241255086046638:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.238044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390900:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.238056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241252953390900:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.238765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241252953390900:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.238818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241254039322245:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.238890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241252953390835:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.238909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.238917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241252953390835:2049] [node 3] ICP22 created new session: [1:7587241252953390919:2049]
2025-12-24T02:11:04.238924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241252953390919:2049] [node 3] ICS09 handshake done sender: [1:7587241252953390900:2109] self: [1:7587241252953390899:0] peer: [3:7587241255086046719:0] socket: 323 qp: -1
2025-12-24T02:11:04.238931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241252953390919:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.238939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.239272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322247:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.238874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241251608629461:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24039334ns) {Configure:104(24038327ns) {ProcessConfigured:135(23656640ns) {StartInitialHandshake:157(23655832ns) {DropHandshakes:470(682ns) {DropIncomingHandshake:442(116ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1027ns) {SwitchToState:179(470ns)}}}}}
2025-12-24T02:11:04.238930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629531:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
2025-12-24T02:11:04.238947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629532:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
2025-12-24T02:11:04.243688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241254039322247:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.243730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241254039322164:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.246840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390912:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.246856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241252953390912:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.246855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241252953390922:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:79
2025-12-24T02:11:04.247561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241252953390912:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.246933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241251608629530:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25866301ns) {Configure:104(25865630ns) {ProcessConfigured:135(25677983ns) {StartInitialHandshake:157(25676996ns) {DropHandshakes:470(201ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(983ns) {SwitchToState:179(673ns)}}}}}
2025-12-24T02:11:04.246993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629532:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.247049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241251608629461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.250875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241252953390908:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.250897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.250907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241252953390908:2112] [node 6] ICP22 created new session: [1:7587241252953390924:2112]
2025-12-24T02:11:04.250913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241252953390924:2112] [node 6] ICS09 handshake done sender: [1:7587241252953390912:2114] self: [1:7587241252953390911:0] peer: [6:7587241254039322248:0] socket: 363 qp: -1
2025-12-24T02:11:04.250923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241252953390924:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:04.250933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.251202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839620:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
2025-12-24T02:11:04.252459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390910:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.252476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241252953390910:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.254725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046720:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.254880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629519:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.261419Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253827839538:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33849561ns) {Configure:104(33847980ns) {ProcessConfigured:135(33508673ns) {StartInitialHandshake:157(33507976ns) {DropHandshakes:470(373ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:477(1254ns) {SwitchToState:179(667ns)}}}}}
2025-12-24T02:11:04.261464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241253827839538:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.261537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241252953390926:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:79
2025-12-24T02:11:04.262736Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241254039322166:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24043046ns) {Configure:104(24041608ns) {ProcessConfigured:135(23653601ns) {StartInitialHandshake:157(23652758ns) {DropHandshakes:470(1081ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(1453ns) {SwitchToState:179(606ns)}}}}}
2025-12-24T02:11:04.262803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241254039322166:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.262846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629531:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.262878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629519:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.263145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241253645069257:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.263157Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241253645069257:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.263752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069266:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.264078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.266194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241251608629529:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.266381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322250:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.266261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069265:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.266345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069268:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.266453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839620:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.266546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839618:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.266869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241254039322247:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.266867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629520:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.266894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241251608629462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.266996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629546:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
2025-12-24T02:11:04.267120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241251608629532:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.267191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241251608629461:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.267220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.267230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241251608629461:2048] [node 1] ICP22 created new session: [5:7587241251608629549:2048]
2025-12-24T02:11:04.267236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241251608629549:2048] [node 1] ICS09 handshake done sender: [5:7587241251608629532:2113] self: [5:7587241251608629543:0] peer: [1:7587241252953390909:0] socket: 354 qp: -1
2025-12-24T02:11:04.267241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241251608629549:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.267252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.267237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241254039322164:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.267251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.267258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241254039322164:2048] [node 1] ICP22 created new session: [6:7587241254039322251:2048]
2025-12-24T02:11:04.267263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241254039322251:2048] [node 1] ICS09 handshake done sender: [6:7587241254039322247:2116] self: [6:7587241254039322248:0] peer: [1:7587241252953390911:0] socket: 365 qp: -1
2025-12-24T02:11:04.267268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241254039322251:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.267284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.267241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839612:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.267249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253827839612:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.268337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253827839612:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.268431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253827839605:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.268443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.268448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253827839605:2110] [node 5] ICP22 created new session: [2:7587241253827839625:2110]
2025-12-24T02:11:04.268452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253827839625:2110] [node 5] ICS09 handshake done sender: [2:7587241253827839612:2114] self: [2:7587241253827839611:0] peer: [5:7587241251608629548:0] socket: 335 qp: -1
2025-12-24T02:11:04.268456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253827839625:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:04.268471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.268653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839626:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
2025-12-24T02:11:04.270815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241253645069257:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.270891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046710:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.270911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255086046710:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.270899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322253:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.271524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255086046694:2101] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.266757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390913:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.266807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241252953390910:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.266894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241252953390907:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.266904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.266911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241252953390907:2111] [node 5] ICP22 created new session: [1:7587241252953390928:2111]
2025-12-24T02:11:04.266916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241252953390928:2111] [node 5] ICS09 handshake done sender: [1:7587241252953390910:2113] self: [1:7587241252953390909:0] peer: [5:7587241251608629543:0] socket: 348 qp: -1
2025-12-24T02:11:04.266922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241252953390928:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:04.266930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.267049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241252953390898:2108] [node 4] ICP18 handshake (actor [1:7587241252953390913:2115]) is too old
2025-12-24T02:11:04.267821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390894:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.267825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241252953390894:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.268207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241252953390926:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.268277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241252953390907:2111] [node 5] ICP18 handshake (actor [1:7587241252953390926:2117]) is too old
2025-12-24T02:11:04.271022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241252953390894:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.271853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.274827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839599:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.275007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253827839598:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.278838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241254039322253:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.278904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241254039322165:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.280060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241253645069265:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.280177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241253645069268:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.280185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241253645069255:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.280194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.280202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241253645069255:2110] [node 5] ICP22 created new session: [4:7587241253645069270:2110]
2025-12-24T02:11:04.280208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241253645069270:2110] [node 5] ICS09 handshake done sender: [4:7587241253645069257:2113] self: [4:7587241253645069256:0] peer: [5:7587241251608629544:0] socket: 344 qp: -1
2025-12-24T02:11:04.280215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241253645069270:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:04.280221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.280340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241253645069245:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.282607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241251608629462:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241251608629520:2106] held: no
2025-12-24T02:11:04.282755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241251608629531:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.282835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241251608629529:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.282857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.282870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241251608629529:2110] [node 4] ICP22 created new session: [5:7587241251608629552:2110]
2025-12-24T02:11:04.282877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241251608629552:2110] [node 4] ICS09 handshake done sender: [5:7587241251608629531:2112] self: [5:7587241251608629544:0] peer: [4:7587241253645069256:0] socket: 353 qp: -1
2025-12-24T02:11:04.282889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241251608629552:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:04.280072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241253827839538:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.280093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.280100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253827839538:2048] [node 1] ICP22 created new session: [2:7587241253827839630:2048]
2025-12-24T02:11:04.282908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.280105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253827839630:2048] [node 1] ICS09 handshake done sender: [2:7587241253827839598:2107] self: [2:7587241253827839621:0] peer: [1:7587241252953390893:0] socket: 320 qp: -1
2025-12-24T02:11:04.280112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253827839630:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.280129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.282952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629551:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
2025-12-24T02:11:04.283062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241255086046695:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(60434362ns) {Configure:104(60433054ns) {ProcessConfigured:135(60086641ns) {StartInitialHandshake:157(60085104ns) {DropHandshakes:470(347ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1754ns) {SwitchToState:179(1124ns)}}}}}
2025-12-24T02:11:04.283141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255086046710:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.283748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.283827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046715:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.283841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241255086046638:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.283851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.283857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255086046638:2048] [node 1] ICP22 created new session: [3:7587241255086046724:2048]
2025-12-24T02:11:04.283862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255086046724:2048] [node 1] ICS09 handshake done sender: [3:7587241255086046694:2101] self: [3:7587241255086046719:0] peer: [1:7587241252953390899:0] socket: 330 qp: -1
2025-12-24T02:11:04.283867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255086046724:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.283883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.285701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322254:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.286131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241254039322245:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.286471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255086046699:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.286487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.286494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255086046699:2103] [node 6] ICP22 created new session: [3:7587241255086046727:2103]
2025-12-24T02:11:04.286500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255086046727:2103] [node 6] ICS09 handshake done sender: [3:7587241255086046710:2111] self: [3:7587241255086046709:0] peer: [6:7587241254039322249:0] socket: 358 qp: -1
2025-12-24T02:11:04.286506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255086046727:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:04.286533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.286507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839614:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.286516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253827839614:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.286390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241254039322166:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.286410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.286418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241254039322166:2050] [node 3] ICP22 created new session: [6:7587241254039322257:2050]
2025-12-24T02:11:04.286423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241254039322257:2050] [node 3] ICS09 handshake done sender: [6:7587241254039322245:2114] self: [6:7587241254039322249:0] peer: [3:7587241255086046709:0] socket: 359 qp: -1
2025-12-24T02:11:04.286429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241254039322257:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.286434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.287642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253827839614:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.287771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839610:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.287779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253827839610:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.288170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253827839633:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10037
2025-12-24T02:11:04.288269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253827839610:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.288334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253827839604:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.288344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.288349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253827839604:2109] [node 4] ICP22 created new session: [2:7587241253827839635:2109]
2025-12-24T02:11:04.288353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253827839635:2109] [node 4] ICS09 handshake done sender: [2:7587241253827839610:2113] self: [2:7587241253827839609:0] peer: [4:7587241253645069277:0] socket: 334 qp: -1
2025-12-24T02:11:04.288358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253827839635:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:04.288363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.286191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241253645069185:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241253645069245:2107] held: no
2025-12-24T02:11:04.286213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241253645069185:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.286219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.286224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241253645069185:2048] [node 1] ICP22 created new session: [4:7587241253645069272:2048]
2025-12-24T02:11:04.286228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241253645069272:2048] [node 1] ICS09 handshake done sender: [4:7587241253645069245:2107] self: [4:7587241253645069258:0] peer: [1:7587241252953390901:0] socket: 325 qp: -1
2025-12-24T02:11:04.286232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241253645069272:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.286238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.286344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241253645069246:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.287232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241253645069248:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.287239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.287244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241253645069248:2109] [node 6] ICP22 created new session: [4:7587241253645069274:2109]
2025-12-24T02:11:04.287248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241253645069274:2109] [node 6] ICS09 handshake done sender: [4:7587241253645069246:2108] self: [4:7587241253645069249:0] peer: [6:7587241254039322228:0] socket: 328 qp: -1
2025-12-24T02:11:04.287253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241253645069274:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.287257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.287427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241253645069186:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.287437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241253645069187:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.288828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069279:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
2025-12-24T02:11:04.285800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241252953390834:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.285817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.285823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241252953390834:2048] [node 2] ICP22 created new session: [1:7587241252953390931:2048]
2025-12-24T02:11:04.285828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241252953390931:2048] [node 2] ICS09 handshake done sender: [1:7587241252953390894:2106] self: [1:7587241252953390893:0] peer: [2:7587241253827839621:0] socket: 318 qp: -1
2025-12-24T02:11:04.285835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241252953390931:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:04.285850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.290913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046716:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.289017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253827839606:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.289033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.289039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253827839606:2111] [node 6] ICP22 created new session: [2:7587241253827839637:2111]
2025-12-24T02:11:04.289043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253827839637:2111] [node 6] ICS09 handshake done sender: [2:7587241253827839614:2115] self: [2:7587241253827839613:0] peer: [6:7587241254039322255:0] socket: 380 qp: -1
2025-12-24T02:11:04.289047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253827839637:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:04.289053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.292090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241254039322164:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24824737ns) {HandleHandshakeStatus:329(24824300ns) {DropIncomingHandshake:442(184ns) DropOutgoingHandshake:457(977ns) SwitchToState:179(68ns) ProcessPendingSessionEvents:495(20182682ns) {ForwardSessionEventToSession:612(20172607ns) ForwardSessionEventToSession:612(1656ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(570ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(828ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(477ns) ForwardSessionEventToSession:612(1117ns)}}}
2025-12-24T02:11:04.290811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241252953390908:2112] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:04.292018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241251608629520:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.291102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046708:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.291110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255086046708:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.291620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241255086046639:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.291670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046726:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.295559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241253645069265:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.295437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241251608629462:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.295453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.295467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241251608629462:2049] [node 2] ICP22 created new session: [5:7587241251608629556:2049]
2025-12-24T02:11:04.295473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241251608629556:2049] [node 2] ICS09 handshake done sender: [5:7587241251608629520:2106] self: [5:7587241251608629548:0] peer: [2:7587241253827839611:0] socket: 337 qp: -1
2025-12-24T02:11:04.295480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241251608629556:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.295487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.295368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046722:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.295654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839633:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.295737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241253827839604:2109] [node 4] ICP18 handshake (actor [2:7587241253827839633:2120]) is too old
2025-12-24T02:11:04.295890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253827839608:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.295895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253827839608:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.296927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253827839608:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.296998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253827839539:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.297011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.297017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253827839539:2049] [node 3] ICP22 created new session: [2:7587241253827839640:2049]
2025-12-24T02:11:04.297024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253827839640:2049] [node 3] ICS09 handshake done sender: [2:7587241253827839608:2112] self: [2:7587241253827839607:0] peer: [3:7587241255086046729:0] socket: 333 qp: -1
2025-12-24T02:11:04.297029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253827839640:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.297036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.298824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255086046708:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.306561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241255086046699:2103] [node 6] ICP35 event processing took too much time PendingConnection:279(20048831ns) {HandleHandshakeStatus:329(20048059ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(176ns) SwitchToState:179(230ns) ProcessPendingSessionEvents:495(19940399ns) {ForwardSessionEventToSession:612(19924564ns) ForwardSessionEventToSession:612(2982ns) ForwardSessionEventToSession:612(941ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(982ns) ForwardSessionEventToSession:612(951ns) ForwardSessionEventToSession:612(735ns) ForwardSessionEventToSession:612(815ns) ForwardSessionEventToSession:612(841ns)}}}
2025-12-24T02:11:04.306654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255086046698:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.306671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.306680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255086046698:2102] [node 4] ICP22 created new session: [3:7587241255086046735:2102]
2025-12-24T02:11:04.306684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255086046735:2102] [node 4] ICS09 handshake done sender: [3:7587241255086046708:2110] self: [3:7587241255086046707:0] peer: [4:7587241253645069278:0] socket: 357 qp: -1
2025-12-24T02:11:04.306703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255086046735:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:04.306711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.306855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629558:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.306882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241254039322165:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241254039322253:2118] held: no
2025-12-24T02:11:04.307253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241253645069186:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.307270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.307275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241253645069186:2049] [node 2] ICP22 created new session: [4:7587241253645069281:2049]
2025-12-24T02:11:04.307281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241253645069281:2049] [node 2] ICS09 handshake done sender: [4:7587241253645069265:2116] self: [4:7587241253645069277:0] peer: [2:7587241253827839609:0] socket: 339 qp: -1
2025-12-24T02:11:04.307286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241253645069281:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.307292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.307346Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241253645069186:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:04.307452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241253645069280:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37420
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.307685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322259:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.307875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322260:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.307891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241254039322253:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.307944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241254039322165:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.307955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.307960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241254039322165:2049] [node 2] ICP22 created new session: [6:7587241254039322261:2049]
2025-12-24T02:11:04.307965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241254039322261:2049] [node 2] ICS09 handshake done sender: [6:7587241254039322253:2118] self: [6:7587241254039322255:0] peer: [2:7587241253827839613:0] socket: 381 qp: -1
2025-12-24T02:11:04.307969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241254039322261:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.307975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.308184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241255086046638:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24306704ns) {HandleHandshakeStatus:329(24306155ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(982ns) SwitchToState:179(214ns) ProcessPendingSessionEvents:495(15842225ns) {ForwardSessionEventToSession:612(15832078ns) ForwardSessionEventToSession:612(1757ns) ForwardSessionEventToSession:612(667ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(1160ns) ForwardSessionEventToSession:612(947ns) ForwardSessionEventToSession:612(505ns) ForwardSessionEventToSession:612(628ns) ForwardSessionEventToSession:612(824ns)}}}
2025-12-24T02:11:04.309430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241255086046639:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13607) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241255086046715:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.310978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255086046734:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63730
2025-12-24T02:11:04.312134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241253645069187:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62200) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241253645069268:2119] held: no
2025-12-24T02:11:04.312166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241253645069268:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.312235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241253645069187:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.312244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.312251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241253645069187:2050] [node 3] ICP22 created new session: [4:7587241253645069283:2050]
2025-12-24T02:11:04.312256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241253645069283:2050] [node 3] ICS09 handshake done sender: [4:7587241253645069268:2119] self: [4:7587241253645069278:0] peer: [3:7587241255086046707:0] socket: 373 qp: -1
2025-12-24T02:11:04.312263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241253645069283:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.312269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.316746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629558:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.316805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241251608629463:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.317874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241251608629560:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18210
2025-12-24T02:11:04.318189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241251608629558:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.318299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241251608629463:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.318317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.318325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241251608629463:2050] [node 3] ICP22 created new session: [5:7587241251608629561:2050]
2025-12-24T02:11:04.318339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241251608629561:2050] [node 3] ICS09 handshake done sender: [5:7587241251608629558:2122] self: [5:7587241251608629559:0] peer: [3:7587241255086046701:0] socket: 387 qp: -1
2025-12-24T02:11:04.318348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241251608629561:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.318359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.316943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255086046721:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.316955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255086046721:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.318003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255086046721:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.318480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255086046695:2100] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.318492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.318498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255086046695:2100] [node 5] ICP22 created new session: [3:7587241255086046738:2100]
2025-12-24T02:11:04.318503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255086046738:2100] [node 5] ICS09 handshake done sender: [3:7587241255086046721:2118] self: [3:7587241255086046701:0] peer: [5:7587241251608629559:0] socket: 385 qp: -1
2025-12-24T02:11:04.318508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255086046738:2100] [node 5] ICS10 traffic start
2025-12-24T02:11:04.318515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.318769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241254039322260:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.318814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241254039322227:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.319508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241251608629540:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.319516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241251608629540:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.320069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241251608629540:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.320155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241251608629530:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.320172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.320178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241251608629530:2111] [node 6] ICP22 created new session: [5:7587241251608629564:2111]
2025-12-24T02:11:04.320183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241251608629564:2111] [node 6] ICS09 handshake done sender: [5:7587241251608629540:2118] self: [5:7587241251608629537:0] peer: [6:7587241254039322263:0] socket: 355 qp: -1
2025-12-24T02:11:04.320187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241251608629564:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:04.320195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.319351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255086046715:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.322457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241255086046639:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.322473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.322481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255086046639:2049] [node 2] ICP22 created new session: [3:7587241255086046740:2049]
2025-12-24T02:11:04.322487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255086046740:2049] [node 2] ICS09 handshake done sender: [3:7587241255086046715:2115] self: [3:7587241255086046729:0] peer: [2:7587241253827839607:0] socket: 338 qp: -1
2025-12-24T02:11:04.322494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255086046740:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.322504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.324289Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241254039322227:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8775) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241254039322260:2121] held: no
2025-12-24T02:11:04.324368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241254039322264:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33910
2025-12-24T02:11:04.331284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241254039322260:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.335302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241254039322227:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.335327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.335336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241254039322227:2108] [node 5] ICP22 created new session: [6:7587241254039322265:2108]
2025-12-24T02:11:04.335342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241254039322265:2108] [node 5] ICS09 handshake done sender: [6:7587241254039322260:2121] self: [6:7587241254039322263:0] peer: [5:7587241251608629537:0] socket: 321 qp: -1
2025-12-24T02:11:04.335353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241254039322265:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:04.335369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.340316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:04.340403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241253827839630:2048] [node 1] ICS01 socket: 320 reason# EndOfStream
2025-12-24T02:11:04.340411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241253827839538:2048] [node 1] ICP30 unregister session Session# [2:7587241253827839630:2048] VirtualId# [2:7587241253827839621:0]
2025-12-24T02:11:04.340418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241253827839630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.340458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.340480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.340895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:04.340983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241255086046724:2048] [node 1] ICS07 socket disconnect 330 reason# EndOfStream
2025-12-24T02:11:04.340988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241255086046724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.341020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.341038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241255086046724:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.343394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241255086046638:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:04.343407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241255086046724:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.343696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241255086046638:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.343710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241255086046724:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.343715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241255086046638:2048] [node 1] ICP30 unregister session Session# [3:7587241255086046724:2048] VirtualId# [3:7587241255086046719:0]
2025-12-24T02:11:04.343732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.343737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241255086046638:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.343740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.346723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:04.346797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241253645069272:2048] [node 1] ICS01 socket: 325 reason# EndOfStream
2025-12-24T02:11:04.346803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241253645069185:2048] [node 1] ICP30 unregister session Session# [4:7587241253645069272:2048] VirtualId# [4:7587241253645069258:0]
2025-12-24T02:11:04.346809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241253645069272:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.346845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.346869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.346958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:04.347009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241251608629549:2048] [node 1] ICS07 socket disconnect 354 reason# EndOfStream
2025-12-24T02:11:04.347013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241251608629549:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.347036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.347053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241251608629549:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.347440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241251608629461:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.347451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241251608629549:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.347454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241251608629461:2048] [node 1] ICP30 unregister session Session# [5:7587241251608629549:2048] VirtualId# [5:7587241251608629543:0]
2025-12-24T02:11:04.347467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.347472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241251608629461:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.347475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.348840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:04.350038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:04.350109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241255086046740:2049] [node 2] ICS01 socket: 338 reason# EndOfStream
2025-12-24T02:11:04.350115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241255086046639:2049] [node 2] ICP30 unregister session Session# [3:7587241255086046740:2049] VirtualId# [3:7587241255086046729:0]
2025-12-24T02:11:04.350120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241255086046740:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.350159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.350179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.348930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241254039322251:2048] [node 1] ICS07 socket disconnect 365 reason# EndOfStream
2025-12-24T02:11:04.348939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241254039322251:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.348989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.349015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241254039322251:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.349393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241254039322164:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.349400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241254039322251:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.349406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241254039322164:2048] [node 1] ICP30 unregister session Session# [6:7587241254039322251:2048] VirtualId# [6:7587241254039322248:0]
2025-12-24T02:11:04.349421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.349426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241254039322164:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.349428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.354731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:04.354816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241253645069281:2049] [node 2] ICS01 socket: 339 reason# EndOfStream
2025-12-24T02:11:04.354822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241253645069186:2049] [node 2] ICP30 unregister session Session# [4:7587241253645069281:2049] VirtualId# [4:7587241253645069277:0]
2025-12-24T02:11:04.354828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241253645069281:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.354870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.354897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.354926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:04.354993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241251608629556:2049] [node 2] ICS01 socket: 337 reason# EndOfStream
2025-12-24T02:11:04.354997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241251608629462:2049] [node 2] ICP30 unregister session Session# [5:7587241251608629556:2049] VirtualId# [5:7587241251608629548:0]
2025-12-24T02:11:04.355001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241251608629556:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.355040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.355060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.354742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:04.354816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241254039322261:2049] [node 2] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:04.354822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241254039322165:2049] [node 2] ICP30 unregister session Session# [6:7587241254039322261:2049] VirtualId# [6:7587241254039322255:0]
2025-12-24T02:11:04.354828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241254039322261:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.354869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.354897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.361143Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:04.361215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241253645069283:2050] [node 3] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:04.361222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241253645069187:2050] [node 3] ICP30 unregister session Session# [4:7587241253645069283:2050] VirtualId# [4:7587241253645069278:0]
2025-12-24T02:11:04.361227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241253645069283:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.361269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.361293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.361446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241251608629461:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.361552Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:04.361625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241251608629561:2050] [node 3] ICS07 socket disconnect 387 reason# EndOfStream
2025-12-24T02:11:04.361630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241251608629561:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.361679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.361701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241251608629561:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:04.359980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241254039322164:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.361952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:04.362014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241254039322257:2050] [node 3] ICS01 socket: 359 reason# EndOfStream
2025-12-24T02:11:04.362020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241254039322166:2050] [node 3] ICP30 unregister session Session# [6:7587241254039322257:2050] VirtualId# [6:7587241254039322249:0]
2025-12-24T02:11:04.362025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241254039322257:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.362059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.362081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.362343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241251608629463:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62200) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.362356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241251608629561:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.362361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241251608629463:2050] [node 3] ICP30 unregister session Session# [5:7587241251608629561:2050] VirtualId# [5:7587241251608629559:0]
2025-12-24T02:11:04.362376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.362381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241251608629463:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.362384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62200) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.371879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241251608629463:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:04.373052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:04.373148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241254039322239:2106] [node 4] ICS01 socket: 326 reason# EndOfStream
2025-12-24T02:11:04.373157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241254039322226:2106] [node 4] ICP30 unregister session Session# [6:7587241254039322239:2106] VirtualId# [6:7587241254039322228:0]
2025-12-24T02:11:04.373164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241254039322239:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.373223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:04.373248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:04.383116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:04.383247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241254039322265:2108] [node 5] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:04.383257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241254039322227:2108] [node 5] ICP30 unregister session Session# [6:7587241254039322265:2108] VirtualId# [6:7587241254039322263:0]
2025-12-24T02:11:04.383267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241254039322265:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.383312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:04.383339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:04.397384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253774646175:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.397397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253774646176:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.395700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251486380751:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:04.395707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251486380752:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.399254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241253963935360:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.399263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241253963935361:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.401173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241255149215232:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.401181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241255149215233:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.401185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241255149215234:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.405145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241252348236035:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.405153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241252348236036:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.405157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241252348236037:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.403386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241252432471368:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.403394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241252432471369:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.403397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241252432471370:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.436904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251486380815:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:04.436988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251486380818:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:04.437914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253774646238:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:04.438019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253774646239:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:04.438098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253774646240:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:04.438053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251486380817:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:04.436344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241252348236095:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.443404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251486380828:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15718
2025-12-24T02:11:04.444556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241255149215299:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:04.444646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241255149215300:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.442865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471429:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.445735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471430:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.449380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251486380830:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15718
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.449425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251486380831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15718
2025-12-24T02:11:04.448161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241252432471440:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:04.448251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241252432471441:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.451712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253774646248:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5159
2025-12-24T02:11:04.451739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253774646249:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5159
2025-12-24T02:11:04.452926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241253963935426:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:04.453043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241253963935425:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:04.453072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241253963935427:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.458303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253774646239:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19508354ns) {Configure:104(16546911ns) {ProcessConfigured:135(16243666ns) {StartInitialHandshake:157(16242747ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(496ns) PrepareNewSessionHandshake:477(1087ns) {SwitchToState:179(656ns)}}}}}
2025-12-24T02:11:04.462810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241253963935360:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25045009ns) {Configure:104(25043822ns) {ProcessConfigured:135(24274822ns) {StartInitialHandshake:157(24274162ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1382ns) {SwitchToState:179(836ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.464179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241253963935361:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26280993ns) {Configure:104(26280511ns) {ProcessConfigured:135(25104733ns) {StartInitialHandshake:157(25104074ns) {DropHandshakes:470(375ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:477(608ns) {SwitchToState:179(17ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.464253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935437:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.464097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253774646175:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26989782ns) {Configure:104(26989054ns) {ProcessConfigured:135(24407578ns) {StartInitialHandshake:157(24407037ns) {DropHandshakes:470(261ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(999ns) {SwitchToState:179(452ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.465589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935438:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.464143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471449:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.466786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253774646176:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29573812ns) {Configure:104(29572723ns) {ProcessConfigured:135(27048181ns) {StartInitialHandshake:157(27047338ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(490ns) PrepareNewSessionHandshake:477(1319ns) {SwitchToState:179(416ns)}}}}}
2025-12-24T02:11:04.466832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251486380833:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15718
2025-12-24T02:11:04.470752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471448:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.474918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241252348236110:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:04.478868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236112:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.478989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241253963935427:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25871415ns) {Configure:104(25870840ns) {ProcessConfigured:135(25605153ns) {StartInitialHandshake:157(25604611ns) {DropHandshakes:470(433ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(65ns) PrepareNewSessionHandshake:477(700ns) {SwitchToState:179(364ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.479198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215310:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.479235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215311:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
2025-12-24T02:11:04.479253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215312:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.479420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935441:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.479439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935442:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.481483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241255149215233:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34531675ns) {Configure:104(34530959ns) {ProcessConfigured:135(34297132ns) {StartInitialHandshake:157(34296659ns) {DropHandshakes:470(201ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1422ns) {SwitchToState:179(147ns)}}}}}
2025-12-24T02:11:04.492316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241252348236112:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.492383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241252348236036:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.492641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935438:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.492603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646247:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.492626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253774646247:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.493369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253774646247:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.493444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253774646240:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.493456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.493463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253774646240:2109] [node 6] ICP22 created new session: [2:7587241253774646257:2109]
2025-12-24T02:11:04.493469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253774646257:2109] [node 6] ICS09 handshake done sender: [2:7587241253774646247:2111] self: [2:7587241253774646246:0] peer: [6:7587241252348236114:0] socket: 349 qp: -1
2025-12-24T02:11:04.493475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253774646257:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.493483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.494904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251486380834:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15718
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.496450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215311:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.496549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241255149215233:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.497444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215314:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
2025-12-24T02:11:04.497888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241255149215311:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.497983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241255149215233:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.494817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471448:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.494865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241252432471369:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.497998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.496349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471451:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.498003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241255149215233:2049] [node 2] ICP22 created new session: [4:7587241255149215315:2049]
2025-12-24T02:11:04.498008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241255149215315:2049] [node 2] ICS09 handshake done sender: [4:7587241255149215311:2116] self: [4:7587241255149215313:0] peer: [2:7587241253774646243:0] socket: 363 qp: -1
2025-12-24T02:11:04.498014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241255149215315:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.498030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.494971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646252:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.494980Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253774646252:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.495533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253774646252:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.495599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253774646239:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.495609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.495614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253774646239:2108] [node 5] ICP22 created new session: [2:7587241253774646261:2108]
2025-12-24T02:11:04.495619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253774646261:2108] [node 5] ICS09 handshake done sender: [2:7587241253774646252:2114] self: [2:7587241253774646244:0] peer: [5:7587241252432471450:0] socket: 353 qp: -1
2025-12-24T02:11:04.495625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253774646261:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:04.495632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.496675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646245:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.496679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253774646245:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.497300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253774646245:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.497541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253774646238:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.497552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.497558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253774646238:2107] [node 4] ICP22 created new session: [2:7587241253774646265:2107]
2025-12-24T02:11:04.497565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253774646265:2107] [node 4] ICS09 handshake done sender: [2:7587241253774646245:2110] self: [2:7587241253774646243:0] peer: [4:7587241255149215313:0] socket: 348 qp: -1
2025-12-24T02:11:04.497572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253774646265:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.497590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.498875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646249:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.498897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241253774646239:2108] [node 5] ICP18 handshake (actor [2:7587241253774646249:2113]) is too old
2025-12-24T02:11:04.498744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380834:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.498801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380828:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.501420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380833:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.501959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215304:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.501971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241255149215304:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.502001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380830:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.502480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215310:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.502535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241255149215232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.502608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380826:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.502615Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251486380826:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.502682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241255149215304:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.502631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236115:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.499156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241252432471448:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.501685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241252432471369:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241252432471448:2117] held: no
2025-12-24T02:11:04.501863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471430:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.501881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241252432471440:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.502497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241252432471369:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.502509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.502516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241252432471369:2049] [node 2] ICP22 created new session: [5:7587241252432471453:2049]
2025-12-24T02:11:04.502523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241252432471453:2049] [node 2] ICS09 handshake done sender: [5:7587241252432471448:2117] self: [5:7587241252432471450:0] peer: [2:7587241253774646244:0] socket: 354 qp: -1
2025-12-24T02:11:04.502764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241255149215299:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.502530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241252432471453:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.502546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.502775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.502780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241255149215299:2109] [node 5] ICP22 created new session: [4:7587241255149215320:2109]
2025-12-24T02:11:04.502784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241255149215320:2109] [node 5] ICS09 handshake done sender: [4:7587241255149215304:2112] self: [4:7587241255149215303:0] peer: [5:7587241252432471452:0] socket: 337 qp: -1
2025-12-24T02:11:04.502788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241255149215320:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:04.502799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.503016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.503251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935444:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.503282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935437:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.503303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241253963935361:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.503313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241253963935360:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.503395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646255:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.503407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253774646255:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.504192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253774646255:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.504306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253774646176:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.504321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.504327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253774646176:2049] [node 3] ICP22 created new session: [2:7587241253774646268:2049]
2025-12-24T02:11:04.504333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253774646268:2049] [node 3] ICS09 handshake done sender: [2:7587241253774646255:2116] self: [2:7587241253774646242:0] peer: [3:7587241253963935445:0] socket: 370 qp: -1
2025-12-24T02:11:04.504340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253774646268:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.504350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.504433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.503180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251486380826:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.503281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251486380815:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.503294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.503301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251486380815:2107] [node 4] ICP22 created new session: [1:7587241251486380840:2107]
2025-12-24T02:11:04.503305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251486380840:2107] [node 4] ICS09 handshake done sender: [1:7587241251486380826:2112] self: [1:7587241251486380820:0] peer: [4:7587241255149215319:0] socket: 329 qp: -1
2025-12-24T02:11:04.503311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251486380840:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.503325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.505011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380822:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.505015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251486380822:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.507074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241252348236112:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.507255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236117:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.507558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241252348236117:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.507584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241252348236035:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.507564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380814:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.507573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251486380814:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.509711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241255149215232:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241255149215310:2115] held: no
2025-12-24T02:11:04.507398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935442:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.507443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935441:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.507651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253963935444:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.507762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241253963935361:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.507774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.507781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241253963935361:2049] [node 2] ICP22 created new session: [3:7587241253963935450:2049]
2025-12-24T02:11:04.507788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253963935450:2049] [node 2] ICS09 handshake done sender: [3:7587241253963935444:2119] self: [3:7587241253963935445:0] peer: [2:7587241253774646242:0] socket: 378 qp: -1
2025-12-24T02:11:04.507797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253963935450:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.507804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.509628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380831:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.509672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380829:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.509675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251486380829:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.510260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251486380829:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.507453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253774646248:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.507788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.507495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241253774646175:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.507878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241252348236036:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.507894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.507899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241252348236036:2049] [node 2] ICP22 created new session: [6:7587241252348236119:2049]
2025-12-24T02:11:04.507904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241252348236119:2049] [node 2] ICS09 handshake done sender: [6:7587241252348236112:2114] self: [6:7587241252348236114:0] peer: [2:7587241253774646246:0] socket: 352 qp: -1
2025-12-24T02:11:04.507911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241252348236119:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.507917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.508355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.514332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.513680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241252432471430:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.513804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241252432471440:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.513818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.513826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241252432471440:2112] [node 4] ICP22 created new session: [5:7587241252432471456:2112]
2025-12-24T02:11:04.513832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241252432471456:2112] [node 4] ICS09 handshake done sender: [5:7587241252432471430:2109] self: [5:7587241252432471452:0] peer: [4:7587241255149215303:0] socket: 338 qp: -1
2025-12-24T02:11:04.513838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241252432471456:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:04.513846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.514766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251486380818:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.514789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.514795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251486380818:2110] [node 6] ICP22 created new session: [1:7587241251486380845:2110]
2025-12-24T02:11:04.514799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251486380845:2110] [node 6] ICS09 handshake done sender: [1:7587241251486380829:2114] self: [1:7587241251486380824:0] peer: [6:7587241252348236118:0] socket: 380 qp: -1
2025-12-24T02:11:04.514806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251486380845:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:04.514813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.514800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215312:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.514835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241255149215234:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.515230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236124:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.518779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935433:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.518795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241253963935433:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.518825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236116:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.518968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241252348236117:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.519042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241252348236122:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.519621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241252348236037:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.519889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471449:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.522813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241255149215299:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(20008959ns) {HandleHandshakeStatus:329(20008228ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(156ns) SwitchToState:179(68ns) ProcessPendingSessionEvents:495(19942596ns) {ForwardSessionEventToSession:612(19927560ns) ForwardSessionEventToSession:612(1916ns) ForwardSessionEventToSession:612(1497ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(1401ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(1059ns)}}}
2025-12-24T02:11:04.522823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241252432471370:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.523453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471429:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.527342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215323:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
2025-12-24T02:11:04.530891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241255149215310:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.535228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935440:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.535247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241253963935440:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.538773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
2025-12-24T02:11:04.538842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241255149215232:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.538856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.538865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241255149215232:2048] [node 1] ICP22 created new session: [4:7587241255149215325:2048]
2025-12-24T02:11:04.538879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241255149215325:2048] [node 1] ICS09 handshake done sender: [4:7587241255149215310:2115] self: [4:7587241255149215319:0] peer: [1:7587241251486380820:0] socket: 330 qp: -1
2025-12-24T02:11:04.538888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241255149215325:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.538897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.538790Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251486380814:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.538895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251486380751:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.538910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.538916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251486380751:2048] [node 2] ICP22 created new session: [1:7587241251486380849:2048]
2025-12-24T02:11:04.538921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251486380849:2048] [node 2] ICS09 handshake done sender: [1:7587241251486380814:2109] self: [1:7587241251486380812:0] peer: [2:7587241253774646270:0] socket: 325 qp: -1
2025-12-24T02:11:04.538926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251486380849:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:04.538932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.539533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241252348236116:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.539627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241252348236110:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.540505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251486380822:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.540716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251486380752:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.540731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.540737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251486380752:2049] [node 3] ICP22 created new session: [1:7587241251486380852:2049]
2025-12-24T02:11:04.540742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251486380852:2049] [node 3] ICS09 handshake done sender: [1:7587241251486380822:2111] self: [1:7587241251486380819:0] peer: [3:7587241253963935446:0] socket: 327 qp: -1
2025-12-24T02:11:04.540747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251486380852:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.540752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.541773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215324:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.541814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241255149215306:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.541817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241255149215306:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.542474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236127:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.538720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241253963935431:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.538738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241253963935431:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.540032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241253963935453:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47959
2025-12-24T02:11:04.540658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253963935437:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.541008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241253963935360:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.541016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.541020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241253963935360:2048] [node 1] ICP22 created new session: [3:7587241253963935454:2048]
2025-12-24T02:11:04.541024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253963935454:2048] [node 1] ICS09 handshake done sender: [3:7587241253963935437:2113] self: [3:7587241253963935446:0] peer: [1:7587241251486380819:0] socket: 351 qp: -1
2025-12-24T02:11:04.541029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253963935454:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.541042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.542806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241255149215306:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.542978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241252348236116:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.543040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241252348236110:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.543051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.543056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241252348236110:2113] [node 4] ICP22 created new session: [6:7587241252348236128:2113]
2025-12-24T02:11:04.543060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241252348236128:2113] [node 4] ICS09 handshake done sender: [6:7587241252348236116:2117] self: [6:7587241252348236126:0] peer: [4:7587241255149215305:0] socket: 376 qp: -1
2025-12-24T02:11:04.543065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241252348236128:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:04.543070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.543541Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241253774646175:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241253774646248:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.547169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253774646271:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5159
2025-12-24T02:11:04.546788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241255149215300:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.546816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.546824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241255149215300:2110] [node 6] ICP22 created new session: [4:7587241255149215329:2110]
2025-12-24T02:11:04.546832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241255149215329:2110] [node 6] ICS09 handshake done sender: [4:7587241255149215306:2113] self: [4:7587241255149215305:0] peer: [6:7587241252348236126:0] socket: 374 qp: -1
2025-12-24T02:11:04.546839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241255149215329:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:04.546848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.547276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253774646248:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.547348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241253774646175:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.547357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.547362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253774646175:2048] [node 1] ICP22 created new session: [2:7587241253774646272:2048]
2025-12-24T02:11:04.547368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253774646272:2048] [node 1] ICS09 handshake done sender: [2:7587241253774646248:2112] self: [2:7587241253774646270:0] peer: [1:7587241251486380812:0] socket: 346 qp: -1
2025-12-24T02:11:04.547374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253774646272:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.547383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.550881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241252348236124:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.550978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241252348236095:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.551499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471461:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.552143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241252348236035:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.552158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.552167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241252348236035:2048] [node 1] ICP22 created new session: [6:7587241252348236131:2048]
2025-12-24T02:11:04.552173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241252348236131:2048] [node 1] ICS09 handshake done sender: [6:7587241252348236117:2118] self: [6:7587241252348236118:0] peer: [1:7587241251486380824:0] socket: 383 qp: -1
2025-12-24T02:11:04.552179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241252348236131:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.552199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.555779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253963935433:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.556017Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253963935440:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.556088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241253963935427:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.556104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.556110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241253963935427:2109] [node 6] ICP22 created new session: [3:7587241253963935458:2109]
2025-12-24T02:11:04.556115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253963935458:2109] [node 6] ICS09 handshake done sender: [3:7587241253963935440:2116] self: [3:7587241253963935428:0] peer: [6:7587241252348236125:0] socket: 384 qp: -1
2025-12-24T02:11:04.556120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253963935458:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:04.556126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.556497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241253963935425:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.556511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.556514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241253963935425:2107] [node 4] ICP22 created new session: [3:7587241253963935460:2107]
2025-12-24T02:11:04.556518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253963935460:2107] [node 4] ICS09 handshake done sender: [3:7587241253963935433:2112] self: [3:7587241253963935432:0] peer: [4:7587241255149215322:0] socket: 357 qp: -1
2025-12-24T02:11:04.556522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253963935460:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.556526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.557509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241255149215234:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241255149215312:2117] held: no
2025-12-24T02:11:04.557556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241255149215331:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33599
2025-12-24T02:11:04.557694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241255149215312:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.557744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241255149215234:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.557754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.557758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241255149215234:2050] [node 3] ICP22 created new session: [4:7587241255149215332:2050]
2025-12-24T02:11:04.557763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241255149215332:2050] [node 3] ICS09 handshake done sender: [4:7587241255149215312:2117] self: [4:7587241255149215322:0] peer: [3:7587241253963935432:0] socket: 364 qp: -1
2025-12-24T02:11:04.557767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241255149215332:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.557783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.554830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471461:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.554877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241252432471368:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.559620Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241252348236037:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241252348236122:2119] held: no
2025-12-24T02:11:04.559674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236133:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.559810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241252348236122:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.559855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241252348236037:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.559863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.559868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241252348236037:2050] [node 3] ICP22 created new session: [6:7587241252348236134:2050]
2025-12-24T02:11:04.559874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241252348236134:2050] [node 3] ICS09 handshake done sender: [6:7587241252348236122:2119] self: [6:7587241252348236125:0] peer: [3:7587241253963935428:0] socket: 386 qp: -1
2025-12-24T02:11:04.559880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241252348236134:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.559885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.563563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241253963935431:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.563774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251486380832:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.563796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251486380832:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.563910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241253963935426:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.563927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.563933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241253963935426:2108] [node 5] ICP22 created new session: [3:7587241253963935463:2108]
2025-12-24T02:11:04.563939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241253963935463:2108] [node 5] ICS09 handshake done sender: [3:7587241253963935431:2111] self: [3:7587241253963935430:0] peer: [5:7587241252432471459:0] socket: 356 qp: -1
2025-12-24T02:11:04.563944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241253963935463:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:04.563950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.563778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241252432471443:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.563794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241252432471443:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.567017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241252348236035:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(47906503ns) {HandleHandshakeStatus:329(47905857ns) {DropIncomingHandshake:442(720ns) DropOutgoingHandshake:457(2282ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(3320633ns) {ForwardSessionEventToSession:612(3310001ns) ForwardSessionEventToSession:612(1429ns) ForwardSessionEventToSession:612(1377ns) ForwardSessionEventToSession:612(553ns) ForwardSessionEventToSession:612(624ns) ForwardSessionEventToSession:612(792ns) ForwardSessionEventToSession:612(665ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(458ns) ForwardSessionEventToSession:612(946ns)}}}
2025-12-24T02:11:04.567043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241252348236035:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.568780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471464:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.568835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241252432471465:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38966
2025-12-24T02:11:04.569907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241252432471443:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.570997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241252348236136:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44861
2025-12-24T02:11:04.570725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241252432471370:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22459) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241252432471449:2118] held: no
2025-12-24T02:11:04.570854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241252432471441:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.570867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.570874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241252432471441:2114] [node 6] ICP22 created new session: [5:7587241252432471466:2114]
2025-12-24T02:11:04.570879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241252432471466:2114] [node 6] ICS09 handshake done sender: [5:7587241252432471443:2115] self: [5:7587241252432471442:0] peer: [6:7587241252348236130:0] socket: 388 qp: -1
2025-12-24T02:11:04.570885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241252432471466:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:04.570892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.571278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241252348236124:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.571313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241252348236095:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13976) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241252348236124:2121] held: no
2025-12-24T02:11:04.571436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241252348236095:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.571446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.571456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241252348236095:2106] [node 5] ICP22 created new session: [6:7587241252348236137:2106]
2025-12-24T02:11:04.571472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241252348236137:2106] [node 5] ICS09 handshake done sender: [6:7587241252348236124:2121] self: [6:7587241252348236130:0] peer: [5:7587241252432471442:0] socket: 390 qp: -1
2025-12-24T02:11:04.571477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241252348236137:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:04.571485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.571781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251486380832:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.572007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241252432471449:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.572011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251486380817:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.572030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.572036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251486380817:2108] [node 5] ICP22 created new session: [1:7587241251486380855:2108]
2025-12-24T02:11:04.572040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251486380855:2108] [node 5] ICS09 handshake done sender: [1:7587241251486380832:2117] self: [1:7587241251486380825:0] peer: [5:7587241252432471462:0] socket: 391 qp: -1
2025-12-24T02:11:04.572045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251486380855:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:04.572054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.572360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241252432471370:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.572372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.572377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241252432471370:2050] [node 3] ICP22 created new session: [5:7587241252432471468:2050]
2025-12-24T02:11:04.572382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241252432471468:2050] [node 3] ICS09 handshake done sender: [5:7587241252432471449:2118] self: [5:7587241252432471459:0] peer: [3:7587241253963935430:0] socket: 359 qp: -1
2025-12-24T02:11:04.572387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241252432471468:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.572392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.572575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241252432471368:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241252432471461:2121] held: no
2025-12-24T02:11:04.572744Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241252432471461:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.572807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241252432471368:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.572825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.572838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241252432471368:2048] [node 1] ICP22 created new session: [5:7587241252432471470:2048]
2025-12-24T02:11:04.572846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241252432471470:2048] [node 1] ICS09 handshake done sender: [5:7587241252432471461:2121] self: [5:7587241252432471462:0] peer: [1:7587241251486380825:0] socket: 392 qp: -1
2025-12-24T02:11:04.572854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241252432471470:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.572861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.586940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:04.586997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241252348236131:2048] [node 1] ICS07 socket disconnect 383 reason# EndOfStream
2025-12-24T02:11:04.587002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241252348236131:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.587026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:04.587084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241253963935454:2048] [node 1] ICS01 socket: 351 reason# EndOfStream
2025-12-24T02:11:04.587088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241253963935360:2048] [node 1] ICP30 unregister session Session# [3:7587241253963935454:2048] VirtualId# [3:7587241253963935446:0]
2025-12-24T02:11:04.587092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241253963935454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.587118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.587139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.587029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.587052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241252348236131:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.586759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:04.586848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241252432471470:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:04.586855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241252432471368:2048] [node 1] ICP30 unregister session Session# [5:7587241252432471470:2048] VirtualId# [5:7587241252432471462:0]
2025-12-24T02:11:04.586861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241252432471470:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.586872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.586906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.587615Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:04.587695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241255149215325:2048] [node 1] ICS01 socket: 330 reason# EndOfStream
2025-12-24T02:11:04.587702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241255149215232:2048] [node 1] ICP30 unregister session Session# [4:7587241255149215325:2048] VirtualId# [4:7587241255149215319:0]
2025-12-24T02:11:04.587709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241255149215325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.587761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.587787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.587448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241252348236035:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.587458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241252348236131:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.587462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241252348236035:2048] [node 1] ICP30 unregister session Session# [6:7587241252348236131:2048] VirtualId# [6:7587241252348236118:0]
2025-12-24T02:11:04.587473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.587478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241252348236035:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.587480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26173) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.595600Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:04.595694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241255149215315:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-12-24T02:11:04.595702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241255149215233:2049] [node 2] ICP30 unregister session Session# [4:7587241255149215315:2049] VirtualId# [4:7587241255149215313:0]
2025-12-24T02:11:04.595708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241255149215315:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.595747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.595768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.599074Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:04.599130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241252348236119:2049] [node 2] ICS07 socket disconnect 352 reason# EndOfStream
2025-12-24T02:11:04.599134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241252348236119:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.599155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.599173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241252348236119:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:04.599228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241252348236035:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.598889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:04.598974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241252432471453:2049] [node 2] ICS01 socket: 354 reason# EndOfStream
2025-12-24T02:11:04.598982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241252432471369:2049] [node 2] ICP30 unregister session Session# [5:7587241252432471453:2049] VirtualId# [5:7587241252432471450:0]
2025-12-24T02:11:04.598987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241252432471453:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.599027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.599047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.599629Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241252348236036:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.599639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241252348236119:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.599643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241252348236036:2049] [node 2] ICP30 unregister session Session# [6:7587241252348236119:2049] VirtualId# [6:7587241252348236114:0]
2025-12-24T02:11:04.599654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.599658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241252348236036:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.599660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.614734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241252348236036:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:04.622835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:04.622911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241252348236134:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:04.622917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241252348236037:2050] [node 3] ICP30 unregister session Session# [6:7587241252348236134:2050] VirtualId# [6:7587241252348236125:0]
2025-12-24T02:11:04.622921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241252348236134:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.622967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.622987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.623045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:04.623098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241252432471468:2050] [node 3] ICS01 socket: 359 reason# EndOfStream
2025-12-24T02:11:04.623101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241252432471370:2050] [node 3] ICP30 unregister session Session# [5:7587241252432471468:2050] VirtualId# [5:7587241252432471459:0]
2025-12-24T02:11:04.623104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241252432471468:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.623126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.623142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.634757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:04.634839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241252432471456:2112] [node 4] ICS01 socket: 338 reason# EndOfStream
2025-12-24T02:11:04.634847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241252432471440:2112] [node 4] ICP30 unregister session Session# [5:7587241252432471456:2112] VirtualId# [5:7587241252432471452:0]
2025-12-24T02:11:04.634853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241252432471456:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.634908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:04.634931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:04.638846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:04.638940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241252348236128:2113] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-12-24T02:11:04.638949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241252348236110:2113] [node 4] ICP30 unregister session Session# [6:7587241252348236128:2113] VirtualId# [6:7587241252348236126:0]
2025-12-24T02:11:04.638956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241252348236128:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.639013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:04.639039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:04.690710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253064714987:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.690725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253064714988:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.689170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251970619510:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:04.689179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251970619511:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:04.691052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241252887072940:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.691060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241252887072941:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.702508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241251147891802:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.702572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241251147891803:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.702587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241251147891804:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.695130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241254359392999:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.695139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241254359393000:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.695144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241254359393001:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.695196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241251994608111:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:04.695229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241251994608112:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:04.695234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241251994608113:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:04.746192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241254359393030:2077] [node 4] ICP01 ready to work
2025-12-24T02:11:04.746880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241254359393036:2080] [node 6] ICP01 ready to work
2025-12-24T02:11:04.747010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241251147891828:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:04.747125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241251147891832:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:04.747752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241251994608132:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:04.748360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241251994608139:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:04.766855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241251970619510:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18739038ns) {Configure:104(18738218ns) {ProcessConfigured:135(18335326ns) {StartInitialHandshake:157(18334584ns) {DropHandshakes:470(601ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(1411ns) {SwitchToState:179(603ns)}}}}}
2025-12-24T02:11:04.767176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241251994608139:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18772856ns) {Configure:104(18772443ns) {ProcessConfigured:135(18563922ns) {StartInitialHandshake:157(18563430ns) {DropHandshakes:470(400ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(40ns)} GenerateSessionVirtualId:988(73ns) PrepareNewSessionHandshake:477(1166ns) {SwitchToState:179(65ns)}}}}}
2025-12-24T02:11:04.767446Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241254359393001:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20528586ns) {Configure:104(20527479ns) {ProcessConfigured:135(16444250ns) {StartInitialHandshake:157(16443774ns) {DropHandshakes:470(505ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:477(712ns) {SwitchToState:179(126ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.769207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393065:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.770904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241254359393000:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23941231ns) {Configure:104(23940432ns) {ProcessConfigured:135(19600045ns) {StartInitialHandshake:157(19598911ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1328ns) {SwitchToState:179(692ns)}}}}}
2025-12-24T02:11:04.771271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241254359392999:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24927054ns) {Configure:104(24920101ns) {ProcessConfigured:135(19817314ns) {StartInitialHandshake:157(19816620ns) {DropHandshakes:470(302ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(1108ns) {SwitchToState:179(667ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.773086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241251147891803:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26159046ns) {Configure:104(26157600ns) {ProcessConfigured:135(21323796ns) {StartInitialHandshake:157(21322380ns) {DropHandshakes:470(566ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(1453ns) {SwitchToState:179(748ns)}}}}}
2025-12-24T02:11:04.773325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891872:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.777151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241251147891828:2068] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29978684ns) {Configure:104(29978111ns) {ProcessConfigured:135(25187188ns) {StartInitialHandshake:157(25186146ns) {DropHandshakes:470(1204ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(463ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(1056ns) {SwitchToState:179(524ns)}}}}}
2025-12-24T02:11:04.780161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608169:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.781596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251970619579:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:04.783134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241254359393065:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.782266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251147891872:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.790046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241254359393030:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43751472ns) {Configure:104(43750190ns) {ProcessConfigured:135(42275146ns) {StartInitialHandshake:157(42273865ns) {DropHandshakes:470(946ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1069ns) {SwitchToState:179(680ns)}}}}}
2025-12-24T02:11:04.790109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241254359393030:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.787040Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241252887072941:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31708071ns) {Configure:104(31706834ns) {ProcessConfigured:135(31312977ns) {StartInitialHandshake:157(31312312ns) {DropHandshakes:470(650ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(93ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1296ns) {SwitchToState:179(835ns)}}}}}
2025-12-24T02:11:04.787265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241252887073008:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:04.788771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241252887072940:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32405316ns) {Configure:104(32404397ns) {ProcessConfigured:135(32141355ns) {StartInitialHandshake:157(32141098ns) {DropHandshakes:470(343ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(59ns)}}}}}
2025-12-24T02:11:04.788114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253064714988:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32206311ns) {Configure:104(32205014ns) {ProcessConfigured:135(31843318ns) {StartInitialHandshake:157(31842888ns) {DropHandshakes:470(417ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1194ns) {SwitchToState:179(927ns)}}}}}
2025-12-24T02:11:04.791045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241251147891804:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26199678ns) {Configure:104(26199048ns) {ProcessConfigured:135(21609867ns) {StartInitialHandshake:157(21609117ns) {DropHandshakes:470(228ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(770ns) {SwitchToState:179(163ns)}}}}}
2025-12-24T02:11:04.790794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241251994608132:2064] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(42971651ns) {Configure:104(42971034ns) {ProcessConfigured:135(42755163ns) {StartInitialHandshake:157(42754893ns) {DropHandshakes:470(287ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(1817ns) {SwitchToState:179(862ns)}}}}}
2025-12-24T02:11:04.802938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241251147891802:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21955486ns) {Configure:104(21954293ns) {ProcessConfigured:135(21585826ns) {StartInitialHandshake:157(21584864ns) {DropHandshakes:470(924ns) {DropIncomingHandshake:442(135ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(1268ns) {SwitchToState:179(824ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.803185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241251970619579:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21515590ns) {Configure:104(21515121ns) {ProcessConfigured:135(21212104ns) {StartInitialHandshake:157(21211820ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:477(830ns) {SwitchToState:179(114ns)}}}}}
2025-12-24T02:11:04.803667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073015:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.803741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073016:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.803774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073017:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.803786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073018:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.803799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073014:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.804081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241251994608111:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21152489ns) {Configure:104(21151687ns) {ProcessConfigured:135(20776764ns) {StartInitialHandshake:157(20776401ns) {DropHandshakes:470(501ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:477(904ns) {SwitchToState:179(563ns)}}}}}
2025-12-24T02:11:04.806069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241251994608112:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23005122ns) {Configure:104(23004428ns) {ProcessConfigured:135(22687412ns) {StartInitialHandshake:157(22686926ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1524ns) {SwitchToState:179(693ns)}}}}}
2025-12-24T02:11:04.805721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073015:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.805760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073016:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.807046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073014:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.807072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241252887072940:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.808217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073017:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.808694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608166:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.808713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241251994608166:2096] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.806311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715059:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.806389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715060:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.806422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715061:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.806457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715062:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.806978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715060:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.808107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715061:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.813343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608169:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.815418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241251147891832:2070] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(68107076ns) {Configure:104(68098466ns) {ProcessConfigured:135(63262655ns) {StartInitialHandshake:157(63261638ns) {DropHandshakes:470(500ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(410ns) PrepareNewSessionHandshake:477(1077ns) {SwitchToState:179(577ns)}}}}}
2025-12-24T02:11:04.815485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241251147891832:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.822097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251970619585:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15130
2025-12-24T02:11:04.822129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251970619586:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15130
2025-12-24T02:11:04.822963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253064715051:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:04.822973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241254359393050:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.822985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241254359393050:2091] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.826950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715062:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.826770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241252887073004:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:04.827081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073018:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.822834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619586:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.822907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619577:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.822914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251970619577:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.826730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251970619582:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:04.827104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241252887072941:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.827270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715054:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.827277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253064715054:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.827872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073024:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.828010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253064715054:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.828173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241252887073018:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.828270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241252887072941:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.828289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.828295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241252887072941:2049] [node 2] ICP22 created new session: [3:7587241252887073025:2049]
2025-12-24T02:11:04.828300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241252887073025:2049] [node 2] ICS09 handshake done sender: [3:7587241252887073018:2117] self: [3:7587241252887073022:0] peer: [2:7587241253064715048:0] socket: 363 qp: -1
2025-12-24T02:11:04.828307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241252887073025:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.828326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.829012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253064714988:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.829028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.829032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253064714988:2049] [node 3] ICP22 created new session: [2:7587241253064715066:2049]
2025-12-24T02:11:04.829038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253064715066:2049] [node 3] ICS09 handshake done sender: [2:7587241253064715054:2110] self: [2:7587241253064715048:0] peer: [3:7587241252887073022:0] socket: 351 qp: -1
2025-12-24T02:11:04.829042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253064715066:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.829048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.830988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241251970619581:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:04.830494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253064715052:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:04.830867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241252887073005:2101] [node 6] ICP01 ready to work
2025-12-24T02:11:04.831310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241253064715053:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.831548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619574:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.831553Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251970619574:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.831433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715059:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.831454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241253064714987:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.833863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241252887073028:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17944
2025-12-24T02:11:04.835584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251970619577:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.835746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241252887073016:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.835837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241252887072940:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.835851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.835859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241252887072940:2048] [node 1] ICP22 created new session: [3:7587241252887073029:2048]
2025-12-24T02:11:04.835866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241252887073029:2048] [node 1] ICS09 handshake done sender: [3:7587241252887073016:2115] self: [3:7587241252887073020:0] peer: [1:7587241251970619576:0] socket: 361 qp: -1
2025-12-24T02:11:04.835873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241252887073029:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.835909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.839026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619585:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.839055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251970619511:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.839106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.839118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251970619511:2049] [node 3] ICP22 created new session: [1:7587241251970619591:2049]
2025-12-24T02:11:04.839124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251970619591:2049] [node 3] ICS09 handshake done sender: [1:7587241251970619577:2111] self: [1:7587241251970619576:0] peer: [3:7587241252887073020:0] socket: 343 qp: -1
2025-12-24T02:11:04.839135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251970619591:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:04.839166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.839543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241254359393050:2091] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.842727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241254359393036:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.842750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.842759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241254359393036:2080] [node 6] ICP22 created new session: [5:7587241254359393080:2080]
2025-12-24T02:11:04.842765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241254359393080:2080] [node 6] ICS09 handshake done sender: [5:7587241254359393050:2091] self: [5:7587241254359393042:0] peer: [6:7587241251147891881:0] socket: 333 qp: -1
2025-12-24T02:11:04.842774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241254359393080:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:04.842788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.843275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608191:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.846291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608191:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.846341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241251994608113:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.859034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891884:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.859472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891882:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.859526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891883:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.859686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251147891872:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.859159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253064715052:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28580568ns) {Configure:104(28579764ns) {ProcessConfigured:135(28228496ns) {StartInitialHandshake:157(28227861ns) {DropHandshakes:470(402ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(1726ns) {SwitchToState:179(746ns)}}}}}
2025-12-24T02:11:04.859818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241251147891832:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.859846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.859860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241251147891832:2070] [node 5] ICP22 created new session: [6:7587241251147891885:2070]
2025-12-24T02:11:04.859867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251147891885:2070] [node 5] ICS09 handshake done sender: [6:7587241251147891872:2111] self: [6:7587241251147891881:0] peer: [5:7587241254359393042:0] socket: 341 qp: -1
2025-12-24T02:11:04.859875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251147891885:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:04.859896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:04.860075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241252887073005:2101] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29104351ns) {Configure:104(29103852ns) {ProcessConfigured:135(28770002ns) {StartInitialHandshake:157(28751309ns) {DropHandshakes:470(553ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(1080ns) {SwitchToState:179(497ns)}} IncomingHandshake:261(7884ns) {DropIncomingHandshake:442(18ns) PrepareNewSessionHandshake:477(165ns) {SwitchToState:179(105ns)} GenerateSessionVirtualId:988(308ns) IssueIncomingHandshakeReply:193(383ns)}}}}
2025-12-24T02:11:04.860686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241251970619581:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29627059ns) {Configure:104(29624687ns) {ProcessConfigured:135(29308766ns) {StartInitialHandshake:157(29308425ns) {DropHandshakes:470(485ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(699ns) {SwitchToState:179(198ns)}}}}}
2025-12-24T02:11:04.861710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251970619574:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.861777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.861942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251147891883:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.861970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241251147891802:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.861931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251970619510:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.862204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715076:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.861952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.861959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251970619510:2048] [node 2] ICP22 created new session: [1:7587241251970619596:2048]
2025-12-24T02:11:04.861964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251970619596:2048] [node 2] ICS09 handshake done sender: [1:7587241251970619574:2110] self: [1:7587241251970619570:0] peer: [2:7587241253064715069:0] socket: 335 qp: -1
2025-12-24T02:11:04.861972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251970619596:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:04.861980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.862399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253064715059:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.862459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619583:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.862626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241253064714987:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241253064715059:2112] held: no
2025-12-24T02:11:04.862468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251970619583:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.862652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241253064714987:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.862659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.862665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253064714987:2048] [node 1] ICP22 created new session: [2:7587241253064715077:2048]
2025-12-24T02:11:04.862670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253064715077:2048] [node 1] ICS09 handshake done sender: [2:7587241253064715059:2112] self: [2:7587241253064715069:0] peer: [1:7587241251970619570:0] socket: 348 qp: -1
2025-12-24T02:11:04.862676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253064715077:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.862707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.863084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073010:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.863096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241252887073010:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.863186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251147891882:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.863238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241251147891828:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.862848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251994608166:2096] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.869448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241251970619511:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(30358072ns) {HandleHandshakeStatus:329(30357100ns) {DropIncomingHandshake:442(29971ns) DropOutgoingHandshake:457(223ns) SwitchToState:179(259ns) ProcessPendingSessionEvents:495(26540624ns) {ForwardSessionEventToSession:612(26527543ns) ForwardSessionEventToSession:612(1862ns) ForwardSessionEventToSession:612(918ns) ForwardSessionEventToSession:612(477ns) ForwardSessionEventToSession:612(922ns) ForwardSessionEventToSession:612(936ns) ForwardSessionEventToSession:612(977ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(432ns) ForwardSessionEventToSession:612(929ns)}}}
2025-12-24T02:11:04.871790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241253064715053:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40377362ns) {Configure:104(40376893ns) {ProcessConfigured:135(40117585ns) {StartInitialHandshake:157(40095099ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(448ns) PrepareNewSessionHandshake:477(1031ns) {SwitchToState:179(308ns)}} IncomingHandshake:261(8566ns) {DropIncomingHandshake:442(305ns) PrepareNewSessionHandshake:477(141ns) {SwitchToState:179(106ns)} GenerateSessionVirtualId:988(176ns) IssueIncomingHandshakeReply:193(318ns)}}}}
2025-12-24T02:11:04.872361Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241252887072940:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36492800ns) {HandleHandshakeStatus:329(36492241ns) {DropIncomingHandshake:442(159ns) DropOutgoingHandshake:457(2174ns) SwitchToState:179(141ns) ProcessPendingSessionEvents:495(28836485ns) {ForwardSessionEventToSession:612(28823360ns) ForwardSessionEventToSession:612(1892ns) ForwardSessionEventToSession:612(1205ns) ForwardSessionEventToSession:612(674ns) ForwardSessionEventToSession:612(867ns) ForwardSessionEventToSession:612(1205ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(980ns) ForwardSessionEventToSession:612(766ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.875930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891891:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.877158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891892:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.877344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251147891888:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.877374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241251147891804:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.877338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241251970619582:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50521142ns) {Configure:104(50520117ns) {ProcessConfigured:135(50187954ns) {StartInitialHandshake:157(50187074ns) {DropHandshakes:470(965ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(669ns) {SwitchToState:179(377ns)}}}}}
2025-12-24T02:11:04.877412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251970619583:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.878125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241252887073010:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.878232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241252887073008:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.878257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.878263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241252887073008:2104] [node 4] ICP22 created new session: [3:7587241252887073038:2104]
2025-12-24T02:11:04.878268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241252887073038:2104] [node 4] ICS09 handshake done sender: [3:7587241252887073010:2111] self: [3:7587241252887073009:0] peer: [4:7587241251994608192:0] socket: 350 qp: -1
2025-12-24T02:11:04.878266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073034:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.878272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241252887073038:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:04.878277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241252887073034:2121] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.878290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.878202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241254359393036:2080] [node 6] ICP35 event processing took too much time PendingConnection:279(35451733ns) {HandleHandshakeStatus:329(35451015ns) {DropIncomingHandshake:442(163ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(131ns) ProcessPendingSessionEvents:495(11402682ns) {ForwardSessionEventToSession:612(11391446ns) ForwardSessionEventToSession:612(1588ns) ForwardSessionEventToSession:612(863ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(614ns) ForwardSessionEventToSession:612(1071ns) ForwardSessionEventToSession:612(862ns) ForwardSessionEventToSession:612(498ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(694ns)}}}
2025-12-24T02:11:04.876683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608194:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.876833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608186:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.876842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241251994608186:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.877062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241251994608139:2070] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.877080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.877090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241251994608139:2070] [node 5] ICP22 created new session: [4:7587241251994608196:2070]
2025-12-24T02:11:04.877096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251994608196:2070] [node 5] ICS09 handshake done sender: [4:7587241251994608166:2096] self: [4:7587241251994608140:0] peer: [5:7587241254359393078:0] socket: 336 qp: -1
2025-12-24T02:11:04.877104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251994608196:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:04.877113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.879065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241252887073034:2121] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.879179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241252887073005:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.879201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.879207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241252887073005:2101] [node 6] ICP22 created new session: [3:7587241252887073041:2101]
2025-12-24T02:11:04.879228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241252887073041:2101] [node 6] ICS09 handshake done sender: [3:7587241252887073034:2121] self: [3:7587241252887073027:0] peer: [6:7587241251147891893:0] socket: 379 qp: -1
2025-12-24T02:11:04.879235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241252887073041:2101] [node 6] ICS10 traffic start
2025-12-24T02:11:04.879242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.879076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891894:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.879279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251147891883:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.879383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251147891888:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.879418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241251147891802:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.879428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.879434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241251147891802:2048] [node 1] ICP22 created new session: [6:7587241251147891895:2048]
2025-12-24T02:11:04.879440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251147891895:2048] [node 1] ICS09 handshake done sender: [6:7587241251147891883:2117] self: [6:7587241251147891889:0] peer: [1:7587241251970619580:0] socket: 375 qp: -1
2025-12-24T02:11:04.879446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251147891895:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.879452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.879677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241251147891804:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.879689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.879694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241251147891804:2050] [node 3] ICP22 created new session: [6:7587241251147891897:2050]
2025-12-24T02:11:04.879700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251147891897:2050] [node 3] ICS09 handshake done sender: [6:7587241251147891888:2119] self: [6:7587241251147891893:0] peer: [3:7587241252887073027:0] socket: 368 qp: -1
2025-12-24T02:11:04.879706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251147891897:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.879712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.882306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715074:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.882315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253064715074:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.883061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241251147891891:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.883095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241251147891803:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.883002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253064715074:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.883223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715079:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.883226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253064715079:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.882864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251970619602:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15130
2025-12-24T02:11:04.882922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251970619603:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15130
2025-12-24T02:11:04.882957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241251970619604:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15130
2025-12-24T02:11:04.883073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251970619579:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.883083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.883091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251970619579:2108] [node 6] ICP22 created new session: [1:7587241251970619605:2108]
2025-12-24T02:11:04.883097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251970619605:2108] [node 6] ICS09 handshake done sender: [1:7587241251970619583:2112] self: [1:7587241251970619580:0] peer: [6:7587241251147891889:0] socket: 359 qp: -1
2025-12-24T02:11:04.883105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251970619605:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:04.883138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.883526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253064715052:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.883537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.883542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253064715052:2109] [node 4] ICP22 created new session: [2:7587241253064715083:2109]
2025-12-24T02:11:04.883547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253064715083:2109] [node 4] ICS09 handshake done sender: [2:7587241253064715074:2117] self: [2:7587241253064715068:0] peer: [4:7587241251994608199:0] socket: 377 qp: -1
2025-12-24T02:11:04.883551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253064715083:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:04.883559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.884287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619604:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.884316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619602:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.882105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608194:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.882166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241251994608112:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.882322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251994608186:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.883634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241251994608132:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.883645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.883648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241251994608132:2064] [node 6] ICP22 created new session: [4:7587241251994608200:2064]
2025-12-24T02:11:04.883652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251994608200:2064] [node 6] ICS09 handshake done sender: [4:7587241251994608186:2113] self: [4:7587241251994608136:0] peer: [6:7587241251147891890:0] socket: 355 qp: -1
2025-12-24T02:11:04.883655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251994608200:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:04.883660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.883919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241251994608113:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241251994608191:2116] held: no
2025-12-24T02:11:04.884089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608204:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.885803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393083:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.884815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891900:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.887806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241251147891828:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26044) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241251147891882:2116] held: no
2025-12-24T02:11:04.888169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251147891882:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.890730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241251970619579:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:04.891083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241254359393083:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.891128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241254359393000:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.894311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393086:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.894328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393087:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.894401Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241254359393065:2104] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.890933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608202:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.890994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608203:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.893631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241251994608202:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.893643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251994608194:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.894104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251994608191:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.894174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241251994608112:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.894185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.894190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241251994608112:2049] [node 2] ICP22 created new session: [4:7587241251994608206:2049]
2025-12-24T02:11:04.894197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251994608206:2049] [node 2] ICS09 handshake done sender: [4:7587241251994608194:2117] self: [4:7587241251994608199:0] peer: [2:7587241253064715068:0] socket: 384 qp: -1
2025-12-24T02:11:04.894203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251994608206:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.894221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.894584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241251994608111:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.894611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241251994608113:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.894620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.894626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241251994608113:2050] [node 3] ICP22 created new session: [4:7587241251994608209:2050]
2025-12-24T02:11:04.894632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251994608209:2050] [node 3] ICS09 handshake done sender: [4:7587241251994608191:2116] self: [4:7587241251994608192:0] peer: [3:7587241252887073009:0] socket: 354 qp: -1
2025-12-24T02:11:04.894637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251994608209:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.894642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.894930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253064715079:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:04.894969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241251147891828:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.894983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.894988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241251147891828:2068] [node 4] ICP22 created new session: [6:7587241251147891901:2068]
2025-12-24T02:11:04.894992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251147891901:2068] [node 4] ICS09 handshake done sender: [6:7587241251147891882:2116] self: [6:7587241251147891890:0] peer: [4:7587241251994608136:0] socket: 358 qp: -1
2025-12-24T02:11:04.894995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251147891901:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:04.895005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.895467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241252887073031:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.895480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241252887073031:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.895516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241253064715071:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.895526Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241253064715071:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.895624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253064715053:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.895640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:04.895645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253064715053:2107] [node 6] ICP22 created new session: [2:7587241253064715087:2107]
2025-12-24T02:11:04.895650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253064715087:2107] [node 6] ICS09 handshake done sender: [2:7587241253064715079:2119] self: [2:7587241253064715070:0] peer: [6:7587241251147891899:0] socket: 352 qp: -1
2025-12-24T02:11:04.895655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253064715087:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:04.895659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:04.894717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241254359393030:2077] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.894751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.894766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241254359393030:2077] [node 4] ICP22 created new session: [5:7587241254359393088:2077]
2025-12-24T02:11:04.894772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241254359393088:2077] [node 4] ICS09 handshake done sender: [5:7587241254359393065:2104] self: [5:7587241254359393078:0] peer: [4:7587241251994608140:0] socket: 338 qp: -1
2025-12-24T02:11:04.894778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241254359393088:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:04.894784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.894839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241254359393087:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.894897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393085:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.894952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241254359392999:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.895088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241254359393085:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.895100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241254359393001:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:04.899054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619600:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.899066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251970619600:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.903391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241251147891803:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25751) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241251147891891:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.906541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241251147891903:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33802
2025-12-24T02:11:04.906905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241251147891891:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.906762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241251970619593:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:04.906777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241251970619593:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:04.906719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241253064715086:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38756
2025-12-24T02:11:04.906985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241251147891803:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.907001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.907008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241251147891803:2049] [node 2] ICP22 created new session: [6:7587241251147891904:2049]
2025-12-24T02:11:04.907016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241251147891904:2049] [node 2] ICS09 handshake done sender: [6:7587241251147891891:2120] self: [6:7587241251147891899:0] peer: [2:7587241253064715070:0] socket: 386 qp: -1
2025-12-24T02:11:04.907024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241251147891904:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.907035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.908399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241253064715071:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.908496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241253064715051:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.908514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.908519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241253064715051:2106] [node 5] ICP22 created new session: [2:7587241253064715090:2106]
2025-12-24T02:11:04.908526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241253064715090:2106] [node 5] ICS09 handshake done sender: [2:7587241253064715071:2116] self: [2:7587241253064715063:0] peer: [5:7587241254359393084:0] socket: 371 qp: -1
2025-12-24T02:11:04.908531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241253064715090:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:04.908538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.908867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393093:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.910885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241252887073031:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.911034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241252887073004:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.911057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.911065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241252887073004:2099] [node 5] ICP22 created new session: [3:7587241252887073044:2099]
2025-12-24T02:11:04.911071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241252887073044:2099] [node 5] ICS09 handshake done sender: [3:7587241252887073031:2120] self: [3:7587241252887073023:0] peer: [5:7587241254359393091:0] socket: 364 qp: -1
2025-12-24T02:11:04.911078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241252887073044:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:04.911092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.911096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241254359393030:2077] [node 4] ICP35 event processing took too much time PendingConnection:279(16342765ns) {HandleHandshakeStatus:329(16342298ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(1811ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(12300843ns) {ForwardSessionEventToSession:612(12289685ns) ForwardSessionEventToSession:612(1423ns) ForwardSessionEventToSession:612(1052ns) ForwardSessionEventToSession:612(628ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(1050ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(729ns) ForwardSessionEventToSession:612(876ns) ForwardSessionEventToSession:612(1064ns)}}}
2025-12-24T02:11:04.911923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241254359393000:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25751) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241254359393083:2116] held: no
2025-12-24T02:11:04.912941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241254359393001:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241254359393085:2117] held: no
2025-12-24T02:11:04.913523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251970619600:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:04.913423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393092:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.913599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241254359393083:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:04.913950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241254359393000:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.913966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:04.913976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241254359393000:2049] [node 2] ICP22 created new session: [5:7587241254359393095:2049]
2025-12-24T02:11:04.913982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241254359393095:2049] [node 2] ICS09 handshake done sender: [5:7587241254359393083:2116] self: [5:7587241254359393084:0] peer: [2:7587241253064715063:0] socket: 373 qp: -1
2025-12-24T02:11:04.913988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241254359393095:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:04.913993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:04.914268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241251970619593:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:04.914434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251970619581:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.914446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:04.914452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251970619581:2107] [node 4] ICP22 created new session: [1:7587241251970619611:2107]
2025-12-24T02:11:04.914456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251970619611:2107] [node 4] ICS09 handshake done sender: [1:7587241251970619593:2115] self: [1:7587241251970619589:0] peer: [4:7587241251994608208:0] socket: 357 qp: -1
2025-12-24T02:11:04.914459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251970619611:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:04.914464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:04.914418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241254359393085:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:04.917635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241254359393094:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10269
2025-12-24T02:11:04.917739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241254359393001:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.917754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:04.917762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241254359393001:2050] [node 3] ICP22 created new session: [5:7587241254359393097:2050]
2025-12-24T02:11:04.917768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241254359393097:2050] [node 3] ICS09 handshake done sender: [5:7587241254359393085:2117] self: [5:7587241254359393091:0] peer: [3:7587241252887073023:0] socket: 397 qp: -1
2025-12-24T02:11:04.917774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241254359393097:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:04.917781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:04.917985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241254359393087:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.918096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241254359392999:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.918109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.918115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241254359392999:2048] [node 1] ICP22 created new session: [5:7587241254359393099:2048]
2025-12-24T02:11:04.918123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241254359393099:2048] [node 1] ICS09 handshake done sender: [5:7587241254359393087:2119] self: [5:7587241254359393090:0] peer: [1:7587241251970619588:0] socket: 399 qp: -1
2025-12-24T02:11:04.918128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241254359393099:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.918139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.917501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241251970619582:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:04.917523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:04.917531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241251970619582:2109] [node 5] ICP22 created new session: [1:7587241251970619613:2109]
2025-12-24T02:11:04.917537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241251970619613:2109] [node 5] ICS09 handshake done sender: [1:7587241251970619600:2116] self: [1:7587241251970619588:0] peer: [5:7587241254359393090:0] socket: 390 qp: -1
2025-12-24T02:11:04.917542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241251970619613:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:04.917550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:04.923067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241251994608211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48229
2025-12-24T02:11:04.923434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241251994608202:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:04.923504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241251994608111:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:04.923522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:04.923527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241251994608111:2048] [node 1] ICP22 created new session: [4:7587241251994608212:2048]
2025-12-24T02:11:04.923534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241251994608212:2048] [node 1] ICS09 handshake done sender: [4:7587241251994608202:2118] self: [4:7587241251994608208:0] peer: [1:7587241251970619589:0] socket: 385 qp: -1
2025-12-24T02:11:04.923540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241251994608212:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:04.923546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:04.946347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:04.946446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241253064715077:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-12-24T02:11:04.946454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241253064714987:2048] [node 1] ICP30 unregister session Session# [2:7587241253064715077:2048] VirtualId# [2:7587241253064715069:0]
2025-12-24T02:11:04.946460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241253064715077:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.946492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.946509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.946826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:04.946886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241252887073029:2048] [node 1] ICS07 socket disconnect 361 reason# EndOfStream
2025-12-24T02:11:04.946890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241252887073029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.946912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.946927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241252887073029:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.948986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241252887072940:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:04.948998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241252887073029:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.949270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241252887072940:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.949281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241252887073029:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.949285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241252887072940:2048] [node 1] ICP30 unregister session Session# [3:7587241252887073029:2048] VirtualId# [3:7587241252887073020:0]
2025-12-24T02:11:04.949300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.949305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241252887072940:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.949307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.958914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:04.959006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241254359393099:2048] [node 1] ICS07 socket disconnect 399 reason# EndOfStream
2025-12-24T02:11:04.959011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241254359393099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.959066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.959086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241254359393099:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.959133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:04.959191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251147891895:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-12-24T02:11:04.959196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241251147891802:2048] [node 1] ICP30 unregister session Session# [6:7587241251147891895:2048] VirtualId# [6:7587241251147891889:0]
2025-12-24T02:11:04.959200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251147891895:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.959238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.959265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.959768Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241254359392999:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.959780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241254359393099:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.959786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241254359392999:2048] [node 1] ICP30 unregister session Session# [5:7587241254359393099:2048] VirtualId# [5:7587241254359393090:0]
2025-12-24T02:11:04.959802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.959807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241254359392999:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.959809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.958746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:04.958844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241251994608212:2048] [node 1] ICS07 socket disconnect 385 reason# EndOfStream
2025-12-24T02:11:04.958851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241251994608212:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.958898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:04.958917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241251994608212:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:04.962858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241252887072940:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.965782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:04.966014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241252887073025:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-12-24T02:11:04.966026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241252887072941:2049] [node 2] ICP30 unregister session Session# [3:7587241252887073025:2049] VirtualId# [3:7587241252887073022:0]
2025-12-24T02:11:04.966032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241252887073025:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.966088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.966112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.965903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241251994608111:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.965923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241251994608212:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.965930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241251994608111:2048] [node 1] ICP30 unregister session Session# [4:7587241251994608212:2048] VirtualId# [4:7587241251994608208:0]
2025-12-24T02:11:04.965948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:04.965956Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241251994608111:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.965959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6715) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.970746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241254359392999:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.973324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:04.973413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251147891904:2049] [node 2] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:04.973422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241251147891803:2049] [node 2] ICP30 unregister session Session# [6:7587241251147891904:2049] VirtualId# [6:7587241251147891899:0]
2025-12-24T02:11:04.973430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251147891904:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.973481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.973504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.973765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:04.973841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241254359393095:2049] [node 2] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:04.973848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241254359393000:2049] [node 2] ICP30 unregister session Session# [5:7587241254359393095:2049] VirtualId# [5:7587241254359393084:0]
2025-12-24T02:11:04.973854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241254359393095:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.973905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.973929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.973785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:04.973849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241251994608206:2049] [node 2] ICS01 socket: 384 reason# EndOfStream
2025-12-24T02:11:04.973855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241251994608112:2049] [node 2] ICP30 unregister session Session# [4:7587241251994608206:2049] VirtualId# [4:7587241251994608199:0]
2025-12-24T02:11:04.973859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241251994608206:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.973896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:04.973918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:04.990063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241251994608111:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:04.992514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:04.992601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241251147891897:2050] [node 3] ICS07 socket disconnect 368 reason# EndOfStream
2025-12-24T02:11:04.992608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251147891897:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.992662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.992687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241251147891897:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:04.993852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:04.993918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241254359393097:2050] [node 3] ICS01 socket: 397 reason# EndOfStream
2025-12-24T02:11:04.993926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241254359393001:2050] [node 3] ICP30 unregister session Session# [5:7587241254359393097:2050] VirtualId# [5:7587241254359393091:0]
2025-12-24T02:11:04.993932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241254359393097:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:04.993974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:04.993995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.994881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241251147891804:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:04.994895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251147891897:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:04.994900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241251147891804:2050] [node 3] ICP30 unregister session Session# [6:7587241251147891897:2050] VirtualId# [6:7587241251147891893:0]
2025-12-24T02:11:04.994915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:04.994920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241251147891804:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:04.994922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.005416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241251147891804:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:05.018239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:05.018334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241251147891901:2068] [node 4] ICS01 socket: 358 reason# EndOfStream
2025-12-24T02:11:05.018343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241251147891828:2068] [node 4] ICP30 unregister session Session# [6:7587241251147891901:2068] VirtualId# [6:7587241251147891890:0]
2025-12-24T02:11:05.018349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241251147891901:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.018401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:05.018425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:05.053757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241259211409736:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.053771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241259211409737:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:05.062911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241255569646870:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:05.062926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241255569646871:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:05.064774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257457858374:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.064788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257457858375:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.064793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257457858376:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.070893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241257055788287:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.070911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241257055788288:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.070917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241257055788289:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.070885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241257139389536:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.070906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241257139389537:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.074639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241256037137966:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.074650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241256037137967:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.074655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241256037137968:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.100472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241255569646927:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:05.100644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241255569646928:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.101769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858416:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
2025-12-24T02:11:05.102594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241259211409759:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:05.101891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241255569646936:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:05.110431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241257139389579:2072] [node 5] ICP01 ready to work
2025-12-24T02:11:05.110525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241257139389580:2077] [node 4] ICP01 ready to work
2025-12-24T02:11:05.110586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241257139389581:2084] [node 6] ICP01 ready to work
2025-12-24T02:11:05.118341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241256037138030:2092] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.127705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241257055788351:2090] [node 4] ICP01 ready to work
2025-12-24T02:11:05.127800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241257055788352:2092] [node 6] ICP01 ready to work
2025-12-24T02:11:05.134912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241259211409759:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32240859ns) {Configure:104(32240112ns) {ProcessConfigured:135(31870017ns) {StartInitialHandshake:157(31869440ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(1437ns) {SwitchToState:179(743ns)}}}}}
2025-12-24T02:11:05.135091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241259211409802:2071] [node 5] ICP01 ready to work
2025-12-24T02:11:05.135159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241259211409803:2074] [node 6] ICP01 ready to work
2025-12-24T02:11:05.137531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788350:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.142837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241255569646870:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39292177ns) {Configure:104(39291382ns) {ProcessConfigured:135(39053922ns) {StartInitialHandshake:157(39053475ns) {DropHandshakes:470(476ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(1585ns) {SwitchToState:179(725ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.144026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241256037137966:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25614752ns) {Configure:104(25612638ns) {ProcessConfigured:135(25253136ns) {StartInitialHandshake:157(25252646ns) {DropHandshakes:470(803ns) {DropIncomingHandshake:442(130ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(823ns) {SwitchToState:179(504ns)}}}}}
2025-12-24T02:11:05.144552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241256037138035:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:05.144671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138029:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.145659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138036:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.145811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241257457858374:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38952430ns) {Configure:104(38951385ns) {ProcessConfigured:135(38524171ns) {StartInitialHandshake:157(38523915ns) {DropHandshakes:470(454ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1528ns) {SwitchToState:179(638ns)}}}}}
2025-12-24T02:11:05.145983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257457858440:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:05.146064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257457858441:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:05.147253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858442:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.147452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241257139389579:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(36817771ns) {Configure:104(36817343ns) {ProcessConfigured:135(36476290ns) {StartInitialHandshake:157(36476034ns) {DropHandshakes:470(621ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(108ns) PrepareNewSessionHandshake:477(1668ns) {SwitchToState:179(982ns)}}}}}
2025-12-24T02:11:05.148621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389610:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.148636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389611:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.158798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241257055788287:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21194639ns) {Configure:104(21192924ns) {ProcessConfigured:135(20878830ns) {StartInitialHandshake:157(20878072ns) {DropHandshakes:470(494ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(191ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:477(1546ns) {SwitchToState:179(854ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.165209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858454:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.166055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788364:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.166112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788365:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.166265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788363:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.165716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241255569646949:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52338
2025-12-24T02:11:05.165774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241255569646950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52338
2025-12-24T02:11:05.165822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241255569646951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52338
2025-12-24T02:11:05.169605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858454:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.169675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257457858376:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.171990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241259211409815:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8476
2025-12-24T02:11:05.172035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241259211409816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8476
2025-12-24T02:11:05.171107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389610:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.171137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389611:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.171164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389607:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.171167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241257139389607:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.171636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.174922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241256037138036:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.174962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241256037137967:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.175001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858442:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.175052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409807:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.175063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241259211409807:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.175639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409815:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.175704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409801:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.175706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241259211409801:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.176434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241259211409807:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.176565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138047:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.176722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241256037138036:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.176778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241256037137967:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.176796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.176801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241256037137967:2049] [node 2] ICP22 created new session: [6:7587241256037138048:2049]
2025-12-24T02:11:05.176807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241256037138048:2049] [node 2] ICS09 handshake done sender: [6:7587241256037138036:2113] self: [6:7587241256037138046:0] peer: [2:7587241259211409806:0] socket: 347 qp: -1
2025-12-24T02:11:05.176815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241256037138048:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.176825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.175076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788364:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.175121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788365:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.177592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646949:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.177625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646950:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.177651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646951:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.177695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241255569646952:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52338
2025-12-24T02:11:05.178311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241259211409801:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.178376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241256037138029:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.178399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241256037137966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.175358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858456:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
2025-12-24T02:11:05.175382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257457858375:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.175890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241257139389607:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.178504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646938:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.178509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241255569646938:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.179040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241255569646938:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.179112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241255569646936:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.179127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.179132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241255569646936:2106] [node 6] ICP22 created new session: [1:7587241255569646957:2106]
2025-12-24T02:11:05.179137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241255569646957:2106] [node 6] ICS09 handshake done sender: [1:7587241255569646938:2107] self: [1:7587241255569646937:0] peer: [6:7587241256037138051:0] socket: 340 qp: -1
2025-12-24T02:11:05.179141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241255569646957:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:05.179148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.178885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241257139389580:2077] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.178907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.178913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241257139389580:2077] [node 4] ICP22 created new session: [3:7587241257139389621:2077]
2025-12-24T02:11:05.178919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241257139389621:2077] [node 4] ICS09 handshake done sender: [3:7587241257139389607:2112] self: [3:7587241257139389606:0] peer: [4:7587241257457858455:0] socket: 360 qp: -1
2025-12-24T02:11:05.178926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241257139389621:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:05.178935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.179478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241259211409803:2074] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.179493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.179499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241259211409803:2074] [node 6] ICP22 created new session: [2:7587241259211409820:2074]
2025-12-24T02:11:05.179503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241259211409820:2074] [node 6] ICS09 handshake done sender: [2:7587241259211409807:2112] self: [2:7587241259211409806:0] peer: [6:7587241256037138046:0] socket: 344 qp: -1
2025-12-24T02:11:05.179509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241259211409820:2074] [node 6] ICS10 traffic start
2025-12-24T02:11:05.179516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.179686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241259211409759:2069] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.179695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.179700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241259211409759:2069] [node 4] ICP22 created new session: [2:7587241259211409822:2069]
2025-12-24T02:11:05.179703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241259211409822:2069] [node 4] ICS09 handshake done sender: [2:7587241259211409801:2110] self: [2:7587241259211409760:0] peer: [4:7587241257457858457:0] socket: 342 qp: -1
2025-12-24T02:11:05.179707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241259211409822:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:05.179711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.187155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241256037137966:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29388) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241256037138029:2110] held: no
2025-12-24T02:11:05.187275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138052:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.187539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788363:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.187569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788350:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.187712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241257055788289:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.187761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241257055788288:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.187771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241257055788287:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.188060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858416:2089] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.189279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409816:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.189318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409805:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.189321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241259211409805:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.194856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.194945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241256037138029:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.195059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241256037137966:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.195078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.195087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241256037137966:2048] [node 1] ICP22 created new session: [6:7587241256037138054:2048]
2025-12-24T02:11:05.195094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241256037138054:2048] [node 1] ICS09 handshake done sender: [6:7587241256037138029:2110] self: [6:7587241256037138051:0] peer: [1:7587241255569646937:0] socket: 341 qp: -1
2025-12-24T02:11:05.195102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241256037138054:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.195113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.195384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646932:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.195397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241255569646932:2105] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.196025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241255569646960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52338
2025-12-24T02:11:05.196194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646952:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.196493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.196617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241255569646932:2105] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.196805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241257055788350:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.196868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241257055788287:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.196883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.196889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241257055788287:2048] [node 1] ICP22 created new session: [5:7587241257055788374:2048]
2025-12-24T02:11:05.196896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241257055788374:2048] [node 1] ICS09 handshake done sender: [5:7587241257055788350:2110] self: [5:7587241257055788372:0] peer: [1:7587241255569646931:0] socket: 339 qp: -1
2025-12-24T02:11:05.196902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241257055788374:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.196908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.198059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389603:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.198074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241257139389603:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.198708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.198946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241256037138053:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.198983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241256037137968:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.199160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241255569646928:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.199182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.199189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241255569646928:2103] [node 5] ICP22 created new session: [1:7587241255569646963:2103]
2025-12-24T02:11:05.199194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241255569646963:2103] [node 5] ICS09 handshake done sender: [1:7587241255569646932:2105] self: [1:7587241255569646931:0] peer: [5:7587241257055788372:0] socket: 338 qp: -1
2025-12-24T02:11:05.199201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241255569646963:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:05.199207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.199238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.199328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389609:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.199332Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241257139389609:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.199939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241257139389603:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.200056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241257139389609:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.200127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241257139389579:2072] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.200140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.200147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241257139389579:2072] [node 5] ICP22 created new session: [3:7587241257139389628:2072]
2025-12-24T02:11:05.200152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241257139389628:2072] [node 5] ICS09 handshake done sender: [3:7587241257139389603:2110] self: [3:7587241257139389585:0] peer: [5:7587241257055788370:0] socket: 358 qp: -1
2025-12-24T02:11:05.200158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241257139389628:2072] [node 5] ICS10 traffic start
2025-12-24T02:11:05.200165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.200275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241257139389581:2084] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.200284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.200288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241257139389581:2084] [node 6] ICP22 created new session: [3:7587241257139389630:2084]
2025-12-24T02:11:05.200292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241257139389630:2084] [node 6] ICS09 handshake done sender: [3:7587241257139389609:2113] self: [3:7587241257139389608:0] peer: [6:7587241256037138056:0] socket: 357 qp: -1
2025-12-24T02:11:05.200296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241257139389630:2084] [node 6] ICS10 traffic start
2025-12-24T02:11:05.200301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.203108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.206944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241259211409827:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8476
2025-12-24T02:11:05.207134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389617:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.207172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389625:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.207269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241257139389579:2072] [node 5] ICP18 handshake (actor [3:7587241257139389617:2117]) is too old
2025-12-24T02:11:05.207286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241257139389537:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.207649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241257055788365:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.207619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409827:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.207680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241259211409805:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.207760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.207807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241259211409802:2071] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.207826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.207833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241259211409802:2071] [node 5] ICP22 created new session: [2:7587241259211409829:2071]
2025-12-24T02:11:05.207839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241259211409829:2071] [node 5] ICS09 handshake done sender: [2:7587241259211409805:2111] self: [2:7587241259211409804:0] peer: [5:7587241257055788371:0] socket: 343 qp: -1
2025-12-24T02:11:05.207845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241259211409829:2071] [node 5] ICS10 traffic start
2025-12-24T02:11:05.207855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.207910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241257055788289:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.207927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.207933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241257055788289:2050] [node 3] ICP22 created new session: [5:7587241257055788379:2050]
2025-12-24T02:11:05.207940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241257055788379:2050] [node 3] ICS09 handshake done sender: [5:7587241257055788365:2118] self: [5:7587241257055788370:0] peer: [3:7587241257139389585:0] socket: 370 qp: -1
2025-12-24T02:11:05.207947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241257055788379:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.207957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.208286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241256037137968:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241256037138053:2119] held: no
2025-12-24T02:11:05.208388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257457858454:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.208478Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241257457858376:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5048) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241257457858454:2117] held: no
2025-12-24T02:11:05.208500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257457858376:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.208508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.208514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257457858376:2050] [node 3] ICP22 created new session: [4:7587241257457858460:2050]
2025-12-24T02:11:05.208519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257457858460:2050] [node 3] ICS09 handshake done sender: [4:7587241257457858454:2117] self: [4:7587241257457858455:0] peer: [3:7587241257139389606:0] socket: 362 qp: -1
2025-12-24T02:11:05.208526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257457858460:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.208533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.208526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409811:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.208534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241259211409811:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.214837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241259211409811:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.214988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241259211409737:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.215012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.215021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241259211409737:2049] [node 3] ICP22 created new session: [2:7587241259211409833:2049]
2025-12-24T02:11:05.215029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241259211409833:2049] [node 3] ICS09 handshake done sender: [2:7587241259211409811:2114] self: [2:7587241259211409810:0] peer: [3:7587241257139389633:0] socket: 386 qp: -1
2025-12-24T02:11:05.215036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241259211409833:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.215046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.216687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646960:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.217891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138057:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.218364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241256037138053:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.218426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241256037137968:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.218442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.218450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241256037137968:2050] [node 3] ICP22 created new session: [6:7587241256037138058:2050]
2025-12-24T02:11:05.218457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241256037138058:2050] [node 3] ICS09 handshake done sender: [6:7587241256037138053:2119] self: [6:7587241256037138056:0] peer: [3:7587241257139389608:0] socket: 388 qp: -1
2025-12-24T02:11:05.218464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241256037138058:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.218472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.218934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241259211409835:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8476
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.219957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241257139389624:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.220049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241257139389537:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241257139389625:2119] held: no
2025-12-24T02:11:05.220130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241257139389536:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.220159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389634:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.220071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241257457858375:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241257457858442:2112] held: no
2025-12-24T02:11:05.220146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257457858374:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.220203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858458:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.220487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858459:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
2025-12-24T02:11:05.220546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858462:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
2025-12-24T02:11:05.220582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257457858442:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.220664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257457858375:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.220682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.220689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257457858375:2049] [node 2] ICP22 created new session: [4:7587241257457858466:2049]
2025-12-24T02:11:05.220695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257457858466:2049] [node 2] ICS09 handshake done sender: [4:7587241257457858442:2112] self: [4:7587241257457858457:0] peer: [2:7587241259211409760:0] socket: 351 qp: -1
2025-12-24T02:11:05.220702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257457858466:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.220708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.221449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241257055788288:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7201) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241257055788363:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.224833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138060:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.224887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138061:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.225023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646930:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.225036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241255569646930:2104] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.225741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241259211409835:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.225795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241259211409736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.230002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241257055788363:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.230093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241257139389625:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.230249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241257139389537:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.230265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.230273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241257139389537:2049] [node 2] ICP22 created new session: [3:7587241257139389636:2049]
2025-12-24T02:11:05.230279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241257139389636:2049] [node 2] ICS09 handshake done sender: [3:7587241257139389625:2119] self: [3:7587241257139389633:0] peer: [2:7587241259211409810:0] socket: 392 qp: -1
2025-12-24T02:11:05.230286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241257139389636:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.230294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.230620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788382:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.230825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646948:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.230840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241255569646948:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.231434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241255569646940:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.231440Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241255569646940:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.232151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241255569646930:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.232849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858462:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.232951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858459:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.233109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257457858469:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9995
2025-12-24T02:11:05.233214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241256037138060:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.233245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241256037138035:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.232500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241257055788288:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.232522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.232531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241257055788288:2049] [node 2] ICP22 created new session: [5:7587241257055788383:2049]
2025-12-24T02:11:05.232540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241257055788383:2049] [node 2] ICS09 handshake done sender: [5:7587241257055788363:2116] self: [5:7587241257055788371:0] peer: [2:7587241259211409804:0] socket: 368 qp: -1
2025-12-24T02:11:05.232549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241257055788383:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.232559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.239152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241259211409837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8476
2025-12-24T02:11:05.239277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241257139389638:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47123
2025-12-24T02:11:05.239498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858450:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.239513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241257457858450:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.239948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257457858416:2089] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.240193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257457858374:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.240208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.240215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257457858374:2048] [node 1] ICP22 created new session: [4:7587241257457858472:2048]
2025-12-24T02:11:05.240222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257457858472:2048] [node 1] ICS09 handshake done sender: [4:7587241257457858416:2089] self: [4:7587241257457858465:0] peer: [1:7587241255569646929:0] socket: 337 qp: -1
2025-12-24T02:11:05.240229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257457858472:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.240238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.241284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257457858450:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.241363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241257457858441:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.241374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.241380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257457858441:2096] [node 6] ICP22 created new session: [4:7587241257457858474:2096]
2025-12-24T02:11:05.241385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257457858474:2096] [node 6] ICS09 handshake done sender: [4:7587241257457858450:2116] self: [4:7587241257457858449:0] peer: [6:7587241256037138062:0] socket: 399 qp: -1
2025-12-24T02:11:05.241390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257457858474:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:05.241396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.243928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241255569646940:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.244046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241255569646927:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.244063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.244070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241255569646927:2102] [node 4] ICP22 created new session: [1:7587241255569646970:2102]
2025-12-24T02:11:05.244077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241255569646970:2102] [node 4] ICS09 handshake done sender: [1:7587241255569646930:2104] self: [1:7587241255569646929:0] peer: [4:7587241257457858465:0] socket: 336 qp: -1
2025-12-24T02:11:05.244083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241255569646970:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:05.244092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.244257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241255569646871:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.244269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.244273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241255569646871:2049] [node 3] ICP22 created new session: [1:7587241255569646972:2049]
2025-12-24T02:11:05.244278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241255569646972:2049] [node 3] ICS09 handshake done sender: [1:7587241255569646940:2108] self: [1:7587241255569646939:0] peer: [3:7587241257139389635:0] socket: 365 qp: -1
2025-12-24T02:11:05.244282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241255569646972:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.244287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.244919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241255569646948:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.245039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241255569646870:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.245055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.245061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241255569646870:2048] [node 2] ICP22 created new session: [1:7587241255569646974:2048]
2025-12-24T02:11:05.245066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241255569646974:2048] [node 2] ICS09 handshake done sender: [1:7587241255569646948:2114] self: [1:7587241255569646942:0] peer: [2:7587241259211409836:0] socket: 376 qp: -1
2025-12-24T02:11:05.245071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241255569646974:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:05.245076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.242890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788382:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.242963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241257055788351:2090] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.246945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257457858448:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.246956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241257457858448:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.248406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241259211409736:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29388) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241259211409835:2118] held: no
2025-12-24T02:11:05.248440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241259211409835:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.250032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241259211409736:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.250049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.250056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241259211409736:2048] [node 1] ICP22 created new session: [2:7587241259211409838:2048]
2025-12-24T02:11:05.250061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241259211409838:2048] [node 1] ICS09 handshake done sender: [2:7587241259211409835:2118] self: [2:7587241259211409836:0] peer: [1:7587241255569646942:0] socket: 396 qp: -1
2025-12-24T02:11:05.250067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241259211409838:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.250075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.250340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257457858448:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.250390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241257055788386:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24862
2025-12-24T02:11:05.250433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241257457858440:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.250447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.250451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257457858440:2095] [node 5] ICP22 created new session: [4:7587241257457858477:2095]
2025-12-24T02:11:05.250456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257457858477:2095] [node 5] ICS09 handshake done sender: [4:7587241257457858448:2115] self: [4:7587241257457858447:0] peer: [5:7587241257055788385:0] socket: 356 qp: -1
2025-12-24T02:11:05.250459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257457858477:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:05.250464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.250560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241256037138035:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2855) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241256037138060:2121] held: no
2025-12-24T02:11:05.250634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241256037138061:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.250672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138063:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.250876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241256037138030:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.250888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241256037138060:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.256315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241257139389536:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29388) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241257139389624:2118] held: no
2025-12-24T02:11:05.256375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241257139389624:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.256477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241257139389536:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.256493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.256502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241257139389536:2048] [node 1] ICP22 created new session: [3:7587241257139389639:2048]
2025-12-24T02:11:05.256509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241257139389639:2048] [node 1] ICS09 handshake done sender: [3:7587241257139389624:2118] self: [3:7587241257139389635:0] peer: [1:7587241255569646939:0] socket: 390 qp: -1
2025-12-24T02:11:05.256518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241257139389639:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.256528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.254913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241257055788382:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.256972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241257055788351:2090] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.256988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.256996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241257055788351:2090] [node 4] ICP22 created new session: [5:7587241257055788387:2090]
2025-12-24T02:11:05.257002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241257055788387:2090] [node 4] ICS09 handshake done sender: [5:7587241257055788382:2122] self: [5:7587241257055788385:0] peer: [4:7587241257457858447:0] socket: 373 qp: -1
2025-12-24T02:11:05.257010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241257055788387:2090] [node 4] ICS10 traffic start
2025-12-24T02:11:05.257018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.257087Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241257055788351:2090] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:05.258989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241256037138035:2106] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.259007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.259014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241256037138035:2106] [node 4] ICP22 created new session: [6:7587241256037138065:2106]
2025-12-24T02:11:05.259020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241256037138065:2106] [node 4] ICS09 handshake done sender: [6:7587241256037138060:2121] self: [6:7587241256037138062:0] peer: [4:7587241257457858449:0] socket: 400 qp: -1
2025-12-24T02:11:05.259026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241256037138065:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:05.259035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.266818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241257055788362:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.266839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241257055788362:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.267926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241256037138067:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12142
2025-12-24T02:11:05.268067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241257055788362:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.268274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241256037138061:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.268350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241256037138030:2092] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.268366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.268373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241256037138030:2092] [node 5] ICP22 created new session: [6:7587241256037138068:2092]
2025-12-24T02:11:05.268379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241256037138068:2092] [node 5] ICS09 handshake done sender: [6:7587241256037138061:2122] self: [6:7587241256037138064:0] peer: [5:7587241257055788361:0] socket: 401 qp: -1
2025-12-24T02:11:05.268386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241256037138068:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:05.268395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.269300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241257055788352:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.269321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.269328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241257055788352:2092] [node 6] ICP22 created new session: [5:7587241257055788390:2092]
2025-12-24T02:11:05.269335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241257055788390:2092] [node 6] ICS09 handshake done sender: [5:7587241257055788362:2115] self: [5:7587241257055788361:0] peer: [6:7587241256037138064:0] socket: 372 qp: -1
2025-12-24T02:11:05.269341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241257055788390:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:05.269350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.278372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:05.278452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241257139389639:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-12-24T02:11:05.278459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241257139389536:2048] [node 1] ICP30 unregister session Session# [3:7587241257139389639:2048] VirtualId# [3:7587241257139389635:0]
2025-12-24T02:11:05.278465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241257139389639:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.278503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.278523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.277746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:05.277860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241259211409838:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-12-24T02:11:05.277871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241259211409736:2048] [node 1] ICP30 unregister session Session# [2:7587241259211409838:2048] VirtualId# [2:7587241259211409836:0]
2025-12-24T02:11:05.277880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241259211409838:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.277950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.277977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.292039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:05.292133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241256037138054:2048] [node 1] ICS01 socket: 341 reason# EndOfStream
2025-12-24T02:11:05.292143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241256037137966:2048] [node 1] ICP30 unregister session Session# [6:7587241256037138054:2048] VirtualId# [6:7587241256037138051:0]
2025-12-24T02:11:05.292151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241256037138054:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.292212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.292238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.292325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:05.292388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241257055788374:2048] [node 1] ICS01 socket: 339 reason# EndOfStream
2025-12-24T02:11:05.292393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241257055788287:2048] [node 1] ICP30 unregister session Session# [5:7587241257055788374:2048] VirtualId# [5:7587241257055788372:0]
2025-12-24T02:11:05.292398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241257055788374:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.292426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.292447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.294914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:05.294996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241257457858472:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-12-24T02:11:05.295004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241257457858374:2048] [node 1] ICP30 unregister session Session# [4:7587241257457858472:2048] VirtualId# [4:7587241257457858465:0]
2025-12-24T02:11:05.295010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241257457858472:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.295072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.295098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.298515Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:05.298593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241257139389636:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:05.298601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241257139389537:2049] [node 2] ICP30 unregister session Session# [3:7587241257139389636:2049] VirtualId# [3:7587241257139389633:0]
2025-12-24T02:11:05.298607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241257139389636:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.298646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.298663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.305713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:05.305818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241257457858466:2049] [node 2] ICS01 socket: 351 reason# EndOfStream
2025-12-24T02:11:05.305828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241257457858375:2049] [node 2] ICP30 unregister session Session# [4:7587241257457858466:2049] VirtualId# [4:7587241257457858457:0]
2025-12-24T02:11:05.305834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241257457858466:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.305896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.305921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.309275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:05.309363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241257055788383:2049] [node 2] ICS01 socket: 368 reason# EndOfStream
2025-12-24T02:11:05.309372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241257055788288:2049] [node 2] ICP30 unregister session Session# [5:7587241257055788383:2049] VirtualId# [5:7587241257055788371:0]
2025-12-24T02:11:05.309379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241257055788383:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.309437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.309462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.309501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:05.309566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241256037138048:2049] [node 2] ICS01 socket: 347 reason# EndOfStream
2025-12-24T02:11:05.309571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241256037137967:2049] [node 2] ICP30 unregister session Session# [6:7587241256037138048:2049] VirtualId# [6:7587241256037138046:0]
2025-12-24T02:11:05.309575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241256037138048:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.309602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.309623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.330797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:05.330918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241257055788379:2050] [node 3] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:05.330926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241257055788289:2050] [node 3] ICP30 unregister session Session# [5:7587241257055788379:2050] VirtualId# [5:7587241257055788370:0]
2025-12-24T02:11:05.330933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241257055788379:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.330991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.331017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.331051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:05.331126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241256037138058:2050] [node 3] ICS01 socket: 388 reason# EndOfStream
2025-12-24T02:11:05.331131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241256037137968:2050] [node 3] ICP30 unregister session Session# [6:7587241256037138058:2050] VirtualId# [6:7587241256037138056:0]
2025-12-24T02:11:05.331135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241256037138058:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.331162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.331182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.337373Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:05.337477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241256037138065:2106] [node 4] ICS01 socket: 400 reason# EndOfStream
2025-12-24T02:11:05.337486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241256037138035:2106] [node 4] ICP30 unregister session Session# [6:7587241256037138065:2106] VirtualId# [6:7587241256037138062:0]
2025-12-24T02:11:05.337493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241256037138065:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.337552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:05.337590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:05.366906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241258990742648:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:05.366915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241258990742649:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:05.366794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241257374637822:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.366810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241257374637823:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:05.367139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241259209781584:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.367148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241259209781585:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.368669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257783149057:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.368690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257783149058:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.368694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257783149059:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.377310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241257069323243:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.377318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241257069323244:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.377322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241257069323245:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.370272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241258036107817:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.370279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241258036107818:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.370283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241258036107819:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.407730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241257069323294:2094] [node 5] ICP01 ready to work
2025-12-24T02:11:05.407830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241257069323295:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:05.409398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241258990742714:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.408424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257783149115:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:05.409124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241257783149121:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:05.410231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149125:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.411791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241258990742718:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.413533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241258036107865:2089] [node 6] ICP01 ready to work
2025-12-24T02:11:05.415047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107863:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.415062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107864:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.415174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241258036107874:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:05.416035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241258990742722:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:05.418949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241257374637885:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:05.419049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241257374637886:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:05.419126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241257374637887:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.422580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323314:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.422607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323315:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.422619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323316:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.423992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637899:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.424028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637900:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.427424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241257069323316:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.427455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241257069323244:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.424694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241259209781649:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:05.424782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241259209781650:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:05.424857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241259209781651:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.435371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781663:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.435471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781664:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
2025-12-24T02:11:05.435492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781665:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
2025-12-24T02:11:05.439710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781663:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.439095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241258036107818:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21913149ns) {Configure:104(21912430ns) {ProcessConfigured:135(21648942ns) {StartInitialHandshake:157(21648489ns) {DropHandshakes:470(375ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(1459ns) {SwitchToState:179(793ns)}}}}}
2025-12-24T02:11:05.442816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241257069323315:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.442863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241257069323294:2094] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.446977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107863:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.447044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107897:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.453115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241257374637822:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29043283ns) {Configure:104(29042513ns) {ProcessConfigured:135(28754374ns) {StartInitialHandshake:157(28754038ns) {DropHandshakes:470(451ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:477(933ns) {SwitchToState:179(279ns)}}}}}
2025-12-24T02:11:05.453156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637893:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.453162Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241257374637893:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.453532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637900:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.453876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637905:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.453887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637906:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.454407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107897:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.454441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241258036107818:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.454682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241257374637893:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.455067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781665:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.454781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107873:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.454794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241258036107873:2099] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.463025Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241258990742718:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51145778ns) {Configure:104(51145143ns) {ProcessConfigured:135(50911549ns) {StartInitialHandshake:157(50911079ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(165ns) PrepareNewSessionHandshake:477(1688ns) {SwitchToState:179(803ns)}}}}}
2025-12-24T02:11:05.465211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637891:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.465236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241257374637891:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.471075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149125:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.471140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149136:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.471183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149137:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.474801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107864:2092] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.474943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241258036107874:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.471292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241257374637886:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.471317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.471327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241257374637886:2108] [node 6] ICP22 created new session: [2:7587241257374637907:2108]
2025-12-24T02:11:05.471333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241257374637907:2108] [node 6] ICS09 handshake done sender: [2:7587241257374637893:2112] self: [2:7587241257374637892:0] peer: [6:7587241257069323322:0] socket: 365 qp: -1
2025-12-24T02:11:05.471341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241257374637907:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:05.471351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.477814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241257069323244:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241257069323316:2113] held: no
2025-12-24T02:11:05.478380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323325:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.475502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149136:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.475542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149123:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.475545Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241257783149123:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.476258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149139:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.477255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241258036107873:2099] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.477383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241258036107865:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.477400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.477409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241258036107865:2089] [node 6] ICP22 created new session: [5:7587241258036107903:2089]
2025-12-24T02:11:05.477416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241258036107903:2089] [node 6] ICS09 handshake done sender: [5:7587241258036107873:2099] self: [5:7587241258036107872:0] peer: [6:7587241257069323323:0] socket: 357 qp: -1
2025-12-24T02:11:05.477423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241258036107903:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:05.477433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.478641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107902:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.480397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241259209781585:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.481995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241258990742724:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.480011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257783149123:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.480132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149139:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.482210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257783149058:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.482313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149142:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.483424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241257069323243:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44146751ns) {Configure:104(44145708ns) {ProcessConfigured:135(43885886ns) {StartInitialHandshake:157(43885158ns) {DropHandshakes:470(442ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:477(762ns) {SwitchToState:179(326ns)}}}}}
2025-12-24T02:11:05.485407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.486847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241258036107864:2092] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.491142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742724:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.491625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637895:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.491637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241257374637895:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.492185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637910:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.492610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241257374637891:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.492837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241257374637895:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.492934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637899:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.492947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241257374637885:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.492960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.492968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241257374637885:2107] [node 5] ICP22 created new session: [2:7587241257374637914:2107]
2025-12-24T02:11:05.492974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241257374637914:2107] [node 5] ICS09 handshake done sender: [2:7587241257374637891:2111] self: [2:7587241257374637890:0] peer: [5:7587241258036107899:0] socket: 364 qp: -1
2025-12-24T02:11:05.492981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241257374637914:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:05.492990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.493156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241257374637887:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.493162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.493166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241257374637887:2109] [node 4] ICP22 created new session: [2:7587241257374637916:2109]
2025-12-24T02:11:05.493171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241257374637916:2109] [node 4] ICS09 handshake done sender: [2:7587241257374637895:2113] self: [2:7587241257374637894:0] peer: [4:7587241257783149144:0] socket: 369 qp: -1
2025-12-24T02:11:05.493176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241257374637916:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:05.493181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.493604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241257374637822:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.493894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742709:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.493903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241258990742709:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.494552Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241258990742729:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-12-24T02:11:05.494764Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241258990742709:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.494874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241257783149121:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.494900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.494908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257783149121:2106] [node 5] ICP22 created new session: [4:7587241257783149145:2106]
2025-12-24T02:11:05.494913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257783149145:2106] [node 5] ICS09 handshake done sender: [4:7587241257783149123:2109] self: [4:7587241257783149122:0] peer: [5:7587241258036107900:0] socket: 351 qp: -1
2025-12-24T02:11:05.494919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257783149145:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:05.494925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.494979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323326:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.498852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637889:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.498875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241257374637889:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.501713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241257069323314:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.501751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241257069323294:2094] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7353) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241257069323315:2112] held: no
2025-12-24T02:11:05.501851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241257069323316:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.501917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241257069323315:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.501989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241257069323244:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.502010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.502017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241257069323244:2049] [node 2] ICP22 created new session: [6:7587241257069323330:2049]
2025-12-24T02:11:05.502023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241257069323330:2049] [node 2] ICS09 handshake done sender: [6:7587241257069323316:2113] self: [6:7587241257069323322:0] peer: [2:7587241257374637892:0] socket: 367 qp: -1
2025-12-24T02:11:05.502032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241257069323330:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.502051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.502197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241257069323294:2094] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.502212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.502216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241257069323294:2094] [node 5] ICP22 created new session: [6:7587241257069323332:2094]
2025-12-24T02:11:05.502222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241257069323332:2094] [node 5] ICS09 handshake done sender: [6:7587241257069323315:2112] self: [6:7587241257069323323:0] peer: [5:7587241258036107872:0] socket: 362 qp: -1
2025-12-24T02:11:05.502227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241257069323332:2094] [node 5] ICS10 traffic start
2025-12-24T02:11:05.502232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.502442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241257069323295:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.502620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241257374637889:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.502866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241257374637823:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.502878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.502886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241257374637823:2049] [node 3] ICP22 created new session: [2:7587241257374637921:2049]
2025-12-24T02:11:05.502892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241257374637921:2049] [node 3] ICS09 handshake done sender: [2:7587241257374637889:2110] self: [2:7587241257374637888:0] peer: [3:7587241259209781667:0] socket: 363 qp: -1
2025-12-24T02:11:05.502897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241257374637921:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.502904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.506971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781664:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.507042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241259209781584:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.507851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781669:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
2025-12-24T02:11:05.507919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781670:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.509219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781671:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
2025-12-24T02:11:05.507390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149137:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.508182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241258990742648:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.508205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.508213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241258990742648:2048] [node 2] ICP22 created new session: [1:7587241258990742731:2048]
2025-12-24T02:11:05.508221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241258990742731:2048] [node 2] ICS09 handshake done sender: [1:7587241258990742709:2107] self: [1:7587241258990742708:0] peer: [2:7587241257374637919:0] socket: 343 qp: -1
2025-12-24T02:11:05.508229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241258990742731:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:05.508248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.508493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742713:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.508500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241258990742713:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.508925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241258990742733:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-12-24T02:11:05.509322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241258990742713:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.509417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241258990742649:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.509431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.509439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241258990742649:2049] [node 3] ICP22 created new session: [1:7587241258990742736:2049]
2025-12-24T02:11:05.509445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241258990742736:2049] [node 3] ICS09 handshake done sender: [1:7587241258990742713:2109] self: [1:7587241258990742712:0] peer: [3:7587241259209781668:0] socket: 345 qp: -1
2025-12-24T02:11:05.509450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241258990742736:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.509458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.511021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149120:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.511031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241257783149120:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.511737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257783149059:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.511820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149147:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.512778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241259209781584:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241259209781664:2116] held: no
2025-12-24T02:11:05.512836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781669:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.512860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781655:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.512868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241259209781655:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.513232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241259209781665:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.513322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241259209781585:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.513341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.513347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241259209781585:2049] [node 2] ICP22 created new session: [3:7587241259209781673:2049]
2025-12-24T02:11:05.513352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241259209781673:2049] [node 2] ICS09 handshake done sender: [3:7587241259209781665:2117] self: [3:7587241259209781667:0] peer: [2:7587241257374637888:0] socket: 377 qp: -1
2025-12-24T02:11:05.513359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241259209781673:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.513366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.514245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323336:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.514285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323337:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.514655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323338:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.514775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241259209781655:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.514880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241259209781649:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.514899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.514906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241259209781649:2107] [node 4] ICP22 created new session: [3:7587241259209781678:2107]
2025-12-24T02:11:05.514911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241259209781678:2107] [node 4] ICS09 handshake done sender: [3:7587241259209781655:2112] self: [3:7587241259209781654:0] peer: [4:7587241257783149149:0] socket: 373 qp: -1
2025-12-24T02:11:05.514920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241259209781678:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:05.514928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.523137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241258036107874:2097] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.523158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.523166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241258036107874:2097] [node 4] ICP22 created new session: [5:7587241258036107908:2097]
2025-12-24T02:11:05.523172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241258036107908:2097] [node 4] ICS09 handshake done sender: [5:7587241258036107864:2092] self: [5:7587241258036107900:0] peer: [4:7587241257783149122:0] socket: 356 qp: -1
2025-12-24T02:11:05.523180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241258036107908:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:05.523188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.525523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241258036107874:2097] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.526901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742733:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.526953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241258990742739:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-12-24T02:11:05.526814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241257374637823:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23905613ns) {HandleHandshakeStatus:329(23904992ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(23838145ns) {ForwardSessionEventToSession:612(23824493ns) ForwardSessionEventToSession:612(2307ns) ForwardSessionEventToSession:612(731ns) ForwardSessionEventToSession:612(1174ns) ForwardSessionEventToSession:612(716ns) ForwardSessionEventToSession:612(607ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(637ns) ForwardSessionEventToSession:612(1039ns)}}}
2025-12-24T02:11:05.527086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742729:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.527103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241258990742648:2048] [node 2] ICP18 handshake (actor [1:7587241258990742729:2116]) is too old
2025-12-24T02:11:05.527379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241257374637906:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.527425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241257374637823:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.534973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241257069323336:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.535013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241257069323243:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.534853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241257374637822:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241257374637899:2114] held: no
2025-12-24T02:11:05.534951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241257374637923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24094
2025-12-24T02:11:05.535148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742726:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.535160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241258990742726:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.535814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241258990742726:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.535082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241257783149121:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(54859564ns) {HandleHandshakeStatus:329(54859153ns) {DropIncomingHandshake:442(6096ns) DropOutgoingHandshake:457(99ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(16074726ns) {ForwardSessionEventToSession:612(16061175ns) ForwardSessionEventToSession:612(2103ns) ForwardSessionEventToSession:612(860ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(1450ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(1478ns)}}}
2025-12-24T02:11:05.535347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257783149120:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.538921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241257783149142:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.542791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241258990742722:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.542828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.542839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241258990742722:2111] [node 6] ICP22 created new session: [1:7587241258990742742:2111]
2025-12-24T02:11:05.542848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241258990742742:2111] [node 6] ICS09 handshake done sender: [1:7587241258990742726:2115] self: [1:7587241258990742725:0] peer: [6:7587241257069323339:0] socket: 360 qp: -1
2025-12-24T02:11:05.542857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241258990742742:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:05.542869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.543011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241257783149059:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5613) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241257783149137:2117] held: no
2025-12-24T02:11:05.543043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241257783149115:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.546730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241259209781680:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60693
2025-12-24T02:11:05.543058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.543065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257783149115:2104] [node 6] ICP22 created new session: [4:7587241257783149151:2104]
2025-12-24T02:11:05.543070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257783149151:2104] [node 6] ICS09 handshake done sender: [4:7587241257783149120:2108] self: [4:7587241257783149119:0] peer: [6:7587241257069323335:0] socket: 350 qp: -1
2025-12-24T02:11:05.543078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257783149151:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:05.543086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.545439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257783149139:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.545561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241257783149057:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.545573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257783149058:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.545583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.545588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257783149058:2049] [node 2] ICP22 created new session: [4:7587241257783149154:2049]
2025-12-24T02:11:05.545594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257783149154:2049] [node 2] ICS09 handshake done sender: [4:7587241257783149139:2118] self: [4:7587241257783149144:0] peer: [2:7587241257374637894:0] socket: 385 qp: -1
2025-12-24T02:11:05.545600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257783149154:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.545608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.546944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241259209781664:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.547029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241259209781584:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.547042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.547049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241259209781584:2048] [node 1] ICP22 created new session: [3:7587241259209781681:2048]
2025-12-24T02:11:05.547054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241259209781681:2048] [node 1] ICS09 handshake done sender: [3:7587241259209781664:2116] self: [3:7587241259209781668:0] peer: [1:7587241258990742712:0] socket: 376 qp: -1
2025-12-24T02:11:05.547060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241259209781681:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.547068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.546932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241257374637887:2109] [node 4] ICP18 handshake (actor [2:7587241257374637906:2118]) is too old
2025-12-24T02:11:05.548450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241257374637899:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.548668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241257374637822:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.548677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.548686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241257374637822:2048] [node 1] ICP22 created new session: [2:7587241257374637924:2048]
2025-12-24T02:11:05.548692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241257374637924:2048] [node 1] ICS09 handshake done sender: [2:7587241257374637899:2114] self: [2:7587241257374637919:0] peer: [1:7587241258990742708:0] socket: 354 qp: -1
2025-12-24T02:11:05.548699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241257374637924:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.548708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.545707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241258036107818:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241258036107897:2117] held: no
2025-12-24T02:11:05.550809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781671:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.551749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241257069323314:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.551321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149150:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.551672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107906:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.555276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241257069323243:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241257069323336:2121] held: no
2025-12-24T02:11:05.555368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241257069323338:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.555415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323340:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.556234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241257069323245:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.556250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241257069323336:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.556485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781657:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.556495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241259209781657:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.557932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241259209781657:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.557729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241257069323243:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.557746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.557753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241257069323243:2048] [node 1] ICP22 created new session: [6:7587241257069323342:2048]
2025-12-24T02:11:05.557758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241257069323342:2048] [node 1] ICS09 handshake done sender: [6:7587241257069323336:2121] self: [6:7587241257069323339:0] peer: [1:7587241258990742725:0] socket: 407 qp: -1
2025-12-24T02:11:05.557764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241257069323342:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.557775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.554947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107907:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.555038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107911:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.555211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241258036107897:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.555765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107911:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.555815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241258036107907:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.555822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241258036107818:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.555835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.555842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241258036107818:2049] [node 2] ICP22 created new session: [5:7587241258036107912:2049]
2025-12-24T02:11:05.555848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241258036107912:2049] [node 2] ICS09 handshake done sender: [5:7587241258036107897:2117] self: [5:7587241258036107899:0] peer: [2:7587241257374637890:0] socket: 370 qp: -1
2025-12-24T02:11:05.555855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241258036107912:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.555862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.556339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241258036107819:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.556351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241258036107817:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.558078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241259209781659:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.558084Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241259209781659:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.558560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241259209781650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.558573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.558577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241259209781650:2108] [node 5] ICP22 created new session: [3:7587241259209781685:2108]
2025-12-24T02:11:05.558582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241259209781685:2108] [node 5] ICS09 handshake done sender: [3:7587241259209781657:2113] self: [3:7587241259209781656:0] peer: [5:7587241258036107914:0] socket: 372 qp: -1
2025-12-24T02:11:05.558588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241259209781685:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:05.558595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.559173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241259209781659:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.559334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241259209781651:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.559349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.559354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241259209781651:2109] [node 6] ICP22 created new session: [3:7587241259209781688:2109]
2025-12-24T02:11:05.559358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241259209781688:2109] [node 6] ICS09 handshake done sender: [3:7587241259209781659:2114] self: [3:7587241259209781658:0] peer: [6:7587241257069323341:0] socket: 409 qp: -1
2025-12-24T02:11:05.559362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241259209781688:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:05.559367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.556046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742739:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.557105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742723:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.557109Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241258990742723:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.558809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241258990742723:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.558920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241257069323295:2097] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.558940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.558947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241257069323295:2097] [node 4] ICP22 created new session: [6:7587241257069323344:2097]
2025-12-24T02:11:05.558953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241257069323344:2097] [node 4] ICS09 handshake done sender: [6:7587241257069323314:2111] self: [6:7587241257069323335:0] peer: [4:7587241257783149119:0] socket: 361 qp: -1
2025-12-24T02:11:05.558959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241257069323344:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:05.558967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.560027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241258990742718:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.560044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.560050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241258990742718:2110] [node 5] ICP22 created new session: [1:7587241258990742746:2110]
2025-12-24T02:11:05.560056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241258990742746:2110] [node 5] ICS09 handshake done sender: [1:7587241258990742723:2113] self: [1:7587241258990742719:0] peer: [5:7587241258036107915:0] socket: 405 qp: -1
2025-12-24T02:11:05.560062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241258990742746:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:05.560071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.564262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241258036107819:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5613) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241258036107911:2121] held: no
2025-12-24T02:11:05.565633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241258036107817:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10482) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241258036107907:2120] held: no
2025-12-24T02:11:05.565696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107916:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.567164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241257069323245:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5613) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241257069323338:2123] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.567313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241258036107917:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47388
2025-12-24T02:11:05.567458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241258036107911:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.567515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241258036107907:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.567890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241257069323346:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42053
2025-12-24T02:11:05.567567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241258036107819:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.567583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.567590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241258036107819:2050] [node 3] ICP22 created new session: [5:7587241258036107918:2050]
2025-12-24T02:11:05.567596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241258036107918:2050] [node 3] ICS09 handshake done sender: [5:7587241258036107911:2121] self: [5:7587241258036107914:0] peer: [3:7587241259209781656:0] socket: 413 qp: -1
2025-12-24T02:11:05.567603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241258036107918:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.567610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.567734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241258036107817:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.567739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.567743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241258036107817:2048] [node 1] ICP22 created new session: [5:7587241258036107920:2048]
2025-12-24T02:11:05.567747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241258036107920:2048] [node 1] ICS09 handshake done sender: [5:7587241258036107907:2120] self: [5:7587241258036107915:0] peer: [1:7587241258990742719:0] socket: 412 qp: -1
2025-12-24T02:11:05.567750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241258036107920:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.567755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.568094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241257069323338:2123] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.570759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742721:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.570781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241258990742721:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.572088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241258990742721:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.572243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241258990742714:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.572259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.572270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241258990742714:2108] [node 4] ICP22 created new session: [1:7587241258990742750:2108]
2025-12-24T02:11:05.572278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241258990742750:2108] [node 4] ICS09 handshake done sender: [1:7587241258990742721:2112] self: [1:7587241258990742716:0] peer: [4:7587241257783149153:0] socket: 389 qp: -1
2025-12-24T02:11:05.572284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241258990742750:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:05.572291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.572466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241258990742749:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61992
2025-12-24T02:11:05.571930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257783149137:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.582914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241257069323245:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.582941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.582952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241257069323245:2050] [node 3] ICP22 created new session: [6:7587241257069323347:2050]
2025-12-24T02:11:05.582960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241257069323347:2050] [node 3] ICS09 handshake done sender: [6:7587241257069323338:2123] self: [6:7587241257069323341:0] peer: [3:7587241259209781658:0] socket: 410 qp: -1
2025-12-24T02:11:05.582968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241257069323347:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.582974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.583264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257783149059:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.583281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.583286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257783149059:2050] [node 3] ICP22 created new session: [4:7587241257783149157:2050]
2025-12-24T02:11:05.583291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257783149157:2050] [node 3] ICS09 handshake done sender: [4:7587241257783149137:2117] self: [4:7587241257783149149:0] peer: [3:7587241259209781654:0] socket: 383 qp: -1
2025-12-24T02:11:05.583297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257783149157:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.583302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.584706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241257783149159:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22795
2025-12-24T02:11:05.587250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241258990742749:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.587289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241258990742714:2108] [node 4] ICP18 handshake (actor [1:7587241258990742749:2119]) is too old
2025-12-24T02:11:05.586985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241257783149142:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.587535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241257783149057:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.587551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.587559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241257783149057:2048] [node 1] ICP22 created new session: [4:7587241257783149160:2048]
2025-12-24T02:11:05.587567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241257783149160:2048] [node 1] ICS09 handshake done sender: [4:7587241257783149142:2119] self: [4:7587241257783149153:0] peer: [1:7587241258990742716:0] socket: 390 qp: -1
2025-12-24T02:11:05.587575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241257783149160:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.587583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.602820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:05.602910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241259209781681:2048] [node 1] ICS01 socket: 376 reason# EndOfStream
2025-12-24T02:11:05.602918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241259209781584:2048] [node 1] ICP30 unregister session Session# [3:7587241259209781681:2048] VirtualId# [3:7587241259209781668:0]
2025-12-24T02:11:05.602924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241259209781681:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.602963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.602986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.603662Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:05.603741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241257374637924:2048] [node 1] ICS01 socket: 354 reason# EndOfStream
2025-12-24T02:11:05.603748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241257374637822:2048] [node 1] ICP30 unregister session Session# [2:7587241257374637924:2048] VirtualId# [2:7587241257374637919:0]
2025-12-24T02:11:05.603753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241257374637924:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.603813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.603839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.609400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:05.609494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241257783149160:2048] [node 1] ICS01 socket: 390 reason# EndOfStream
2025-12-24T02:11:05.609504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241257783149057:2048] [node 1] ICP30 unregister session Session# [4:7587241257783149160:2048] VirtualId# [4:7587241257783149153:0]
2025-12-24T02:11:05.609511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241257783149160:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.609571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.609597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.610797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:05.610871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241258036107920:2048] [node 1] ICS01 socket: 412 reason# EndOfStream
2025-12-24T02:11:05.610877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241258036107817:2048] [node 1] ICP30 unregister session Session# [5:7587241258036107920:2048] VirtualId# [5:7587241258036107915:0]
2025-12-24T02:11:05.610881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241258036107920:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.610920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.610947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.610975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:05.611033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241257069323342:2048] [node 1] ICS01 socket: 407 reason# EndOfStream
2025-12-24T02:11:05.611037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241257069323243:2048] [node 1] ICP30 unregister session Session# [6:7587241257069323342:2048] VirtualId# [6:7587241257069323339:0]
2025-12-24T02:11:05.611042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241257069323342:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.611067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.611086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.619300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:05.619393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241259209781673:2049] [node 2] ICS01 socket: 377 reason# EndOfStream
2025-12-24T02:11:05.619403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241259209781585:2049] [node 2] ICP30 unregister session Session# [3:7587241259209781673:2049] VirtualId# [3:7587241259209781667:0]
2025-12-24T02:11:05.619411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241259209781673:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.619463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.619487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.630781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:05.630864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241258036107912:2049] [node 2] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:05.630872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241258036107818:2049] [node 2] ICP30 unregister session Session# [5:7587241258036107912:2049] VirtualId# [5:7587241258036107899:0]
2025-12-24T02:11:05.630878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241258036107912:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.630931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.630951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.630985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:05.631043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241257069323330:2049] [node 2] ICS01 socket: 367 reason# EndOfStream
2025-12-24T02:11:05.631047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241257069323244:2049] [node 2] ICP30 unregister session Session# [6:7587241257069323330:2049] VirtualId# [6:7587241257069323322:0]
2025-12-24T02:11:05.631051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241257069323330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.631074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.631093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.630780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:05.630885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241257783149154:2049] [node 2] ICS07 socket disconnect 385 reason# EndOfStream
2025-12-24T02:11:05.630891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241257783149154:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.630941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.630961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241257783149154:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:05.631340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241257783149058:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7774) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.631351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241257783149154:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.631356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241257783149058:2049] [node 2] ICP30 unregister session Session# [4:7587241257783149154:2049] VirtualId# [4:7587241257783149144:0]
2025-12-24T02:11:05.631373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.631379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241257783149058:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7774) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.631381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7774) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.642783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:05.642873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241258036107918:2050] [node 3] ICS01 socket: 413 reason# EndOfStream
2025-12-24T02:11:05.642880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241258036107819:2050] [node 3] ICP30 unregister session Session# [5:7587241258036107918:2050] VirtualId# [5:7587241258036107914:0]
2025-12-24T02:11:05.642887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241258036107918:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.642948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.642970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.643001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:05.643059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241257069323347:2050] [node 3] ICS01 socket: 410 reason# EndOfStream
2025-12-24T02:11:05.643063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241257069323245:2050] [node 3] ICP30 unregister session Session# [6:7587241257069323347:2050] VirtualId# [6:7587241257069323341:0]
2025-12-24T02:11:05.643068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241257069323347:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.643090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.643107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.665337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:05.665437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241257069323344:2097] [node 4] ICS01 socket: 361 reason# EndOfStream
2025-12-24T02:11:05.665445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241257069323295:2097] [node 4] ICP30 unregister session Session# [6:7587241257069323344:2097] VirtualId# [6:7587241257069323335:0]
2025-12-24T02:11:05.665453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241257069323344:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.665521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:05.665548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:05.693938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241257269137281:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:05.693945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241257269137282:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:05.709145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241259145975993:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.704877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241259129038493:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.704885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241259129038494:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.709155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241259145975994:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.704889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241259129038495:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.709161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241259145975995:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.706415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241258805982109:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.706422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241258805982110:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.706426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241258805982111:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:05.698827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255424588659:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.698835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255424588660:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:05.705657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241258360115026:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:05.705669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241258360115027:2049] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.761113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241258360115086:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61007
2025-12-24T02:11:05.760024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241258805982158:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:05.762077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241258360115088:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.762956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038554:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.762980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038555:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.764080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241258360115093:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:05.771521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241258805982173:2100] [node 4] ICP01 ready to work
2025-12-24T02:11:05.766961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588719:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
2025-12-24T02:11:05.767799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255424588723:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:05.767877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255424588724:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:05.767941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241255424588725:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.774720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241258360115102:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:05.776206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241258360115101:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61007
2025-12-24T02:11:05.775202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241257269137350:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:05.778923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241257269137345:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.787424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982180:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
2025-12-24T02:11:05.788788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241259145975993:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19248922ns) {Configure:104(19248438ns) {ProcessConfigured:135(18955476ns) {StartInitialHandshake:157(18955253ns) {DropHandshakes:470(545ns) {DropIncomingHandshake:442(103ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:477(1108ns) {SwitchToState:179(61ns)}}}}}
2025-12-24T02:11:05.793193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241257269137348:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:05.792498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241258805982173:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20902056ns) {Configure:104(20901203ns) {ProcessConfigured:135(20562140ns) {StartInitialHandshake:157(20561629ns) {DropHandshakes:470(838ns) {DropIncomingHandshake:442(138ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(274ns) PrepareNewSessionHandshake:477(1673ns) {SwitchToState:179(497ns)}}}}}
2025-12-24T02:11:05.793947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241258805982180:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.795282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241258805982110:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.795454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241259129038555:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.795499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241259129038554:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.795693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241259129038557:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:05.796038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115095:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.796050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241258360115095:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.797048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241258360115095:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.797792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241259129038494:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.798023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115090:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.798036Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241258360115090:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.794878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241255424588659:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27562220ns) {Configure:104(27561307ns) {ProcessConfigured:135(19791933ns) {StartInitialHandshake:157(19790890ns) {DropHandshakes:470(1275ns) {DropIncomingHandshake:442(312ns) DropOutgoingHandshake:457(268ns)} GenerateSessionVirtualId:988(332ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(674ns)}}}}}
2025-12-24T02:11:05.795838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241255424588660:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28425498ns) {Configure:104(28424759ns) {ProcessConfigured:135(20648000ns) {StartInitialHandshake:157(20647501ns) {DropHandshakes:470(246ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(318ns) {SwitchToState:179(112ns)}}}}}
2025-12-24T02:11:05.795937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241255424588723:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27961575ns) {Configure:104(27960803ns) {ProcessConfigured:135(20585658ns) {StartInitialHandshake:157(20584966ns) {DropHandshakes:470(615ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(484ns) {SwitchToState:179(176ns)}}}}}
2025-12-24T02:11:05.795775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976060:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.800424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241258360115102:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25584306ns) {Configure:104(25582425ns) {ProcessConfigured:135(23868301ns) {StartInitialHandshake:157(23867807ns) {DropHandshakes:470(744ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(591ns) {SwitchToState:179(310ns)}}}}}
2025-12-24T02:11:05.800440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241258360115093:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.800449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.800455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241258360115093:2110] [node 6] ICP22 created new session: [2:7587241258360115108:2110]
2025-12-24T02:11:05.799474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241257269137347:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9044
2025-12-24T02:11:05.800461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241258360115108:2110] [node 6] ICS09 handshake done sender: [2:7587241258360115095:2111] self: [2:7587241258360115094:0] peer: [6:7587241258805982184:0] socket: 355 qp: -1
2025-12-24T02:11:05.800469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241258360115108:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:05.800181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241257269137354:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9044
2025-12-24T02:11:05.800489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.800202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241257269137353:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9044
2025-12-24T02:11:05.800647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241257269137350:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21617393ns) {Configure:104(21616237ns) {ProcessConfigured:135(21238256ns) {StartInitialHandshake:157(21237915ns) {DropHandshakes:470(727ns) {DropIncomingHandshake:442(109ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:477(1197ns) {SwitchToState:179(795ns)}}}}}
2025-12-24T02:11:05.801014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241258805982160:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.801030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241258805982160:2096] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.801151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038558:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.798954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241259145976060:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.802678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588719:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.808632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137354:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.812505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241258360115090:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.812698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241258360115088:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.812713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.812723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241258360115088:2108] [node 5] ICP22 created new session: [2:7587241258360115112:2108]
2025-12-24T02:11:05.812730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241258360115112:2108] [node 5] ICS09 handshake done sender: [2:7587241258360115090:2109] self: [2:7587241258360115089:0] peer: [5:7587241259129038559:0] socket: 351 qp: -1
2025-12-24T02:11:05.812737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241258360115112:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:05.812748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.814937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982185:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
2025-12-24T02:11:05.815000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
2025-12-24T02:11:05.815010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982187:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
2025-12-24T02:11:05.815470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241259129038557:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.815536Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241259129038557:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19660791ns) {Configure:104(19660126ns) {ProcessConfigured:135(17402047ns) {IncomingHandshake:261(17390783ns) {DropIncomingHandshake:442(48ns) PrepareNewSessionHandshake:477(242ns) {SwitchToState:179(82ns)} GenerateSessionVirtualId:988(553ns) IssueIncomingHandshakeReply:193(17385351ns) {SwitchToState:179(898ns)}}}}}
2025-12-24T02:11:05.815901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241259145976062:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:05.816397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241259145976062:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.818876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241258805982185:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.818896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137347:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.819102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241259129038558:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.819501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241259129038495:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.822624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115086:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.822676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241258360115026:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.820229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588740:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
2025-12-24T02:11:05.820250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588741:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
2025-12-24T02:11:05.819569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241258805982180:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.826978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241259129038563:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.828224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241259145976066:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:05.828304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976065:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.827056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241258805982181:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.827083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241258805982181:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.828918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115105:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.828930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241258360115105:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.829620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241258360115105:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.829777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241258360115027:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.829796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.829804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241258360115027:2049] [node 3] ICP22 created new session: [2:7587241258360115117:2049]
2025-12-24T02:11:05.829810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241258360115117:2049] [node 3] ICS09 handshake done sender: [2:7587241258360115105:2115] self: [2:7587241258360115104:0] peer: [3:7587241255424588743:0] socket: 380 qp: -1
2025-12-24T02:11:05.829817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241258360115117:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.829826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.827163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588731:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.827179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255424588731:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.827826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588741:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.827857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241255424588660:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.829278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255424588731:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.830785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255424588724:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.830801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.830810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255424588724:2109] [node 5] ICP22 created new session: [3:7587241255424588744:2109]
2025-12-24T02:11:05.830817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255424588744:2109] [node 5] ICS09 handshake done sender: [3:7587241255424588731:2112] self: [3:7587241255424588730:0] peer: [5:7587241259129038561:0] socket: 370 qp: -1
2025-12-24T02:11:05.830824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255424588744:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:05.830832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.831468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241258805982110:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.831486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.831493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241258805982110:2049] [node 2] ICP22 created new session: [6:7587241258805982189:2049]
2025-12-24T02:11:05.831500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241258805982189:2049] [node 2] ICS09 handshake done sender: [6:7587241258805982180:2111] self: [6:7587241258805982184:0] peer: [2:7587241258360115094:0] socket: 362 qp: -1
2025-12-24T02:11:05.831506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241258805982189:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.831538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.833154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241259145976065:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.833293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241259145975994:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.833448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115101:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.834614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137342:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.834635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241257269137342:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.844115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241258805982111:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.844402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137353:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.845882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588740:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.845926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241255424588659:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.844221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241258805982187:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.844385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241258805982160:2096] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.845134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241258805982158:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.845144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.845151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241258805982158:2095] [node 5] ICP22 created new session: [6:7587241258805982194:2095]
2025-12-24T02:11:05.845158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241258805982194:2095] [node 5] ICS09 handshake done sender: [6:7587241258805982160:2096] self: [6:7587241258805982159:0] peer: [5:7587241259129038560:0] socket: 348 qp: -1
2025-12-24T02:11:05.845167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241258805982194:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:05.845178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.845366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241258805982109:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.846320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115107:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.846338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241258360115107:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.846591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241258805982181:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.851644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038567:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.851782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241258360115120:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61007
2025-12-24T02:11:05.851917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588748:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.855406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038568:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.855495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038569:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.855704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241259129038558:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.856134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241259129038554:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.856219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241259129038495:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.856235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.856242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241259129038495:2050] [node 3] ICP22 created new session: [5:7587241259129038570:2050]
2025-12-24T02:11:05.856248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241259129038570:2050] [node 3] ICS09 handshake done sender: [5:7587241259129038558:2111] self: [5:7587241259129038561:0] peer: [3:7587241255424588730:0] socket: 374 qp: -1
2025-12-24T02:11:05.856256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241259129038570:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.856273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.856440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241259129038557:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.856446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.856450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241259129038557:2110] [node 6] ICP22 created new session: [5:7587241259129038572:2110]
2025-12-24T02:11:05.856454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241259129038572:2110] [node 6] ICS09 handshake done sender: [5:7587241259129038554:2108] self: [5:7587241259129038560:0] peer: [6:7587241258805982159:0] socket: 352 qp: -1
2025-12-24T02:11:05.856458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241259129038572:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:05.856462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.856594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038566:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.856793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241259129038555:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.856842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241259129038494:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.856851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.856855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241259129038494:2049] [node 2] ICP22 created new session: [5:7587241259129038574:2049]
2025-12-24T02:11:05.856859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241259129038574:2049] [node 2] ICS09 handshake done sender: [5:7587241259129038555:2109] self: [5:7587241259129038559:0] peer: [2:7587241258360115089:0] socket: 353 qp: -1
2025-12-24T02:11:05.856863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241259129038574:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.856868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.857020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241259129038568:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.858005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241259129038493:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35184556ns) {Configure:104(35183349ns) {ProcessConfigured:135(23388469ns) {StartInitialHandshake:157(23387021ns) {DropHandshakes:470(1299ns) {DropIncomingHandshake:442(158ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(789ns) PrepareNewSessionHandshake:477(965ns) {SwitchToState:179(547ns)}}}}}
2025-12-24T02:11:05.858681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137358:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.858715Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241257269137358:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.859641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241257269137342:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.859968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241257269137358:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.860062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241257269137281:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.860085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.860093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241257269137281:2048] [node 2] ICP22 created new session: [1:7587241257269137368:2048]
2025-12-24T02:11:05.860100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241257269137368:2048] [node 2] ICS09 handshake done sender: [1:7587241257269137342:2107] self: [1:7587241257269137341:0] peer: [2:7587241258360115115:0] socket: 349 qp: -1
2025-12-24T02:11:05.860107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241257269137368:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:05.860116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.860350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241257269137282:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.860362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.860367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241257269137282:2049] [node 3] ICP22 created new session: [1:7587241257269137371:2049]
2025-12-24T02:11:05.860372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241257269137371:2049] [node 3] ICS09 handshake done sender: [1:7587241257269137358:2115] self: [1:7587241257269137357:0] peer: [3:7587241255424588747:0] socket: 379 qp: -1
2025-12-24T02:11:05.860377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241257269137371:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:05.860382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.860967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241259145976068:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.860985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241259145976068:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.859722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241259129038563:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32672496ns) {Configure:104(32671162ns) {ProcessConfigured:135(20677905ns) {StartInitialHandshake:157(20676833ns) {DropHandshakes:470(1123ns) {DropIncomingHandshake:442(155ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(627ns) PrepareNewSessionHandshake:477(1318ns) {SwitchToState:179(671ns)}}}}}
2025-12-24T02:11:05.859759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241259129038563:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.862819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241258360115120:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.862857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241258360115027:2049] [node 3] ICP18 handshake (actor [2:7587241258360115120:2117]) is too old
2025-12-24T02:11:05.863021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588733:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.863037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255424588733:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.862936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241258805982173:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.862959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.862969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241258805982173:2100] [node 4] ICP22 created new session: [6:7587241258805982199:2100]
2025-12-24T02:11:05.862977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241258805982199:2100] [node 4] ICS09 handshake done sender: [6:7587241258805982181:2112] self: [6:7587241258805982174:0] peer: [4:7587241259145976063:0] socket: 367 qp: -1
2025-12-24T02:11:05.862985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241258805982199:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:05.862997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.866794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137356:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.866826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241257269137356:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.870707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976073:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.873097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241259145976060:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.881329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038581:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.881355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038582:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.884671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241257269137373:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9044
2025-12-24T02:11:05.884726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241257269137374:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9044
2025-12-24T02:11:05.883487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241255424588660:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241255424588741:2117] held: no
2025-12-24T02:11:05.883665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241255424588659:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241255424588740:2116] held: no
2025-12-24T02:11:05.883694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255424588741:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.883901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241255424588660:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.883920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.883928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255424588660:2049] [node 2] ICP22 created new session: [3:7587241255424588751:2049]
2025-12-24T02:11:05.883934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255424588751:2049] [node 2] ICS09 handshake done sender: [3:7587241255424588741:2117] self: [3:7587241255424588743:0] peer: [2:7587241258360115104:0] socket: 388 qp: -1
2025-12-24T02:11:05.883942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255424588751:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.883948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.885975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588749:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
2025-12-24T02:11:05.887163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241257269137356:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.887054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255424588733:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:05.887962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137373:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.887981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241257269137348:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.887992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.887999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241257269137348:2109] [node 6] ICP22 created new session: [1:7587241257269137377:2109]
2025-12-24T02:11:05.888004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241257269137377:2109] [node 6] ICS09 handshake done sender: [1:7587241257269137356:2114] self: [1:7587241257269137355:0] peer: [6:7587241258805982197:0] socket: 378 qp: -1
2025-12-24T02:11:05.888011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241257269137377:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:05.888017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.887998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241259129038581:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.888050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241259129038493:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.890839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976074:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.890981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241259145976068:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.904680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241259145976062:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.904713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.904726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241259145976062:2111] [node 6] ICP22 created new session: [4:7587241259145976076:2111]
2025-12-24T02:11:05.904733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241259145976076:2111] [node 6] ICS09 handshake done sender: [4:7587241259145976060:2110] self: [4:7587241259145976063:0] peer: [6:7587241258805982174:0] socket: 369 qp: -1
2025-12-24T02:11:05.904747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241259145976076:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:05.904759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:05.905133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976078:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.905146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976079:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.905701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976080:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.905908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241259145976066:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.905926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.905937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241259145976066:2112] [node 5] ICP22 created new session: [4:7587241259145976081:2112]
2025-12-24T02:11:05.905942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241259145976081:2112] [node 5] ICS09 handshake done sender: [4:7587241259145976068:2114] self: [4:7587241259145976067:0] peer: [5:7587241259129038580:0] socket: 390 qp: -1
2025-12-24T02:11:05.905950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241259145976081:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:05.905957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.905917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241259129038568:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.905258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241258360115026:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241258360115086:2107] held: no
2025-12-24T02:11:05.906560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241258360115107:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.906594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241259129038563:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.906612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.906619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241259129038563:2112] [node 4] ICP22 created new session: [5:7587241259129038584:2112]
2025-12-24T02:11:05.906625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241259129038584:2112] [node 4] ICS09 handshake done sender: [5:7587241259129038568:2115] self: [5:7587241259129038580:0] peer: [4:7587241259145976067:0] socket: 401 qp: -1
2025-12-24T02:11:05.906630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241259129038584:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:05.906650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.906990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241258360115102:2113] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.907008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.907013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241258360115102:2113] [node 4] ICP22 created new session: [2:7587241258360115122:2113]
2025-12-24T02:11:05.907017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241258360115122:2113] [node 4] ICS09 handshake done sender: [2:7587241258360115107:2116] self: [2:7587241258360115103:0] peer: [4:7587241259145976072:0] socket: 383 qp: -1
2025-12-24T02:11:05.907021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241258360115122:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:05.907027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.907320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137362:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.907334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241257269137362:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.909098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241257269137362:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:05.909206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241257269137345:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.909219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:05.909227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241257269137345:2108] [node 5] ICP22 created new session: [1:7587241257269137380:2108]
2025-12-24T02:11:05.909234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241257269137380:2108] [node 5] ICS09 handshake done sender: [1:7587241257269137362:2117] self: [1:7587241257269137352:0] peer: [5:7587241259129038583:0] socket: 407 qp: -1
2025-12-24T02:11:05.909241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241257269137380:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:05.909250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:05.909274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241258805982173:2100] [node 4] ICP35 event processing took too much time PendingConnection:279(46310984ns) {HandleHandshakeStatus:329(46310250ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(322ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(30468900ns) {ForwardSessionEventToSession:612(30460143ns) ForwardSessionEventToSession:612(2143ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(805ns) ForwardSessionEventToSession:612(1369ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.909907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255424588725:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.909933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:05.909938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255424588725:2110] [node 6] ICP22 created new session: [3:7587241255424588754:2110]
2025-12-24T02:11:05.909943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255424588754:2110] [node 6] ICS09 handshake done sender: [3:7587241255424588733:2113] self: [3:7587241255424588732:0] peer: [6:7587241258805982191:0] socket: 371 qp: -1
2025-12-24T02:11:05.909948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255424588754:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:05.909953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.910213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982201:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
2025-12-24T02:11:05.909746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241257269137348:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(21737977ns) {HandleHandshakeStatus:329(21736988ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(318ns) SwitchToState:179(181ns) ProcessPendingSessionEvents:495(23542ns) {ForwardSessionEventToSession:612(15948ns) ForwardSessionEventToSession:612(623ns) ForwardSessionEventToSession:612(684ns) ForwardSessionEventToSession:612(486ns) ForwardSessionEventToSession:612(917ns) ForwardSessionEventToSession:612(913ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(479ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(659ns)}}}
2025-12-24T02:11:05.909780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241257269137348:2109] [node 6] ICP18 handshake (actor [1:7587241257269137373:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.910443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241258805982202:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46698
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.910787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241259129038586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4168
2025-12-24T02:11:05.910925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241259129038581:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.911762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241258805982109:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241258805982187:2116] held: no
2025-12-24T02:11:05.911791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241258805982185:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.912013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241258805982187:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.912567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241258805982111:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.912591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.912598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241258805982111:2050] [node 3] ICP22 created new session: [6:7587241258805982203:2050]
2025-12-24T02:11:05.912604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241258805982203:2050] [node 3] ICS09 handshake done sender: [6:7587241258805982185:2114] self: [6:7587241258805982191:0] peer: [3:7587241255424588732:0] socket: 375 qp: -1
2025-12-24T02:11:05.912610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241258805982203:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.912618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.912898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241258805982109:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.912911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.912918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241258805982109:2048] [node 1] ICP22 created new session: [6:7587241258805982205:2048]
2025-12-24T02:11:05.912923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241258805982205:2048] [node 1] ICS09 handshake done sender: [6:7587241258805982187:2116] self: [6:7587241258805982197:0] peer: [1:7587241257269137355:0] socket: 385 qp: -1
2025-12-24T02:11:05.912928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241258805982205:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.912934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.913101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241259129038493:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.913116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.913121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241259129038493:2048] [node 1] ICP22 created new session: [5:7587241259129038587:2048]
2025-12-24T02:11:05.913125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241259129038587:2048] [node 1] ICS09 handshake done sender: [5:7587241259129038581:2119] self: [5:7587241259129038583:0] peer: [1:7587241257269137352:0] socket: 409 qp: -1
2025-12-24T02:11:05.913129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241259129038587:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.913133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.913811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241258360115124:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61007
2025-12-24T02:11:05.913983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241258360115086:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.913716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241259145976065:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:05.913771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241259145975994:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241259145976065:2113] held: no
2025-12-24T02:11:05.913845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241259145976078:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.913881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241259145975993:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.914160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241258805982111:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:05.910080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255424588740:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.910958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241255424588659:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.910970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.910976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255424588659:2048] [node 1] ICP22 created new session: [3:7587241255424588757:2048]
2025-12-24T02:11:05.910982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255424588757:2048] [node 1] ICS09 handshake done sender: [3:7587241255424588740:2116] self: [3:7587241255424588747:0] peer: [1:7587241257269137357:0] socket: 381 qp: -1
2025-12-24T02:11:05.910988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255424588757:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.910996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.911907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241255424588756:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44856
2025-12-24T02:11:05.914586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241259145975994:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.914601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:05.914607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241259145975994:2049] [node 2] ICP22 created new session: [4:7587241259145976085:2049]
2025-12-24T02:11:05.914611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241259145976085:2049] [node 2] ICS09 handshake done sender: [4:7587241259145976065:2113] self: [4:7587241259145976072:0] peer: [2:7587241258360115103:0] socket: 386 qp: -1
2025-12-24T02:11:05.914617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241259145976085:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:05.914623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:05.914793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588756:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.915309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241257269137361:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.915327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241257269137361:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:05.922959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241259145976079:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.923022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241259145975995:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:05.923188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241255424588738:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:05.923199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241255424588738:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.927574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241258360115026:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.927606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.927621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241258360115026:2048] [node 1] ICP22 created new session: [2:7587241258360115125:2048]
2025-12-24T02:11:05.927628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241258360115125:2048] [node 1] ICS09 handshake done sender: [2:7587241258360115086:2107] self: [2:7587241258360115115:0] peer: [1:7587241257269137341:0] socket: 350 qp: -1
2025-12-24T02:11:05.927636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241258360115125:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.927648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.928507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241257269137361:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.927942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976088:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:05.928956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241259145976089:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27736
2025-12-24T02:11:05.928800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241257269137350:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.928819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.928825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241257269137350:2111] [node 4] ICP22 created new session: [1:7587241257269137383:2111]
2025-12-24T02:11:05.928830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241257269137383:2111] [node 4] ICS09 handshake done sender: [1:7587241257269137361:2116] self: [1:7587241257269137351:0] peer: [4:7587241259145976084:0] socket: 392 qp: -1
2025-12-24T02:11:05.928835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241257269137383:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:05.928841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.929285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241259145976078:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:05.929352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241259145975993:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.929362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:05.929366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241259145975993:2048] [node 1] ICP22 created new session: [4:7587241259145976090:2048]
2025-12-24T02:11:05.929372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241259145976090:2048] [node 1] ICS09 handshake done sender: [4:7587241259145976078:2118] self: [4:7587241259145976084:0] peer: [1:7587241257269137351:0] socket: 415 qp: -1
2025-12-24T02:11:05.929376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241259145976090:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:05.929381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:05.932960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241259145976079:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:05.933005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241259145975995:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:05.933019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:05.933025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241259145975995:2050] [node 3] ICP22 created new session: [4:7587241259145976092:2050]
2025-12-24T02:11:05.934003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241255424588738:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:05.933030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241259145976092:2050] [node 3] ICS09 handshake done sender: [4:7587241259145976079:2119] self: [4:7587241259145976087:0] peer: [3:7587241255424588728:0] socket: 416 qp: -1
2025-12-24T02:11:05.933036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241259145976092:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:05.933041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:05.934113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241255424588723:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:05.934128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:05.934134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241255424588723:2108] [node 4] ICP22 created new session: [3:7587241255424588761:2108]
2025-12-24T02:11:05.934139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241255424588761:2108] [node 4] ICS09 handshake done sender: [3:7587241255424588738:2115] self: [3:7587241255424588728:0] peer: [4:7587241259145976087:0] socket: 368 qp: -1
2025-12-24T02:11:05.934144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241255424588761:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:05.934151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:05.946812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:05.946895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241258360115125:2048] [node 1] ICS01 socket: 350 reason# EndOfStream
2025-12-24T02:11:05.946906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241258360115026:2048] [node 1] ICP30 unregister session Session# [2:7587241258360115125:2048] VirtualId# [2:7587241258360115115:0]
2025-12-24T02:11:05.946913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241258360115125:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.946951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.946970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.947398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:05.947453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241255424588757:2048] [node 1] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:05.947458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241255424588659:2048] [node 1] ICP30 unregister session Session# [3:7587241255424588757:2048] VirtualId# [3:7587241255424588747:0]
2025-12-24T02:11:05.947462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241255424588757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.947486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.947499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.948638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:05.948713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241258805982205:2048] [node 1] ICS01 socket: 385 reason# EndOfStream
2025-12-24T02:11:05.948720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241258805982109:2048] [node 1] ICP30 unregister session Session# [6:7587241258805982205:2048] VirtualId# [6:7587241258805982197:0]
2025-12-24T02:11:05.948726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241258805982205:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.948788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.948817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.953169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:05.953226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241259129038587:2048] [node 1] ICS07 socket disconnect 409 reason# EndOfStream
2025-12-24T02:11:05.953230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241259129038587:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.953257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.953275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241259129038587:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:05.952959Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:05.953038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241259145976090:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-12-24T02:11:05.953044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241259145975993:2048] [node 1] ICP30 unregister session Session# [4:7587241259145976090:2048] VirtualId# [4:7587241259145976084:0]
2025-12-24T02:11:05.953051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241259145976090:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.953110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:05.953133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.955906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241259129038493:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.955921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241259129038587:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.955925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241259129038493:2048] [node 1] ICP30 unregister session Session# [5:7587241259129038587:2048] VirtualId# [5:7587241259129038583:0]
2025-12-24T02:11:05.955940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:05.955945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241259129038493:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.955948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21539) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.958337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:05.958407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241255424588751:2049] [node 2] ICS01 socket: 388 reason# EndOfStream
2025-12-24T02:11:05.958412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241255424588660:2049] [node 2] ICP30 unregister session Session# [3:7587241255424588751:2049] VirtualId# [3:7587241255424588743:0]
2025-12-24T02:11:05.958417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241255424588751:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.958446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.958462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.961574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:05.961663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241258805982189:2049] [node 2] ICS07 socket disconnect 362 reason# EndOfStream
2025-12-24T02:11:05.961669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241258805982189:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.961721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.961738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241258805982189:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:05.963678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:05.963753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241259129038574:2049] [node 2] ICS01 socket: 353 reason# EndOfStream
2025-12-24T02:11:05.963761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241259129038494:2049] [node 2] ICP30 unregister session Session# [5:7587241259129038574:2049] VirtualId# [5:7587241259129038559:0]
2025-12-24T02:11:05.963766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241259129038574:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.963832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.963856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.963889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:05.963947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241259145976085:2049] [node 2] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:05.963951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241259145975994:2049] [node 2] ICP30 unregister session Session# [4:7587241259145976085:2049] VirtualId# [4:7587241259145976072:0]
2025-12-24T02:11:05.963954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241259145976085:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.963981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:05.964001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.966931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241259129038493:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:05.966852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241258805982110:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20462) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.966868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241258805982189:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.966873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241258805982110:2049] [node 2] ICP30 unregister session Session# [6:7587241258805982189:2049] VirtualId# [6:7587241258805982184:0]
2025-12-24T02:11:05.966888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:05.966895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241258805982110:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.966898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20462) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.971360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:05.971452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241258805982203:2050] [node 3] ICS07 socket disconnect 375 reason# EndOfStream
2025-12-24T02:11:05.971458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241258805982203:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.971520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.971543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241258805982203:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:05.971871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:05.971940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241259129038570:2050] [node 3] ICS07 socket disconnect 374 reason# EndOfStream
2025-12-24T02:11:05.971945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241259129038570:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.971987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:05.972010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241259129038570:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:05.972376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241259129038495:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.972387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241259129038570:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.972392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241259129038495:2050] [node 3] ICP30 unregister session Session# [5:7587241259129038570:2050] VirtualId# [5:7587241259129038561:0]
2025-12-24T02:11:05.972402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.972407Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241259129038495:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.972410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.972596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241258805982111:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.972605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241258805982203:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.972610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241258805982111:2050] [node 3] ICP30 unregister session Session# [6:7587241258805982203:2050] VirtualId# [6:7587241258805982191:0]
2025-12-24T02:11:05.972619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:05.972622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241258805982111:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.972623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.977491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241258805982110:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:05.982790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241258805982111:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:05.982875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241259129038495:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:05.985432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:05.985514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241258805982199:2100] [node 4] ICS07 socket disconnect 367 reason# EndOfStream
2025-12-24T02:11:05.985521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241258805982199:2100] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:05.985573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:05.985592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241258805982199:2100] [node 4] ICS15 start handshake
2025-12-24T02:11:05.986250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241258805982173:2100] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22636) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:05.986261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241258805982199:2100] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:05.986265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241258805982173:2100] [node 4] ICP30 unregister session Session# [6:7587241258805982199:2100] VirtualId# [6:7587241258805982174:0]
2025-12-24T02:11:05.986279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:05.986284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241258805982173:2100] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22636) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:05.986287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22636) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:06.009481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241259743974360:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:06.009489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241259743974361:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.010682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241261343460520:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.010704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241261343460521:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.022375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241261592433743:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.022382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241261592433744:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.022387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241261592433745:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.028434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263331380597:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.028455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263331380598:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.019032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263358259495:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.019040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263358259496:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.020778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241260664539875:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.020786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241260664539876:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.020791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241260664539877:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.019045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263358259497:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.070940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241261592433802:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:06.074648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241261592433808:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.075144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241259743974427:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.076046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259557:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.076069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259556:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.075088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460581:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.075157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460580:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.078795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241261592433811:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.078807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241261592433811:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.079385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241261592433811:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.079725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241261592433808:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.079742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.079750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241261592433808:2108] [node 4] ICP22 created new session: [6:7587241261592433818:2108]
2025-12-24T02:11:06.079756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241261592433818:2108] [node 4] ICS09 handshake done sender: [6:7587241261592433811:2110] self: [6:7587241261592433810:0] peer: [4:7587241263358259560:0] socket: 365 qp: -1
2025-12-24T02:11:06.079764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241261592433818:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:06.079774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.079185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460581:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.079378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241261343460583:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:06.079740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241261343460583:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.080855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241261343460588:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:06.080946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241261343460589:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:06.078017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259556:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.078217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263358259559:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:06.078605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263358259559:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.081032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259561:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.081267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263358259556:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.086798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539937:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.086826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241260664539937:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.091479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241259743974432:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.093635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380657:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.092776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460580:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.093218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263358259559:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.093241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.093251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263358259559:2110] [node 6] ICP22 created new session: [4:7587241263358259562:2110]
2025-12-24T02:11:06.093259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263358259562:2110] [node 6] ICS09 handshake done sender: [4:7587241263358259556:2108] self: [4:7587241263358259560:0] peer: [6:7587241261592433810:0] socket: 367 qp: -1
2025-12-24T02:11:06.093267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263358259562:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:06.093292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.098841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241260664539937:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.098889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241259743974434:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:06.098977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263331380661:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:06.099546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259557:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.099884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263358259495:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.101297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259565:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.102302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539941:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.100038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974429:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.100047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241259743974429:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.100717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241259743974429:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.100816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241259743974427:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.100827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.100836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241259743974427:2109] [node 4] ICP22 created new session: [1:7587241259743974440:2109]
2025-12-24T02:11:06.100843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241259743974440:2109] [node 4] ICS09 handshake done sender: [1:7587241259743974429:2110] self: [1:7587241259743974428:0] peer: [4:7587241263358259564:0] socket: 366 qp: -1
2025-12-24T02:11:06.100850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241259743974440:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:06.100877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.101927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380657:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.101973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380663:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.102031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380664:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.102432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380663:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.102664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241260664539876:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.102680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.102686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241260664539876:2049] [node 2] ICP22 created new session: [5:7587241260664539942:2049]
2025-12-24T02:11:06.102704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241260664539942:2049] [node 2] ICS09 handshake done sender: [5:7587241260664539937:2108] self: [5:7587241260664539936:0] peer: [2:7587241261343460584:0] socket: 355 qp: -1
2025-12-24T02:11:06.102711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241260664539942:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.102730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.102888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263358259557:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.102995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263358259495:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.103006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.103014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263358259495:2048] [node 1] ICP22 created new session: [4:7587241263358259566:2048]
2025-12-24T02:11:06.103021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263358259566:2048] [node 1] ICS09 handshake done sender: [4:7587241263358259557:2109] self: [4:7587241263358259564:0] peer: [1:7587241259743974428:0] socket: 368 qp: -1
2025-12-24T02:11:06.103028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263358259566:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.103046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.101746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460597:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.103192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539940:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.106736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974421:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.106758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241259743974421:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.106952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241263358259559:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(25630821ns) {HandleHandshakeStatus:329(25629816ns) {DropIncomingHandshake:442(1036ns) DropOutgoingHandshake:457(80ns) SwitchToState:179(202ns) ProcessPendingSessionEvents:495(592ns)}}
2025-12-24T02:11:06.111864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241261592433815:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.111879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241261592433815:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.113494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241261592433815:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.113590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241261592433745:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.113610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.113618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241261592433745:2050] [node 3] ICP22 created new session: [6:7587241261592433821:2050]
2025-12-24T02:11:06.113625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241261592433821:2050] [node 3] ICS09 handshake done sender: [6:7587241261592433815:2112] self: [6:7587241261592433814:0] peer: [3:7587241263331380667:0] socket: 369 qp: -1
2025-12-24T02:11:06.113633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241261592433821:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.113641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.111760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539941:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.114892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539940:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.115824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263358259569:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:06.116347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241261343460581:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.117717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241261343460583:2109] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.117738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.117745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241261343460583:2109] [node 5] ICP22 created new session: [2:7587241261343460601:2109]
2025-12-24T02:11:06.117753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241261343460601:2109] [node 5] ICS09 handshake done sender: [2:7587241261343460581:2108] self: [2:7587241261343460584:0] peer: [5:7587241260664539936:0] socket: 364 qp: -1
2025-12-24T02:11:06.117759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241261343460601:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:06.117780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.110814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263331380666:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:06.111202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241263331380666:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.112313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263331380669:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:06.116551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974436:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.116576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241259743974436:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.119001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241261343460520:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.119051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241261343460520:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17821731ns) {IncomingHandshake:261(17821262ns) {DropIncomingHandshake:442(67ns) PrepareNewSessionHandshake:477(396ns) {SwitchToState:179(132ns)} GenerateSessionVirtualId:988(104ns) IssueIncomingHandshakeReply:193(17814334ns) {SwitchToState:179(722ns)}}}
2025-12-24T02:11:06.119145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241260664539948:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:06.119297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241261343460588:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20145541ns) {Configure:104(12174184ns) {ProcessConfigured:135(11806666ns) {StartInitialHandshake:157(11806003ns) {DropHandshakes:470(1111ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1260ns) {SwitchToState:179(497ns)}}}}}
2025-12-24T02:11:06.119303Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241263358259495:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16286790ns) {HandleHandshakeStatus:329(16286059ns) {DropIncomingHandshake:442(550ns) DropOutgoingHandshake:457(77ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(490ns)}}
2025-12-24T02:11:06.119261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241259743974443:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38972
2025-12-24T02:11:06.119363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241259743974436:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.121771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241263331380661:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22734195ns) {Configure:104(22732924ns) {ProcessConfigured:135(22427301ns) {StartInitialHandshake:157(22426555ns) {DropHandshakes:470(756ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(133ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(1344ns) {SwitchToState:179(566ns)}}}}}
2025-12-24T02:11:06.122070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380672:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.122743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241259743974432:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.122761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.122766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241259743974432:2111] [node 5] ICP22 created new session: [1:7587241259743974445:2111]
2025-12-24T02:11:06.122773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241259743974445:2111] [node 5] ICS09 handshake done sender: [1:7587241259743974436:2113] self: [1:7587241259743974435:0] peer: [5:7587241260664539944:0] socket: 380 qp: -1
2025-12-24T02:11:06.122780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241259743974445:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:06.122788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.124500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433827:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.125536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974443:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.125599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241259743974421:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.123213Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241260664539876:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20538198ns) {HandleHandshakeStatus:329(20537681ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(270ns) SwitchToState:179(168ns) ProcessPendingSessionEvents:495(16327376ns) {ForwardSessionEventToSession:612(16325562ns)}}}
2025-12-24T02:11:06.125724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241259743974360:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.125738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.125744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241259743974360:2048] [node 2] ICP22 created new session: [1:7587241259743974449:2048]
2025-12-24T02:11:06.125750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241259743974449:2048] [node 2] ICS09 handshake done sender: [1:7587241259743974421:2107] self: [1:7587241259743974420:0] peer: [2:7587241261343460598:0] socket: 354 qp: -1
2025-12-24T02:11:06.125756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241259743974449:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:06.125763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.127168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241259743974360:2048] [node 2] ICP18 handshake (actor [1:7587241259743974443:2115]) is too old
2025-12-24T02:11:06.126833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380664:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.128844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241260664539875:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.128894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241260664539875:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17060400ns) {Configure:104(17059966ns) {ProcessConfigured:135(16713981ns) {IncomingHandshake:261(16703684ns) {DropIncomingHandshake:442(59ns) PrepareNewSessionHandshake:477(365ns) {SwitchToState:179(85ns)} GenerateSessionVirtualId:988(466ns) IssueIncomingHandshakeReply:193(16698394ns) {SwitchToState:179(824ns)}}}}}
2025-12-24T02:11:06.130743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241260664539946:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:06.130940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241261592433827:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.130988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241261592433743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.135262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241263331380597:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44266235ns) {Configure:104(44264479ns) {ProcessConfigured:135(43829135ns) {StartInitialHandshake:157(43828432ns) {DropHandshakes:470(473ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(1257ns) {SwitchToState:179(672ns)}}}}}
2025-12-24T02:11:06.135304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263331380663:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.135713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241259743974451:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38972
2025-12-24T02:11:06.135832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241263358259497:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19930886ns) {Configure:104(19930428ns) {ProcessConfigured:135(19667008ns) {StartInitialHandshake:157(19666673ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(722ns) {SwitchToState:179(115ns)}}}}}
2025-12-24T02:11:06.136005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259574:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.136047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.136057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.136149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241263358259569:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16789310ns) {Configure:104(16788285ns) {ProcessConfigured:135(16508308ns) {StartInitialHandshake:157(16508023ns) {DropHandshakes:470(433ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(100ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(662ns) {SwitchToState:179(261ns)}}}}}
2025-12-24T02:11:06.135381Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241261592433744:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16350013ns) {Configure:104(16349386ns) {ProcessConfigured:135(16009606ns) {StartInitialHandshake:157(16009217ns) {DropHandshakes:470(505ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(538ns) {SwitchToState:179(242ns)}}}}}
2025-12-24T02:11:06.136723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241263358259496:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20842271ns) {Configure:104(20841450ns) {ProcessConfigured:135(20547237ns) {StartInitialHandshake:157(20546841ns) {DropHandshakes:470(461ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(770ns) {SwitchToState:179(255ns)}}}}}
2025-12-24T02:11:06.137762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974438:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.137781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241259743974438:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.137884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974451:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.135158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539956:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.136664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241260664539941:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.138930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259574:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.138965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263358259496:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.146543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539957:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.146571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241260664539875:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.146586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.146596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241260664539875:2048] [node 1] ICP22 created new session: [5:7587241260664539958:2048]
2025-12-24T02:11:06.146604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241260664539958:2048] [node 1] ICS09 handshake done sender: [5:7587241260664539941:2110] self: [5:7587241260664539944:0] peer: [1:7587241259743974435:0] socket: 383 qp: -1
2025-12-24T02:11:06.146611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241260664539958:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.146620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.143734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241261343460520:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15512) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241261343460580:2107] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.143141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241263331380597:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.145948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380677:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.145965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380678:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.146935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241263331380666:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.146943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.146949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263331380666:2111] [node 6] ICP22 created new session: [3:7587241263331380682:2111]
2025-12-24T02:11:06.146954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263331380682:2111] [node 6] ICS09 handshake done sender: [3:7587241263331380663:2109] self: [3:7587241263331380667:0] peer: [6:7587241261592433814:0] socket: 372 qp: -1
2025-12-24T02:11:06.146959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263331380682:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:06.146965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.148414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241259743974424:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.148426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241259743974424:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.148566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259575:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.148612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263358259497:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.149261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241259743974424:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:06.149351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241259743974361:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.149364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.149369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241259743974361:2049] [node 3] ICP22 created new session: [1:7587241259743974454:2049]
2025-12-24T02:11:06.149373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241259743974454:2049] [node 3] ICS09 handshake done sender: [1:7587241259743974424:2108] self: [1:7587241259743974423:0] peer: [3:7587241263331380679:0] socket: 356 qp: -1
2025-12-24T02:11:06.149378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241259743974454:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.149383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:06.146972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539960:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.149448Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460608:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.149768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460607:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.149825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460608:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.154733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460605:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.154768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460600:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.154794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241261343460600:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.154846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539960:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.154898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241260664539877:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.149823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380684:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
2025-12-24T02:11:06.149985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241263331380598:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36341027ns) {Configure:104(34344228ns) {ProcessConfigured:135(34057877ns) {StartInitialHandshake:157(34057379ns) {DropHandshakes:470(445ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(411ns) PrepareNewSessionHandshake:477(754ns) {SwitchToState:179(525ns)}}}}}
2025-12-24T02:11:06.150008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241263331380598:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.155614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259576:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.155033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380671:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.155044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241263331380671:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.156062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263331380657:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.155055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460595:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.155061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241261343460595:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.158392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241261343460606:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56425
2025-12-24T02:11:06.158429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241261343460580:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.156958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380674:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.156970Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241263331380674:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.157309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263331380671:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.163017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263331380677:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.163040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241263331380597:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.163060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.163070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263331380597:2048] [node 1] ICP22 created new session: [3:7587241263331380687:2048]
2025-12-24T02:11:06.163076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263331380687:2048] [node 1] ICS09 handshake done sender: [3:7587241263331380657:2107] self: [3:7587241263331380679:0] peer: [1:7587241259743974423:0] socket: 361 qp: -1
2025-12-24T02:11:06.163084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263331380687:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.163093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.163319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241263331380661:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.163327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.163336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263331380661:2108] [node 4] ICP22 created new session: [3:7587241263331380689:2108]
2025-12-24T02:11:06.163341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263331380689:2108] [node 4] ICS09 handshake done sender: [3:7587241263331380671:2113] self: [3:7587241263331380662:0] peer: [4:7587241263358259583:0] socket: 391 qp: -1
2025-12-24T02:11:06.163349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263331380689:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:06.163357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.163130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433832:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.163153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433831:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.163472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241259743974438:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.163692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241259743974434:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.163712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.163717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241259743974434:2112] [node 6] ICP22 created new session: [1:7587241259743974457:2112]
2025-12-24T02:11:06.163722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241259743974457:2112] [node 6] ICS09 handshake done sender: [1:7587241259743974438:2114] self: [1:7587241259743974437:0] peer: [6:7587241261592433828:0] socket: 393 qp: -1
2025-12-24T02:11:06.163726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241259743974457:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:06.163731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.164691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241261343460520:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.164714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.164722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241261343460520:2048] [node 1] ICP22 created new session: [2:7587241261343460610:2048]
2025-12-24T02:11:06.164729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241261343460610:2048] [node 1] ICS09 handshake done sender: [2:7587241261343460580:2107] self: [2:7587241261343460598:0] peer: [1:7587241259743974420:0] socket: 363 qp: -1
2025-12-24T02:11:06.164738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241261343460610:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.164747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.166845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241261592433831:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.167265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.167298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263358259585:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50963
2025-12-24T02:11:06.167938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263331380692:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51996
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.167063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241261592433827:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.168502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263358259574:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.168595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263358259496:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.168613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.168621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263358259496:2049] [node 2] ICP22 created new session: [4:7587241263358259587:2049]
2025-12-24T02:11:06.168628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263358259587:2049] [node 2] ICS09 handshake done sender: [4:7587241263358259574:2115] self: [4:7587241263358259582:0] peer: [2:7587241261343460594:0] socket: 400 qp: -1
2025-12-24T02:11:06.168634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263358259587:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.168643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.169043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263358259575:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.169660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263331380674:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.169152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241261592433743:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.169175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.169183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241261592433743:2048] [node 1] ICP22 created new session: [6:7587241261592433834:2048]
2025-12-24T02:11:06.169189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241261592433834:2048] [node 1] ICS09 handshake done sender: [6:7587241261592433827:2114] self: [6:7587241261592433828:0] peer: [1:7587241259743974437:0] socket: 396 qp: -1
2025-12-24T02:11:06.169196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241261592433834:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.169202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.167856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241261343460600:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.169999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241261343460595:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.168221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539962:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.169798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241260664539960:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.169961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263358259497:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.169978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.169984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263358259497:2050] [node 3] ICP22 created new session: [4:7587241263358259589:2050]
2025-12-24T02:11:06.169952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241263331380669:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.169990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263358259589:2050] [node 3] ICS09 handshake done sender: [4:7587241263358259575:2116] self: [4:7587241263358259583:0] peer: [3:7587241263331380662:0] socket: 401 qp: -1
2025-12-24T02:11:06.169970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.169995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263358259589:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.170002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.169976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263331380669:2112] [node 5] ICP22 created new session: [3:7587241263331380694:2112]
2025-12-24T02:11:06.169982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263331380694:2112] [node 5] ICS09 handshake done sender: [3:7587241263331380674:2115] self: [3:7587241263331380670:0] peer: [5:7587241260664539961:0] socket: 409 qp: -1
2025-12-24T02:11:06.169988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263331380694:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:06.170008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.171722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241260664539877:2050] [node 3] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.172522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460599:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.172536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241261343460599:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.174826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241261343460599:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.171746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.171757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241260664539877:2050] [node 3] ICP22 created new session: [5:7587241260664539963:2050]
2025-12-24T02:11:06.171763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241260664539963:2050] [node 3] ICS09 handshake done sender: [5:7587241260664539960:2117] self: [5:7587241260664539961:0] peer: [3:7587241263331380670:0] socket: 412 qp: -1
No RDMA context found for GID: 2025-12-24T02:11:06.171770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241260664539963:2050] [node 3] ICS10 traffic start
::ffff:127.1.0.0
2025-12-24T02:11:06.171779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.174805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241260664539946:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43965772ns) {Configure:104(43965349ns) {ProcessConfigured:135(43651613ns) {StartInitialHandshake:157(43635805ns) {DropHandshakes:470(628ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(1163ns) PrepareNewSessionHandshake:477(864ns) {SwitchToState:179(482ns)}} IncomingHandshake:261(7443ns) {DropIncomingHandshake:442(66ns) PrepareNewSessionHandshake:477(179ns) {SwitchToState:179(148ns)} GenerateSessionVirtualId:988(186ns) IssueIncomingHandshakeReply:193(401ns)}}}}
2025-12-24T02:11:06.175091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.176801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263331380664:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.175365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241261343460588:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.175383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.175394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241261343460588:2110] [node 4] ICP22 created new session: [2:7587241261343460615:2110]
2025-12-24T02:11:06.175400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241261343460615:2110] [node 4] ICS09 handshake done sender: [2:7587241261343460600:2116] self: [2:7587241261343460594:0] peer: [4:7587241263358259582:0] socket: 387 qp: -1
2025-12-24T02:11:06.175409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241261343460615:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:06.175417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.175724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241261343460521:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.175741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.175748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241261343460521:2049] [node 3] ICP22 created new session: [2:7587241261343460617:2049]
2025-12-24T02:11:06.175755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241261343460617:2049] [node 3] ICS09 handshake done sender: [2:7587241261343460595:2113] self: [2:7587241261343460590:0] peer: [3:7587241263331380685:0] socket: 376 qp: -1
2025-12-24T02:11:06.175762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241261343460617:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.175768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.178772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433836:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.178979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241261343460606:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.178997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241261343460589:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.179006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.179015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241261343460589:2111] [node 6] ICP22 created new session: [2:7587241261343460619:2111]
2025-12-24T02:11:06.179021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241261343460619:2111] [node 6] ICS09 handshake done sender: [2:7587241261343460599:2115] self: [2:7587241261343460593:0] peer: [6:7587241261592433833:0] socket: 375 qp: -1
2025-12-24T02:11:06.179027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241261343460619:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:06.179036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.182812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241263331380598:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.182840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.182849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263331380598:2049] [node 2] ICP22 created new session: [3:7587241263331380696:2049]
2025-12-24T02:11:06.182855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263331380696:2049] [node 2] ICS09 handshake done sender: [3:7587241263331380664:2110] self: [3:7587241263331380685:0] peer: [2:7587241261343460590:0] socket: 382 qp: -1
2025-12-24T02:11:06.182863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263331380696:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.182874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.183805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241259743974434:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(20077551ns) {HandleHandshakeStatus:329(20077373ns) {DropIncomingHandshake:442(9940ns) DropOutgoingHandshake:457(94ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(20013111ns) {ForwardSessionEventToSession:612(20003310ns) ForwardSessionEventToSession:612(1819ns) ForwardSessionEventToSession:612(1320ns) ForwardSessionEventToSession:612(445ns) ForwardSessionEventToSession:612(424ns) ForwardSessionEventToSession:612(563ns) ForwardSessionEventToSession:612(457ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(929ns) ForwardSessionEventToSession:612(546ns)}}}
2025-12-24T02:11:06.184156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241261343460589:2111] [node 6] ICP18 handshake (actor [2:7587241261343460606:2118]) is too old
2025-12-24T02:11:06.186752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241261592433744:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.186796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241261592433744:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19883390ns) {IncomingHandshake:261(19882862ns) {DropIncomingHandshake:442(14ns) PrepareNewSessionHandshake:477(461ns) {SwitchToState:179(109ns)} GenerateSessionVirtualId:988(183ns) IssueIncomingHandshakeReply:193(19877244ns) {SwitchToState:179(871ns)}}}
2025-12-24T02:11:06.186843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241261592433831:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.190856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241261592433744:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.190886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.190896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241261592433744:2049] [node 2] ICP22 created new session: [6:7587241261592433838:2049]
2025-12-24T02:11:06.190903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241261592433838:2049] [node 2] ICS09 handshake done sender: [6:7587241261592433831:2116] self: [6:7587241261592433833:0] peer: [2:7587241261343460593:0] socket: 411 qp: -1
2025-12-24T02:11:06.190914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241261592433838:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.190924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.192726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241261592433837:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.192766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241261592433802:2105] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.192896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241261343460521:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(17136365ns) {HandleHandshakeStatus:329(17135565ns) {DropIncomingHandshake:442(4139ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(168ns) ProcessPendingSessionEvents:495(8894159ns) {ForwardSessionEventToSession:612(8881355ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(619ns) ForwardSessionEventToSession:612(898ns) ForwardSessionEventToSession:612(963ns) ForwardSessionEventToSession:612(777ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(1041ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(457ns)}}}
2025-12-24T02:11:06.192899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539965:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.192911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241260664539965:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.200573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241261592433841:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3137
2025-12-24T02:11:06.200259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241260664539957:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.200347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241260664539948:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.200732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241260664539965:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.200819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241261592433837:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.200947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241260664539946:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.200967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.200973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241260664539946:2111] [node 6] ICP22 created new session: [5:7587241260664539970:2111]
2025-12-24T02:11:06.200980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241260664539970:2111] [node 6] ICS09 handshake done sender: [5:7587241260664539965:2119] self: [5:7587241260664539955:0] peer: [6:7587241261592433840:0] socket: 378 qp: -1
2025-12-24T02:11:06.200986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241260664539970:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:06.201028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.200525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263358259578:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.200539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241263358259578:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.201415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263358259578:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.201872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241263358259569:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.201891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.201898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263358259569:2113] [node 5] ICP22 created new session: [4:7587241263358259592:2113]
2025-12-24T02:11:06.201906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263358259592:2113] [node 5] ICS09 handshake done sender: [4:7587241263358259578:2118] self: [4:7587241263358259572:0] peer: [5:7587241260664539969:0] socket: 404 qp: -1
2025-12-24T02:11:06.201913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263358259592:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:06.201923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.202429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241260664539972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20215
2025-12-24T02:11:06.202591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241260664539957:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.202658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241260664539948:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.202672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.202678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241260664539948:2112] [node 4] ICP22 created new session: [5:7587241260664539973:2112]
2025-12-24T02:11:06.202686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241260664539973:2112] [node 4] ICS09 handshake done sender: [5:7587241260664539957:2116] self: [5:7587241260664539969:0] peer: [4:7587241263358259572:0] socket: 405 qp: -1
2025-12-24T02:11:06.202710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241260664539973:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:06.202731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.210895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241261592433802:2105] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.210934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.210946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241261592433802:2105] [node 5] ICP22 created new session: [6:7587241261592433842:2105]
2025-12-24T02:11:06.210954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241261592433842:2105] [node 5] ICS09 handshake done sender: [6:7587241261592433837:2119] self: [6:7587241261592433840:0] peer: [5:7587241260664539955:0] socket: 398 qp: -1
2025-12-24T02:11:06.210965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241261592433842:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:06.210978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.211125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241261592433802:2105] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:06.219584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:06.219652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241263331380687:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-12-24T02:11:06.219659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241263331380597:2048] [node 1] ICP30 unregister session Session# [3:7587241263331380687:2048] VirtualId# [3:7587241263331380679:0]
2025-12-24T02:11:06.219665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241263331380687:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.219694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.219711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.219474Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:06.219581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241261343460610:2048] [node 1] ICS01 socket: 363 reason# EndOfStream
2025-12-24T02:11:06.219590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241261343460520:2048] [node 1] ICP30 unregister session Session# [2:7587241261343460610:2048] VirtualId# [2:7587241261343460598:0]
2025-12-24T02:11:06.219598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241261343460610:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.219673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.219699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.229452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:06.229538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241263358259566:2048] [node 1] ICS07 socket disconnect 368 reason# EndOfStream
2025-12-24T02:11:06.229544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241263358259566:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.229601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.229623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241263358259566:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:06.229827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:06.229889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241260664539958:2048] [node 1] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:06.229894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241260664539875:2048] [node 1] ICP30 unregister session Session# [5:7587241260664539958:2048] VirtualId# [5:7587241260664539944:0]
2025-12-24T02:11:06.229898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241260664539958:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.229923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.229945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.229981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:06.230033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241261592433834:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-12-24T02:11:06.230037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241261592433743:2048] [node 1] ICP30 unregister session Session# [6:7587241261592433834:2048] VirtualId# [6:7587241261592433828:0]
2025-12-24T02:11:06.230041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241261592433834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.230064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.230081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.230711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241263358259495:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15512) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.230723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241263358259566:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.230727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241263358259495:2048] [node 1] ICP30 unregister session Session# [4:7587241263358259566:2048] VirtualId# [4:7587241263358259564:0]
2025-12-24T02:11:06.230739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.230744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241263358259495:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15512) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.230746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15512) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.231412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:06.231471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241263331380696:2049] [node 2] ICS01 socket: 382 reason# EndOfStream
2025-12-24T02:11:06.231476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241263331380598:2049] [node 2] ICP30 unregister session Session# [3:7587241263331380696:2049] VirtualId# [3:7587241263331380685:0]
2025-12-24T02:11:06.231480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241263331380696:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.231506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.231536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.271081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241263358259495:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:06.275087Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:06.275145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241261592433838:2049] [node 2] ICS01 socket: 411 reason# EndOfStream
2025-12-24T02:11:06.275149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241261592433744:2049] [node 2] ICP30 unregister session Session# [6:7587241261592433838:2049] VirtualId# [6:7587241261592433833:0]
2025-12-24T02:11:06.275153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241261592433838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.275181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.275200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.274876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:06.274964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241263358259587:2049] [node 2] ICS01 socket: 400 reason# EndOfStream
2025-12-24T02:11:06.274973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241263358259496:2049] [node 2] ICP30 unregister session Session# [4:7587241263358259587:2049] VirtualId# [4:7587241263358259582:0]
2025-12-24T02:11:06.274980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241263358259587:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.275025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.275050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.275231Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:06.275313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241260664539942:2049] [node 2] ICS07 socket disconnect 355 reason# EndOfStream
2025-12-24T02:11:06.275317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241260664539942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.275346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.275375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241260664539942:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:06.276012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241260664539876:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27100) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.276023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241260664539942:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.276029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241260664539876:2049] [node 2] ICP30 unregister session Session# [5:7587241260664539942:2049] VirtualId# [5:7587241260664539936:0]
2025-12-24T02:11:06.276039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.276044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241260664539876:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27100) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.276046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27100) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.286934Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:06.287015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241260664539963:2050] [node 3] ICS01 socket: 412 reason# EndOfStream
2025-12-24T02:11:06.287022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241260664539877:2050] [node 3] ICP30 unregister session Session# [5:7587241260664539963:2050] VirtualId# [5:7587241260664539961:0]
2025-12-24T02:11:06.287029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241260664539963:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.287092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:06.287116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:06.287151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241260664539876:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:06.287259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:06.287316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241261592433821:2050] [node 3] ICS01 socket: 369 reason# EndOfStream
2025-12-24T02:11:06.287319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241261592433745:2050] [node 3] ICP30 unregister session Session# [6:7587241261592433821:2050] VirtualId# [6:7587241261592433814:0]
2025-12-24T02:11:06.287323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241261592433821:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.287349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:06.287379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:06.306948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:06.307034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241261592433818:2108] [node 4] ICS07 socket disconnect 365 reason# EndOfStream
2025-12-24T02:11:06.307040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241261592433818:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.307078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:06.307104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241261592433818:2108] [node 4] ICS15 start handshake
2025-12-24T02:11:06.309888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241261592433808:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5063) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.309903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241261592433818:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.309910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241261592433808:2108] [node 4] ICP30 unregister session Session# [6:7587241261592433818:2108] VirtualId# [6:7587241261592433810:0]
2025-12-24T02:11:06.309926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:06.309932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241261592433808:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.309934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.321673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241261592433808:2108] [node 4] ICP33 wake up from error state
2025-12-24T02:11:06.322761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:06.322841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241261592433842:2105] [node 5] ICS01 socket: 398 reason# EndOfStream
2025-12-24T02:11:06.322850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241261592433802:2105] [node 5] ICP30 unregister session Session# [6:7587241261592433842:2105] VirtualId# [6:7587241261592433840:0]
2025-12-24T02:11:06.322857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241261592433842:2105] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.322884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:06.322904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:06.350467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260965303230:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.350479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260965303231:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.354821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241260985221893:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:06.354842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241260985221894:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.360089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241260310497392:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.360112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241260310497393:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.361792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241262590413489:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.361800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241262590413490:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.361814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241262590413491:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.386916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241261191761785:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.389612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241261191761783:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.393252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241261191761784:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.390930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259799330983:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.390946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259799330984:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.396061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259799330982:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.430866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259799331042:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.433042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331044:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.436044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331049:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.431366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241261191761844:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:06.435943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761851:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.438313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761851:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.444688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259799331044:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.444895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259799331055:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:06.445223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259799331055:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.429229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241262590413548:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:06.447305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413550:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.447320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241262590413550:2106] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.454976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241260985221893:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25609595ns) {Configure:104(25608730ns) {ProcessConfigured:135(22310695ns) {StartInitialHandshake:157(22309488ns) {DropHandshakes:470(372ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(1164ns) {SwitchToState:179(589ns)}}}}}
2025-12-24T02:11:06.455340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241260985221958:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39528
2025-12-24T02:11:06.456803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241260985221960:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:06.457409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259799331049:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.457460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259799331042:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.457839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241260985221962:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.460135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331058:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.460480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259799331049:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.461030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331059:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.461167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241262590413550:2106] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.461403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259799331044:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.461515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241262590413548:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.461533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.461541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241262590413548:2103] [node 6] ICP22 created new session: [4:7587241262590413561:2103]
2025-12-24T02:11:06.461548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241262590413561:2103] [node 6] ICS09 handshake done sender: [4:7587241262590413550:2106] self: [4:7587241262590413549:0] peer: [6:7587241259799331056:0] socket: 363 qp: -1
2025-12-24T02:11:06.461556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241262590413561:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:06.461575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.450959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260965303291:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:06.459410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241260310497456:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:06.459444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761849:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.459460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241261191761849:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.460352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241261191761849:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.462894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761855:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.463058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241261191761844:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.463081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.463088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241261191761844:2105] [node 6] ICP22 created new session: [5:7587241261191761856:2105]
2025-12-24T02:11:06.463095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241261191761856:2105] [node 6] ICS09 handshake done sender: [5:7587241261191761849:2109] self: [5:7587241261191761845:0] peer: [6:7587241259799331057:0] socket: 367 qp: -1
2025-12-24T02:11:06.463102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241261191761856:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:06.463121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.463591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241259799330982:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29758446ns) {Configure:104(29757727ns) {ProcessConfigured:135(29438089ns) {StartInitialHandshake:157(29437677ns) {DropHandshakes:470(746ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(872ns) {SwitchToState:179(391ns)}}}}}
2025-12-24T02:11:06.462963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497461:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
2025-12-24T02:11:06.463891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241260310497450:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:06.466690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241260985221956:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:06.466782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221958:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.470846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241260965303231:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19764964ns) {Configure:104(14561755ns) {ProcessConfigured:135(14288181ns) {StartInitialHandshake:157(14287289ns) {DropHandshakes:470(867ns) {DropIncomingHandshake:442(80ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(1904ns) {SwitchToState:179(987ns)}}}}}
2025-12-24T02:11:06.470914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303300:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
2025-12-24T02:11:06.470934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303301:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
2025-12-24T02:11:06.475144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241262590413489:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18812752ns) {Configure:104(18811496ns) {ProcessConfigured:135(18450300ns) {StartInitialHandshake:157(18449734ns) {DropHandshakes:470(553ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(658ns) PrepareNewSessionHandshake:477(1024ns) {SwitchToState:179(516ns)}}}}}
2025-12-24T02:11:06.476051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241260985221960:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19168054ns) {Configure:104(19167046ns) {ProcessConfigured:135(18885151ns) {StartInitialHandshake:157(18884515ns) {DropHandshakes:470(420ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(836ns) {SwitchToState:179(384ns)}}}}}
2025-12-24T02:11:06.476726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241260985221962:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18844167ns) {Configure:104(18843684ns) {ProcessConfigured:135(18582835ns) {StartInitialHandshake:157(18582448ns) {DropHandshakes:470(425ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(97ns) PrepareNewSessionHandshake:477(494ns) {SwitchToState:179(168ns)}}}}}
2025-12-24T02:11:06.478871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303301:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.479411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761858:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.482242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761858:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.479028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241260310497392:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20028184ns) {Configure:104(20026737ns) {ProcessConfigured:135(19641385ns) {StartInitialHandshake:157(19640657ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(292ns) PrepareNewSessionHandshake:477(1781ns) {SwitchToState:179(774ns)}}}}}
2025-12-24T02:11:06.479284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497464:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.483111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761860:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.480049Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241260985221894:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21148922ns) {Configure:104(21148240ns) {ProcessConfigured:135(20708030ns) {StartInitialHandshake:157(20707668ns) {DropHandshakes:470(491ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(960ns) {SwitchToState:179(353ns)}}}}}
2025-12-24T02:11:06.482493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221965:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.482500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241260985221965:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.483500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303295:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
2025-12-24T02:11:06.483288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241260985221965:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.483824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241260985221960:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.483846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.483853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241260985221960:2107] [node 5] ICP22 created new session: [1:7587241260985221974:2107]
2025-12-24T02:11:06.483860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241260985221974:2107] [node 5] ICS09 handshake done sender: [1:7587241260985221965:2111] self: [1:7587241260985221961:0] peer: [5:7587241261191761859:0] socket: 394 qp: -1
2025-12-24T02:11:06.483868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241260985221974:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:06.483888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.482927Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241260310497393:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23458421ns) {Configure:104(23457262ns) {ProcessConfigured:135(23185270ns) {StartInitialHandshake:157(23184605ns) {DropHandshakes:470(280ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(101ns) PrepareNewSessionHandshake:477(907ns) {SwitchToState:179(443ns)}}}}}
2025-12-24T02:11:06.486791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303300:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.486727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259799331055:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.486762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.486774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259799331055:2113] [node 4] ICP22 created new session: [6:7587241259799331062:2113]
2025-12-24T02:11:06.486784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259799331062:2113] [node 4] ICS09 handshake done sender: [6:7587241259799331044:2107] self: [6:7587241259799331056:0] peer: [4:7587241262590413549:0] socket: 364 qp: -1
2025-12-24T02:11:06.486793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259799331062:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:06.486815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.486851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497464:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.486889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241260310497453:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:06.486905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241260310497393:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.487016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260965303297:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:06.487395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260965303298:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:06.495071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761855:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.506813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241260310497450:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(42851817ns) {Configure:104(42851021ns) {ProcessConfigured:135(42560179ns) {StartInitialHandshake:157(42559212ns) {DropHandshakes:470(402ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(1494ns) {SwitchToState:179(66ns)}}}}}
2025-12-24T02:11:06.507544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497461:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.509273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.510074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241259799331055:2113] [node 4] ICP35 event processing took too much time PendingConnection:279(48175222ns) {HandleHandshakeStatus:329(48174412ns) {DropIncomingHandshake:442(988ns) DropOutgoingHandshake:457(162ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(777ns)}}
2025-12-24T02:11:06.510242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259799331042:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.510262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.510272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259799331042:2106] [node 5] ICP22 created new session: [6:7587241259799331065:2106]
2025-12-24T02:11:06.510281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259799331065:2106] [node 5] ICS09 handshake done sender: [6:7587241259799331049:2110] self: [6:7587241259799331057:0] peer: [5:7587241261191761845:0] socket: 368 qp: -1
2025-12-24T02:11:06.510290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259799331065:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:06.510317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.510532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221957:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.510550Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241260985221957:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.510729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497459:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.510746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241260310497459:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.510786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241261191761783:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.510827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241261191761783:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28517654ns) {IncomingHandshake:261(28517011ns) {DropIncomingHandshake:442(174ns) PrepareNewSessionHandshake:477(361ns) {SwitchToState:179(91ns)} GenerateSessionVirtualId:988(356ns) IssueIncomingHandshakeReply:193(28512267ns) {SwitchToState:179(472ns)}}}
2025-12-24T02:11:06.510870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241261191761858:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.510948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303306:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.510964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260965303306:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.511160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241261191761783:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241261191761858:2113] held: no
2025-12-24T02:11:06.511057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241262590413568:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:06.511251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241261191761783:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.511265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.511273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241261191761783:2048] [node 1] ICP22 created new session: [5:7587241261191761862:2048]
2025-12-24T02:11:06.511281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241261191761862:2048] [node 1] ICS09 handshake done sender: [5:7587241261191761858:2113] self: [5:7587241261191761859:0] peer: [1:7587241260985221961:0] socket: 398 qp: -1
2025-12-24T02:11:06.511290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241261191761862:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.511313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.522275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259799331064:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.522338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259799330982:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.522803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221967:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.522817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241260985221967:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.524978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241260985221967:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.525059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241260985221956:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.525070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.525078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241260985221956:2106] [node 6] ICP22 created new session: [1:7587241260985221979:2106]
2025-12-24T02:11:06.525086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241260985221979:2106] [node 6] ICS09 handshake done sender: [1:7587241260985221967:2112] self: [1:7587241260985221966:0] peer: [6:7587241259799331067:0] socket: 395 qp: -1
2025-12-24T02:11:06.525093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241260985221979:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:06.525103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.526865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497473:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.526853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303295:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.526904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303312:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
2025-12-24T02:11:06.527280Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241260310497453:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40326666ns) {Configure:104(40325260ns) {ProcessConfigured:135(39933602ns) {StartInitialHandshake:157(39932648ns) {DropHandshakes:470(558ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(686ns) PrepareNewSessionHandshake:477(1289ns) {SwitchToState:179(780ns)}}}}}
2025-12-24T02:11:06.527357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497473:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.527379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241260310497392:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.527449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221971:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.527466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241260985221971:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.527059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413569:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.527120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413570:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.527289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413570:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.527332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241262590413489:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.527672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413569:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.536373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241260965303298:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(48913068ns) {Configure:104(48912425ns) {ProcessConfigured:135(48728808ns) {StartInitialHandshake:157(48728162ns) {DropHandshakes:470(204ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(965ns) {SwitchToState:179(231ns)}}}}}
2025-12-24T02:11:06.537118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241260965303230:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.537141Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241260965303230:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(50290118ns) {IncomingHandshake:261(50289718ns) {DropIncomingHandshake:442(65ns) PrepareNewSessionHandshake:477(567ns) {SwitchToState:179(112ns)} GenerateSessionVirtualId:988(204ns) IssueIncomingHandshakeReply:193(50284822ns) {SwitchToState:179(245ns)}}}
2025-12-24T02:11:06.539166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303299:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.539181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260965303299:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.538738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241261191761785:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.538848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241261191761785:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43657745ns) {Configure:104(43656835ns) {ProcessConfigured:135(43223811ns) {IncomingHandshake:261(43193467ns) {DropIncomingHandshake:442(358ns) PrepareNewSessionHandshake:477(788ns) {SwitchToState:179(369ns)} GenerateSessionVirtualId:988(511ns) IssueIncomingHandshakeReply:193(43186875ns) {SwitchToState:179(507ns)}}}}}
2025-12-24T02:11:06.539097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241261191761865:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:06.539561Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241261191761783:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28302384ns) {HandleHandshakeStatus:329(28301870ns) {DropIncomingHandshake:442(356ns) DropOutgoingHandshake:457(48ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(891ns)}}
2025-12-24T02:11:06.546895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303312:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.546739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241260985221960:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(62852107ns) {HandleHandshakeStatus:329(62850891ns) {DropIncomingHandshake:442(4036ns) DropOutgoingHandshake:457(354ns) SwitchToState:179(109ns) ProcessPendingSessionEvents:495(36209455ns) {ForwardSessionEventToSession:612(36199552ns) ForwardSessionEventToSession:612(2688ns) ForwardSessionEventToSession:612(990ns) ForwardSessionEventToSession:612(841ns) ForwardSessionEventToSession:612(475ns) ForwardSessionEventToSession:612(1297ns) ForwardSessionEventToSession:612(727ns)}}}
2025-12-24T02:11:06.546922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221972:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.546926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241260985221972:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.550905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497477:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
2025-12-24T02:11:06.550999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303316:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.553551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241260985221972:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.553951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241260310497459:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.553772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241260985221957:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.554224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241260985221971:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.553380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413574:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.555386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241260985221985:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39528
2025-12-24T02:11:06.553221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260965303317:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19000
2025-12-24T02:11:06.554132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260965303306:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.554547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260965303300:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.554814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241260310497473:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.556388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241260310497392:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.556418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.556434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241260310497392:2048] [node 1] ICP22 created new session: [3:7587241260310497479:2048]
2025-12-24T02:11:06.556441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241260310497479:2048] [node 1] ICS09 handshake done sender: [3:7587241260310497473:2116] self: [3:7587241260310497476:0] peer: [1:7587241260985221964:0] socket: 406 qp: -1
2025-12-24T02:11:06.556450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241260310497479:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.556462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.554905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241262590413490:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.554942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241262590413490:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27218178ns) {IncomingHandshake:261(27217863ns) {DropIncomingHandshake:442(21ns) PrepareNewSessionHandshake:477(503ns) {SwitchToState:179(214ns)} GenerateSessionVirtualId:988(152ns) IssueIncomingHandshakeReply:193(27214105ns) {SwitchToState:179(1042ns)}}}
2025-12-24T02:11:06.558945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241260965303297:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71839553ns) {Configure:104(71838649ns) {ProcessConfigured:135(71503912ns) {StartInitialHandshake:157(71503180ns) {DropHandshakes:470(464ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(1782ns) {SwitchToState:179(1184ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.558981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241260965303230:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.558997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.559007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260965303230:2048] [node 1] ICP22 created new session: [2:7587241260965303321:2048]
2025-12-24T02:11:06.559013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260965303321:2048] [node 1] ICS09 handshake done sender: [2:7587241260965303300:2112] self: [2:7587241260965303309:0] peer: [1:7587241260985221952:0] socket: 382 qp: -1
2025-12-24T02:11:06.559022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260965303321:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.559047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.560280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241260985221894:2049] [node 3] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.560306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.560317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241260985221894:2049] [node 3] ICP22 created new session: [1:7587241260985221989:2049]
2025-12-24T02:11:06.560325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241260985221989:2049] [node 3] ICS09 handshake done sender: [1:7587241260985221972:2114] self: [1:7587241260985221964:0] peer: [3:7587241260310497476:0] socket: 405 qp: -1
2025-12-24T02:11:06.560333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241260985221989:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.560345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.560898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413574:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.561199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260965303231:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.561222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.561229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260965303231:2049] [node 3] ICP22 created new session: [2:7587241260965303323:2049]
2025-12-24T02:11:06.561235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260965303323:2049] [node 3] ICS09 handshake done sender: [2:7587241260965303306:2115] self: [2:7587241260965303296:0] peer: [3:7587241260310497470:0] socket: 392 qp: -1
2025-12-24T02:11:06.561242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260965303323:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.561253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.562346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241259799330984:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20310034ns) {Configure:104(20308106ns) {ProcessConfigured:135(19929002ns) {StartInitialHandshake:157(19928208ns) {DropHandshakes:470(906ns) {DropIncomingHandshake:442(125ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(391ns) PrepareNewSessionHandshake:477(970ns) {SwitchToState:179(599ns)}}}}}
2025-12-24T02:11:06.564074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241260985221986:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39528
2025-12-24T02:11:06.564159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241260985221962:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.564171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.564182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241260985221962:2108] [node 4] ICP22 created new session: [1:7587241260985221991:2108]
2025-12-24T02:11:06.564190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241260985221991:2108] [node 4] ICS09 handshake done sender: [1:7587241260985221971:2113] self: [1:7587241260985221963:0] peer: [4:7587241262590413571:0] socket: 362 qp: -1
2025-12-24T02:11:06.564197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241260985221991:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:06.564207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.564573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241260985221988:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39528
2025-12-24T02:11:06.564627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241260985221893:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.564639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.564645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241260985221893:2048] [node 2] ICP22 created new session: [1:7587241260985221993:2048]
2025-12-24T02:11:06.564651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241260985221993:2048] [node 2] ICS09 handshake done sender: [1:7587241260985221957:2109] self: [1:7587241260985221952:0] peer: [2:7587241260965303309:0] socket: 373 qp: -1
2025-12-24T02:11:06.564657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241260985221993:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:06.564664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.565263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221985:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.562598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241260310497456:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.562625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.562633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241260310497456:2107] [node 5] ICP22 created new session: [3:7587241260310497481:2107]
2025-12-24T02:11:06.562639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241260310497481:2107] [node 5] ICS09 handshake done sender: [3:7587241260310497459:2108] self: [3:7587241260310497458:0] peer: [5:7587241261191761861:0] socket: 377 qp: -1
2025-12-24T02:11:06.562648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241260310497481:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:06.562670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.566788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241259799331042:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(106156510ns) {HandleHandshakeStatus:329(106155665ns) {DropIncomingHandshake:442(788ns) DropOutgoingHandshake:457(2138ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(29712203ns) {ForwardSessionEventToSession:612(29709642ns)}}}
2025-12-24T02:11:06.566827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241259799331042:2106] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:06.566876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331069:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.566944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221986:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.566977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241260985221956:2106] [node 6] ICP18 handshake (actor [1:7587241260985221986:2117]) is too old
2025-12-24T02:11:06.566934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331070:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.567123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259799331064:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.567182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259799330982:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.567196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.567202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259799330982:2048] [node 1] ICP22 created new session: [6:7587241259799331073:2048]
2025-12-24T02:11:06.567207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259799331073:2048] [node 1] ICS09 handshake done sender: [6:7587241259799331064:2117] self: [6:7587241259799331067:0] peer: [1:7587241260985221966:0] socket: 403 qp: -1
2025-12-24T02:11:06.567212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259799331073:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.567219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.567557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241260985221987:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39528
2025-12-24T02:11:06.575949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241260985221987:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.577063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761872:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.577217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.576568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241260310497393:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14410) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241260310497464:2111] held: no
2025-12-24T02:11:06.576746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497483:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
2025-12-24T02:11:06.576849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241260310497484:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62014
2025-12-24T02:11:06.576368Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260965303299:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.577403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260965303291:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.577421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.577428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260965303291:2107] [node 4] ICP22 created new session: [2:7587241260965303327:2107]
2025-12-24T02:11:06.577434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260965303327:2107] [node 4] ICS09 handshake done sender: [2:7587241260965303299:2111] self: [2:7587241260965303293:0] peer: [4:7587241262590413572:0] socket: 387 qp: -1
2025-12-24T02:11:06.577440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260965303327:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:06.577459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.578957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241261191761855:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.579188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241262590413490:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14410) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241262590413569:2114] held: no
2025-12-24T02:11:06.579553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241261191761785:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.579583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.579593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241261191761785:2050] [node 3] ICP22 created new session: [5:7587241261191761874:2050]
2025-12-24T02:11:06.579600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241261191761874:2050] [node 3] ICS09 handshake done sender: [5:7587241261191761855:2112] self: [5:7587241261191761861:0] peer: [3:7587241260310497458:0] socket: 384 qp: -1
2025-12-24T02:11:06.579609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241261191761874:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.579622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.579209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241260985221962:2108] [node 4] ICP18 handshake (actor [1:7587241260985221987:2118]) is too old
2025-12-24T02:11:06.579419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241260985221893:2048] [node 2] ICP18 handshake (actor [1:7587241260985221985:2116]) is too old
2025-12-24T02:11:06.580384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241262590413568:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(69239807ns) {Configure:104(69238393ns) {ProcessConfigured:135(41177100ns) {StartInitialHandshake:157(41176164ns) {DropHandshakes:470(1434ns) {DropIncomingHandshake:442(131ns) DropOutgoingHandshake:457(152ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(739ns) {SwitchToState:179(427ns)}}}}}
2025-12-24T02:11:06.579637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303317:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.588780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259799331070:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.588833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259799330983:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.589010Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241259799330982:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21793697ns) {HandleHandshakeStatus:329(21793391ns) {DropIncomingHandshake:442(256ns) DropOutgoingHandshake:457(1191ns) SwitchToState:179(111ns) ProcessPendingSessionEvents:495(21697302ns) {ForwardSessionEventToSession:612(21686461ns) ForwardSessionEventToSession:612(1172ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(1183ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(686ns) ForwardSessionEventToSession:612(626ns) ForwardSessionEventToSession:612(780ns) ForwardSessionEventToSession:612(888ns)}}}
2025-12-24T02:11:06.589333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241260985221894:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:06.588628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241260310497464:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.589586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497484:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.589596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241260310497393:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.589609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.589616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241260310497393:2049] [node 2] ICP22 created new session: [3:7587241260310497487:2049]
2025-12-24T02:11:06.589622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241260310497487:2049] [node 2] ICS09 handshake done sender: [3:7587241260310497464:2111] self: [3:7587241260310497470:0] peer: [2:7587241260965303296:0] socket: 396 qp: -1
2025-12-24T02:11:06.589630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241260310497487:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.589638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.591211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303313:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.591224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260965303313:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.590967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241260310497392:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34541406ns) {HandleHandshakeStatus:329(34539930ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(2509ns) SwitchToState:179(134ns) ProcessPendingSessionEvents:495(23613770ns) {ForwardSessionEventToSession:612(23599229ns) ForwardSessionEventToSession:612(2408ns) ForwardSessionEventToSession:612(1374ns) ForwardSessionEventToSession:612(682ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(1414ns) ForwardSessionEventToSession:612(762ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(1324ns)}}}
2025-12-24T02:11:06.591625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331075:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.603048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.603509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241262590413570:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.604451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331077:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.604563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761873:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.604621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241261191761784:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.604898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260965303313:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.605240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259799331075:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.605266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259799331070:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.605350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260965303320:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.605358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260965303320:2120] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.606006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260965303320:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.606045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260965303298:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.606067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.606075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260965303298:2109] [node 6] ICP22 created new session: [2:7587241260965303332:2109]
2025-12-24T02:11:06.606082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260965303332:2109] [node 6] ICS09 handshake done sender: [2:7587241260965303313:2117] self: [2:7587241260965303311:0] peer: [6:7587241259799331076:0] socket: 422 qp: -1
2025-12-24T02:11:06.606088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260965303332:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:06.606098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.606478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761877:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.606616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259799330984:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.606900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259799330983:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.606913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.606920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259799330983:2049] [node 2] ICP22 created new session: [6:7587241259799331079:2049]
2025-12-24T02:11:06.606925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259799331079:2049] [node 2] ICS09 handshake done sender: [6:7587241259799331070:2119] self: [6:7587241259799331076:0] peer: [2:7587241260965303311:0] socket: 426 qp: -1
2025-12-24T02:11:06.606930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259799331079:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.606936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.607028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241261191761873:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.607129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241261191761784:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.607150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.607155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241261191761784:2049] [node 2] ICP22 created new session: [5:7587241261191761878:2049]
2025-12-24T02:11:06.607160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241261191761878:2049] [node 2] ICS09 handshake done sender: [5:7587241261191761873:2119] self: [5:7587241261191761876:0] peer: [2:7587241260965303310:0] socket: 430 qp: -1
2025-12-24T02:11:06.607165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241261191761878:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.607171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.607625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260965303297:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.607643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.607649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260965303297:2108] [node 5] ICP22 created new session: [2:7587241260965303334:2108]
2025-12-24T02:11:06.607654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260965303334:2108] [node 5] ICS09 handshake done sender: [2:7587241260965303320:2120] self: [2:7587241260965303310:0] peer: [5:7587241261191761876:0] socket: 402 qp: -1
2025-12-24T02:11:06.607658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260965303334:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:06.607664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.606785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497472:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.606802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241260310497472:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.610808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413578:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.610876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413579:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.610946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241262590413489:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.610962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.610968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241262590413489:2048] [node 1] ICP22 created new session: [4:7587241262590413580:2048]
2025-12-24T02:11:06.610974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241262590413580:2048] [node 1] ICS09 handshake done sender: [4:7587241262590413570:2115] self: [4:7587241262590413571:0] peer: [1:7587241260985221963:0] socket: 408 qp: -1
2025-12-24T02:11:06.610980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241262590413580:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.610989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.616096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259799331081:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7516
2025-12-24T02:11:06.616479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259799331075:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.616579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259799330984:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.616602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.616609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259799330984:2050] [node 3] ICP22 created new session: [6:7587241259799331082:2050]
2025-12-24T02:11:06.616615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259799331082:2050] [node 3] ICS09 handshake done sender: [6:7587241259799331075:2121] self: [6:7587241259799331078:0] peer: [3:7587241260310497462:0] socket: 383 qp: -1
2025-12-24T02:11:06.616625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259799331082:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.616634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.616356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241260310497472:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.616795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241260310497450:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.616812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.616817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241260310497450:2104] [node 6] ICP22 created new session: [3:7587241260310497491:2104]
2025-12-24T02:11:06.616823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241260310497491:2104] [node 6] ICS09 handshake done sender: [3:7587241260310497472:2115] self: [3:7587241260310497462:0] peer: [6:7587241259799331078:0] socket: 428 qp: -1
2025-12-24T02:11:06.616828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241260310497491:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:06.616849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.623450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241259799330983:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16507959ns) {HandleHandshakeStatus:329(16507361ns) {DropIncomingHandshake:442(149ns) DropOutgoingHandshake:457(1037ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(15781191ns) {ForwardSessionEventToSession:612(15768927ns) ForwardSessionEventToSession:612(2307ns) ForwardSessionEventToSession:612(749ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(562ns) ForwardSessionEventToSession:612(1374ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(1430ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(1132ns)}}}
2025-12-24T02:11:06.624350Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241260965303297:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(16690550ns) {HandleHandshakeStatus:329(16689925ns) {DropIncomingHandshake:442(3154ns) DropOutgoingHandshake:457(115ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(16620576ns) {ForwardSessionEventToSession:612(16608467ns) ForwardSessionEventToSession:612(2559ns) ForwardSessionEventToSession:612(1125ns) ForwardSessionEventToSession:612(949ns) ForwardSessionEventToSession:612(552ns) ForwardSessionEventToSession:612(1162ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(583ns) ForwardSessionEventToSession:612(1190ns)}}}
2025-12-24T02:11:06.626740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241260310497474:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.626755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241260310497474:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.627572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241260310497474:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.624825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413579:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.626190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241262590413569:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.626267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241262590413490:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.626281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.626289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241262590413490:2049] [node 2] ICP22 created new session: [4:7587241262590413583:2049]
2025-12-24T02:11:06.626295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241262590413583:2049] [node 2] ICS09 handshake done sender: [4:7587241262590413569:2114] self: [4:7587241262590413572:0] peer: [2:7587241260965303293:0] socket: 390 qp: -1
2025-12-24T02:11:06.626303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241262590413583:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.626312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.626556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241262590413491:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.627668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241260310497453:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.627686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.627692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241260310497453:2106] [node 4] ICP22 created new session: [3:7587241260310497494:2106]
2025-12-24T02:11:06.627698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241260310497494:2106] [node 4] ICS09 handshake done sender: [3:7587241260310497474:2117] self: [3:7587241260310497471:0] peer: [4:7587241262590413585:0] socket: 429 qp: -1
2025-12-24T02:11:06.627703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241260310497494:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:06.627723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.628720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761880:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.628707Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241262590413491:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241262590413579:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.634743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241262590413586:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43270
2025-12-24T02:11:06.642958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241261191761880:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.643020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241261191761865:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.643135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241262590413579:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.643483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241262590413575:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.643489Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241262590413575:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.644353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241261191761882:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2565
2025-12-24T02:11:06.644538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241262590413575:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.644562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241262590413491:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.644578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.644585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241262590413491:2050] [node 3] ICP22 created new session: [4:7587241262590413588:2050]
2025-12-24T02:11:06.644591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241262590413588:2050] [node 3] ICS09 handshake done sender: [4:7587241262590413579:2120] self: [4:7587241262590413585:0] peer: [3:7587241260310497471:0] socket: 401 qp: -1
2025-12-24T02:11:06.644597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241262590413588:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.644605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.644976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241262590413568:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.644992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.644997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241262590413568:2113] [node 5] ICP22 created new session: [4:7587241262590413590:2113]
2025-12-24T02:11:06.645001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241262590413590:2113] [node 5] ICS09 handshake done sender: [4:7587241262590413575:2117] self: [4:7587241262590413573:0] peer: [5:7587241261191761881:0] socket: 380 qp: -1
2025-12-24T02:11:06.645005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241262590413590:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:06.645011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.645380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241261191761880:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.645563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241261191761865:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.645577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.645581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241261191761865:2115] [node 4] ICP22 created new session: [5:7587241261191761883:2115]
2025-12-24T02:11:06.645587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241261191761883:2115] [node 4] ICS09 handshake done sender: [5:7587241261191761880:2121] self: [5:7587241261191761881:0] peer: [4:7587241262590413573:0] socket: 413 qp: -1
2025-12-24T02:11:06.645592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241261191761883:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:06.645601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.657319Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:06.657409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241260965303321:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-12-24T02:11:06.657418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241260965303230:2048] [node 1] ICP30 unregister session Session# [2:7587241260965303321:2048] VirtualId# [2:7587241260965303309:0]
2025-12-24T02:11:06.657425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241260965303321:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.657468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.657489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.662792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:06.662890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241260310497479:2048] [node 1] ICS07 socket disconnect 406 reason# EndOfStream
2025-12-24T02:11:06.662896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241260310497479:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.662961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.662985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241260310497479:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:06.663476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241260310497392:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.663493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241260310497479:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.663500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241260310497392:2048] [node 1] ICP30 unregister session Session# [3:7587241260310497479:2048] VirtualId# [3:7587241260310497476:0]
2025-12-24T02:11:06.663518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.663524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241260310497392:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.663526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.665361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:06.665445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241261191761862:2048] [node 1] ICS07 socket disconnect 398 reason# EndOfStream
2025-12-24T02:11:06.665452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241261191761862:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.665505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.665528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241261191761862:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:06.666003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241261191761783:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.666017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241261191761862:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.666022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241261191761783:2048] [node 1] ICP30 unregister session Session# [5:7587241261191761862:2048] VirtualId# [5:7587241261191761859:0]
2025-12-24T02:11:06.666039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.666044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241261191761783:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.666048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.669626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:06.669744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241259799331073:2048] [node 1] ICS07 socket disconnect 403 reason# EndOfStream
2025-12-24T02:11:06.669750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259799331073:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.669819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.669846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241259799331073:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:06.674380Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:06.674454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241260310497487:2049] [node 2] ICS01 socket: 396 reason# EndOfStream
2025-12-24T02:11:06.674461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241260310497393:2049] [node 2] ICP30 unregister session Session# [3:7587241260310497487:2049] VirtualId# [3:7587241260310497470:0]
2025-12-24T02:11:06.674467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241260310497487:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.674500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.674519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.674650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241260310497392:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:06.671371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:06.671457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241262590413580:2048] [node 1] ICS07 socket disconnect 408 reason# EndOfStream
2025-12-24T02:11:06.671463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241262590413580:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.671523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.671547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241262590413580:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:06.671863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241262590413489:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.671872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241262590413580:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.671877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241262590413489:2048] [node 1] ICP30 unregister session Session# [4:7587241262590413580:2048] VirtualId# [4:7587241262590413571:0]
2025-12-24T02:11:06.671893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.671899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241262590413489:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.671901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.674910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259799330982:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.674925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259799331073:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.674930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259799330982:2048] [node 1] ICP30 unregister session Session# [6:7587241259799331073:2048] VirtualId# [6:7587241259799331067:0]
2025-12-24T02:11:06.674944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.674949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241259799330982:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.674951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26778) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.675594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241261191761783:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:06.682076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:06.682170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241262590413583:2049] [node 2] ICS01 socket: 390 reason# EndOfStream
2025-12-24T02:11:06.682178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241262590413490:2049] [node 2] ICP30 unregister session Session# [4:7587241262590413583:2049] VirtualId# [4:7587241262590413572:0]
2025-12-24T02:11:06.682185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241262590413583:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.682245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.682270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.682305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241262590413489:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:06.682876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:06.682949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241259799331079:2049] [node 2] ICS07 socket disconnect 426 reason# EndOfStream
2025-12-24T02:11:06.682953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259799331079:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.682999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.683021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241259799331079:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:06.683473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259799330983:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14410) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.683485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259799331079:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.683489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259799330983:2049] [node 2] ICP30 unregister session Session# [6:7587241259799331079:2049] VirtualId# [6:7587241259799331076:0]
2025-12-24T02:11:06.683503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.683507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241259799330983:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.683511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.684602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:06.684690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241261191761878:2049] [node 2] ICS01 socket: 430 reason# EndOfStream
2025-12-24T02:11:06.684697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241261191761784:2049] [node 2] ICP30 unregister session Session# [5:7587241261191761878:2049] VirtualId# [5:7587241261191761876:0]
2025-12-24T02:11:06.684703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241261191761878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.684780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.684811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.686061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241259799330982:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:06.695265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241259799330983:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:06.697553Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:06.697648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241259799331082:2050] [node 3] ICS07 socket disconnect 383 reason# EndOfStream
2025-12-24T02:11:06.697654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259799331082:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.697710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:06.697730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241259799331082:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:06.700536Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:06.700615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241261191761874:2050] [node 3] ICS07 socket disconnect 384 reason# EndOfStream
2025-12-24T02:11:06.700620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241261191761874:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.700667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:06.700687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241261191761874:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:06.702946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259799330984:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.702963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259799331082:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.702968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259799330984:2050] [node 3] ICP30 unregister session Session# [6:7587241259799331082:2050] VirtualId# [6:7587241259799331078:0]
2025-12-24T02:11:06.702982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:06.702987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241259799330984:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.702989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.702935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241261191761785:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.702951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241261191761874:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.702956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241261191761785:2050] [node 3] ICP30 unregister session Session# [5:7587241261191761874:2050] VirtualId# [5:7587241261191761861:0]
2025-12-24T02:11:06.702973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:06.702979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241261191761785:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.702982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.709056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:06.709148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241259799331062:2113] [node 4] ICS07 socket disconnect 364 reason# EndOfStream
2025-12-24T02:11:06.709153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259799331062:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.709212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:06.709234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241259799331062:2113] [node 4] ICS15 start handshake
2025-12-24T02:11:06.709678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259799331055:2113] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.709689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259799331062:2113] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.709693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259799331055:2113] [node 4] ICP30 unregister session Session# [6:7587241259799331062:2113] VirtualId# [6:7587241259799331056:0]
2025-12-24T02:11:06.709708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:06.709713Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241259799331055:2113] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.709715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1705) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.717530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241259799330984:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:06.722748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241259799331055:2113] [node 4] ICP33 wake up from error state
2025-12-24T02:11:06.728078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:06.728168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241259799331065:2106] [node 5] ICS07 socket disconnect 368 reason# EndOfStream
2025-12-24T02:11:06.728175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259799331065:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.728214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:06.728236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241259799331065:2106] [node 5] ICS15 start handshake
2025-12-24T02:11:06.728743Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259799331042:2106] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1290) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.728758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259799331065:2106] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.728764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259799331042:2106] [node 5] ICP30 unregister session Session# [6:7587241259799331065:2106] VirtualId# [6:7587241259799331057:0]
2025-12-24T02:11:06.728782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:06.728790Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241259799331042:2106] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.728793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1290) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:06.758763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241261634279533:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:06.758776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241261634279534:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.761658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263698766790:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.761666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263698766791:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.760120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260510756126:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.760132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260510756127:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:06.769929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241261744078478:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.769945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241261744078479:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.769950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241261744078480:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.771756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241263654974187:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.771764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241263654974188:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.771769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241263654974189:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.794761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259528086552:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:06.794807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259528086553:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:06.794811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259528086554:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:06.810994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259528086608:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:06.811609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260510756187:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:06.811231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241261634279595:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:06.811345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241261634279596:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:06.812483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241261634279606:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:06.812713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241263654974246:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:06.814912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263698766850:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.823563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263698766856:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:06.824083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241263698766859:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:06.824529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766854:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.824543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766855:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.825375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241263654974261:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:06.825427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974259:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.825441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974260:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.845394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241261744078543:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:06.845514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241261744078544:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:06.845589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078541:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.845606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078542:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.844637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241259528086554:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18189744ns) {Configure:104(18188745ns) {ProcessConfigured:135(17673276ns) {StartInitialHandshake:157(17672536ns) {DropHandshakes:470(730ns) {DropIncomingHandshake:442(113ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(742ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(853ns)}}}}}
2025-12-24T02:11:06.844841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241259528086623:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.847267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260510756197:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:06.847358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241260510756198:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:06.847415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260510756196:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64551
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.854727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241261634279612:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44883
2025-12-24T02:11:06.855020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241261634279613:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44883
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.862949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241261634279614:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44883
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.866870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086617:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.866959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086618:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.871111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756196:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.874894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279612:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.875805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259528086617:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.876016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259528086552:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.876132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.876616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279601:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.876632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241261634279601:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.877174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241261634279616:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44883
2025-12-24T02:11:06.882859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241261634279601:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.886867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241261634279595:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.886889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.886900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241261634279595:2107] [node 6] ICP22 created new session: [1:7587241261634279618:2107]
2025-12-24T02:11:06.886906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241261634279618:2107] [node 6] ICS09 handshake done sender: [1:7587241261634279601:2111] self: [1:7587241261634279600:0] peer: [6:7587241259528086633:0] socket: 372 qp: -1
2025-12-24T02:11:06.886918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241261634279618:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:06.886929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.895440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078542:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.904493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086634:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.906815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974259:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.910738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260510756205:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64551
2025-12-24T02:11:06.910873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241260510756126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.911309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279599:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.911323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241261634279599:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.911968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241261634279621:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44883
2025-12-24T02:11:06.914904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241261634279599:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.915004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241261634279533:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.915017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.915027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241261634279533:2048] [node 2] ICP22 created new session: [1:7587241261634279623:2048]
2025-12-24T02:11:06.915034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241261634279623:2048] [node 2] ICS09 handshake done sender: [1:7587241261634279599:2110] self: [1:7587241261634279598:0] peer: [2:7587241260510756207:0] socket: 371 qp: -1
2025-12-24T02:11:06.915041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241261634279623:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:06.915050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.916010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259528086617:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.916122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259528086552:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.916133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.916140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259528086552:2048] [node 1] ICP22 created new session: [6:7587241259528086635:2048]
2025-12-24T02:11:06.916145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259528086635:2048] [node 1] ICS09 handshake done sender: [6:7587241259528086617:2109] self: [6:7587241259528086633:0] peer: [1:7587241261634279600:0] socket: 378 qp: -1
2025-12-24T02:11:06.916151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259528086635:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.916159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.926334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078558:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.923744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279616:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.923872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241261634279595:2107] [node 6] ICP18 handshake (actor [1:7587241261634279616:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.927131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078558:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.927162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241261744078478:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.928567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078541:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.928596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241261744078479:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.926905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974268:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.927732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974268:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.927754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241263654974187:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.927386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756205:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.927432Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260510756208:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64551
2025-12-24T02:11:06.927633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260510756209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64551
2025-12-24T02:11:06.927664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260510756196:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.927877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241260510756126:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.927892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.927899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260510756126:2048] [node 1] ICP22 created new session: [2:7587241260510756211:2048]
2025-12-24T02:11:06.927905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260510756211:2048] [node 1] ICS09 handshake done sender: [2:7587241260510756196:2112] self: [2:7587241260510756207:0] peer: [1:7587241261634279598:0] socket: 379 qp: -1
2025-12-24T02:11:06.927913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260510756211:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.927921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.928775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756189:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.928779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260510756189:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.930475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279614:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.930613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260510756189:2107] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.930771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.931877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260510756187:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.931896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.931957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260510756187:2105] [node 4] ICP22 created new session: [2:7587241260510756215:2105]
2025-12-24T02:11:06.931964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260510756215:2105] [node 4] ICS09 handshake done sender: [2:7587241260510756189:2107] self: [2:7587241260510756188:0] peer: [4:7587241261744078561:0] socket: 367 qp: -1
2025-12-24T02:11:06.931970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260510756215:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:06.931978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.932025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078562:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.932060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279605:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.932077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241261634279605:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.932709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241261744078541:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.932972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241261744078479:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.932997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.933003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241261744078479:2049] [node 2] ICP22 created new session: [4:7587241261744078563:2049]
2025-12-24T02:11:06.933008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241261744078563:2049] [node 2] ICS09 handshake done sender: [4:7587241261744078541:2110] self: [4:7587241261744078561:0] peer: [2:7587241260510756188:0] socket: 376 qp: -1
2025-12-24T02:11:06.933014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241261744078563:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.933020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.934332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259528086630:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.934411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259528086623:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.936504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756209:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.937294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.937354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974270:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.937383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241263654974188:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.937519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756202:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.937531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260510756202:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.938835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279608:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.938852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241261634279608:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.939417Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241261634279533:2048] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.939720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.939902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078548:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.939912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241261744078548:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.940551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086640:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.942806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241261634279605:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.943396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974260:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.943429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241263654974189:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.943764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078565:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.944044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241261744078558:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.944175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241261744078548:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.944259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.943092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766855:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.943286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766871:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.943334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766872:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.943374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.943396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241263698766790:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.944498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259528086630:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.944608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259528086623:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.944623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.944628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259528086623:2108] [node 4] ICP22 created new session: [6:7587241259528086641:2108]
2025-12-24T02:11:06.944634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259528086641:2108] [node 4] ICS09 handshake done sender: [6:7587241259528086630:2117] self: [6:7587241259528086638:0] peer: [4:7587241261744078547:0] socket: 398 qp: -1
2025-12-24T02:11:06.944640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259528086641:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:06.944646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.945103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260510756202:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.944370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241261744078478:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.944388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.944394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241261744078478:2048] [node 1] ICP22 created new session: [4:7587241261744078568:2048]
2025-12-24T02:11:06.944400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241261744078568:2048] [node 1] ICS09 handshake done sender: [4:7587241261744078558:2117] self: [4:7587241261744078560:0] peer: [1:7587241261634279604:0] socket: 407 qp: -1
2025-12-24T02:11:06.944405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241261744078568:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.944412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.944519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241261744078543:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.944525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.944529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241261744078543:2104] [node 6] ICP22 created new session: [4:7587241261744078570:2104]
2025-12-24T02:11:06.944533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241261744078570:2104] [node 6] ICS09 handshake done sender: [4:7587241261744078548:2113] self: [4:7587241261744078547:0] peer: [6:7587241259528086638:0] socket: 394 qp: -1
2025-12-24T02:11:06.944536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241261744078570:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:06.944541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.945314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241263654974270:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.945393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241263654974188:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.945415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.945420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241263654974188:2049] [node 2] ICP22 created new session: [5:7587241263654974275:2049]
2025-12-24T02:11:06.945426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241263654974275:2049] [node 2] ICS09 handshake done sender: [5:7587241263654974270:2118] self: [5:7587241263654974271:0] peer: [2:7587241260510756201:0] socket: 415 qp: -1
2025-12-24T02:11:06.945433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241263654974275:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.945441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.946075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086643:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.946834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766871:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.946864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766872:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.947004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766861:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.947018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241263698766861:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.947281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260510756197:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.947298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.947304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260510756197:2110] [node 5] ICP22 created new session: [2:7587241260510756220:2110]
2025-12-24T02:11:06.947314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260510756220:2110] [node 5] ICS09 handshake done sender: [2:7587241260510756202:2114] self: [2:7587241260510756201:0] peer: [5:7587241263654974271:0] socket: 414 qp: -1
2025-12-24T02:11:06.947320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260510756220:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:06.947333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.947463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766854:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.947624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241263698766791:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.948027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241261634279608:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.948149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279613:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.948177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241261634279603:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.948179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241261634279603:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.948507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241261634279596:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.948518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.948522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241261634279596:2108] [node 4] ICP22 created new session: [1:7587241261634279629:2108]
2025-12-24T02:11:06.948526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241261634279629:2108] [node 4] ICS09 handshake done sender: [1:7587241261634279605:2113] self: [1:7587241261634279604:0] peer: [4:7587241261744078560:0] socket: 406 qp: -1
2025-12-24T02:11:06.948534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241261634279629:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:06.948540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.948689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241261634279606:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.948700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.948704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241261634279606:2109] [node 5] ICP22 created new session: [1:7587241261634279631:2109]
2025-12-24T02:11:06.948708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241261634279631:2109] [node 5] ICS09 handshake done sender: [1:7587241261634279608:2114] self: [1:7587241261634279607:0] peer: [5:7587241263654974269:0] socket: 411 qp: -1
2025-12-24T02:11:06.948713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241261634279631:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:06.948717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.949109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263698766861:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.948966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259528086618:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.948988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259528086639:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.949006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259528086554:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.949013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259528086553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.949203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766863:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.949210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241263698766863:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.949561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241263698766850:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.949571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.949575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263698766850:2103] [node 5] ICP22 created new session: [3:7587241263698766879:2103]
2025-12-24T02:11:06.949578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263698766879:2103] [node 5] ICS09 handshake done sender: [3:7587241263698766861:2113] self: [3:7587241263698766860:0] peer: [5:7587241263654974273:0] socket: 382 qp: -1
2025-12-24T02:11:06.949582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263698766879:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:06.949587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.950119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241261634279603:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.950178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241261634279534:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.950189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.950194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241261634279534:2049] [node 3] ICP22 created new session: [1:7587241261634279635:2049]
2025-12-24T02:11:06.950198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241261634279635:2049] [node 3] ICS09 handshake done sender: [1:7587241261634279603:2112] self: [1:7587241261634279602:0] peer: [3:7587241263698766874:0] socket: 373 qp: -1
2025-12-24T02:11:06.950204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241261634279635:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.950209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.950652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766882:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.950787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086646:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.950995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263698766863:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.951074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241259528086643:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.952456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756186:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.952472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260510756186:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.953538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259528086618:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.953519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241260510756204:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.953531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241260510756204:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.954243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241260510756222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64551
2025-12-24T02:11:06.954340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260510756186:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.954430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241263654974187:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241263654974268:2117] held: no
2025-12-24T02:11:06.954466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241263654974189:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25860) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241263654974260:2114] held: no
2025-12-24T02:11:06.954453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241259528086608:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.954465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259528086554:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.954476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.954481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259528086554:2050] [node 3] ICP22 created new session: [6:7587241259528086648:2050]
2025-12-24T02:11:06.954486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259528086648:2050] [node 3] ICS09 handshake done sender: [6:7587241259528086618:2110] self: [6:7587241259528086644:0] peer: [3:7587241263698766862:0] socket: 387 qp: -1
2025-12-24T02:11:06.954493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259528086648:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.954501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.954780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241260510756204:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.955404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086650:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.955546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259528086639:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.955619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259528086553:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.955632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.955636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259528086553:2049] [node 2] ICP22 created new session: [6:7587241259528086651:2049]
2025-12-24T02:11:06.955641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259528086651:2049] [node 2] ICS09 handshake done sender: [6:7587241259528086639:2119] self: [6:7587241259528086645:0] peer: [2:7587241260510756203:0] socket: 409 qp: -1
2025-12-24T02:11:06.955646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259528086651:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.955652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.956291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241263698766790:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241263698766855:2111] held: no
2025-12-24T02:11:06.956317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241263698766856:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.956328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.956333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263698766856:2108] [node 6] ICP22 created new session: [3:7587241263698766884:2108]
2025-12-24T02:11:06.956338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263698766884:2108] [node 6] ICS09 handshake done sender: [3:7587241263698766863:2114] self: [3:7587241263698766862:0] peer: [6:7587241259528086644:0] socket: 383 qp: -1
2025-12-24T02:11:06.956343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263698766884:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:06.956350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.956514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766873:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.956562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263698766855:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:06.956711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260510756127:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.956724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.956729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260510756127:2049] [node 3] ICP22 created new session: [2:7587241260510756225:2049]
2025-12-24T02:11:06.956734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260510756225:2049] [node 3] ICS09 handshake done sender: [2:7587241260510756186:2106] self: [2:7587241260510756185:0] peer: [3:7587241263698766878:0] socket: 366 qp: -1
2025-12-24T02:11:06.956739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260510756225:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:06.956746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.956834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241260510756198:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.956841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.956845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241260510756198:2111] [node 6] ICP22 created new session: [2:7587241260510756227:2111]
2025-12-24T02:11:06.956848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241260510756227:2111] [node 6] ICS09 handshake done sender: [2:7587241260510756204:2115] self: [2:7587241260510756203:0] peer: [6:7587241259528086645:0] socket: 400 qp: -1
2025-12-24T02:11:06.956852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241260510756227:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:06.956856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.956899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974265:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.956910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241263654974265:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.957539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241263654974268:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.957715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241263654974187:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.957729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.957735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241263654974187:2048] [node 1] ICP22 created new session: [5:7587241263654974281:2048]
2025-12-24T02:11:06.957740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241263654974281:2048] [node 1] ICS09 handshake done sender: [5:7587241263654974268:2117] self: [5:7587241263654974269:0] peer: [1:7587241261634279607:0] socket: 412 qp: -1
2025-12-24T02:11:06.957747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241263654974281:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.957753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.957922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974278:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.957970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974279:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.958274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241263654974265:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:06.958394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241263654974278:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.958505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241263654974261:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.958519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:06.958524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241263654974261:2111] [node 6] ICP22 created new session: [5:7587241263654974283:2111]
2025-12-24T02:11:06.958528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241263654974283:2111] [node 6] ICS09 handshake done sender: [5:7587241263654974265:2116] self: [5:7587241263654974264:0] peer: [6:7587241259528086647:0] socket: 395 qp: -1
2025-12-24T02:11:06.958533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241263654974283:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:06.958540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:06.958801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241263654974260:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.959147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241259528086653:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20281
2025-12-24T02:11:06.959069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241263654974189:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.959081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.959086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241263654974189:2050] [node 3] ICP22 created new session: [5:7587241263654974285:2050]
2025-12-24T02:11:06.959091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241263654974285:2050] [node 3] ICS09 handshake done sender: [5:7587241263654974260:2114] self: [5:7587241263654974273:0] peer: [3:7587241263698766860:0] socket: 385 qp: -1
2025-12-24T02:11:06.959096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241263654974285:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.959101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.959200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241263654974246:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.961222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241263654974246:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241263654974278:2121] held: no
2025-12-24T02:11:06.961273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241263654974288:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53371
2025-12-24T02:11:06.961658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241263654974278:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.961721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241263654974246:2105] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.961732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.961736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241263654974246:2105] [node 4] ICP22 created new session: [5:7587241263654974289:2105]
2025-12-24T02:11:06.961740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241263654974289:2105] [node 4] ICS09 handshake done sender: [5:7587241263654974278:2121] self: [5:7587241263654974287:0] peer: [4:7587241261744078553:0] socket: 431 qp: -1
2025-12-24T02:11:06.961744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241263654974289:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:06.961751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.959428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078554:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.959456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241261744078554:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:06.960124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241261744078554:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.960195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241261744078544:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.960208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.960213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241261744078544:2106] [node 5] ICP22 created new session: [4:7587241261744078574:2106]
2025-12-24T02:11:06.960218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241261744078574:2106] [node 5] ICS09 handshake done sender: [4:7587241261744078554:2116] self: [4:7587241261744078553:0] peer: [5:7587241263654974287:0] socket: 429 qp: -1
2025-12-24T02:11:06.960223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241261744078574:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:06.960229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.962355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078576:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.961452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241263698766790:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.961466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:06.961471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263698766790:2048] [node 1] ICP22 created new session: [3:7587241263698766888:2048]
2025-12-24T02:11:06.961476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263698766888:2048] [node 1] ICS09 handshake done sender: [3:7587241263698766855:2111] self: [3:7587241263698766874:0] peer: [1:7587241261634279602:0] socket: 380 qp: -1
2025-12-24T02:11:06.961482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263698766888:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:06.961489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:06.962244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241263698766886:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1125
2025-12-24T02:11:06.962633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263698766854:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:06.962769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241263698766791:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.962784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:06.962788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263698766791:2049] [node 2] ICP22 created new session: [3:7587241263698766891:2049]
2025-12-24T02:11:06.962793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263698766891:2049] [node 2] ICS09 handshake done sender: [3:7587241263698766854:2110] self: [3:7587241263698766878:0] peer: [2:7587241260510756185:0] socket: 375 qp: -1
2025-12-24T02:11:06.962799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263698766891:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:06.962804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:06.963538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241259528086608:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31696) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241259528086643:2121] held: no
2025-12-24T02:11:06.963583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241259528086643:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:06.963641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241259528086608:2102] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.963665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:06.963670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241259528086608:2102] [node 5] ICP22 created new session: [6:7587241259528086654:2102]
2025-12-24T02:11:06.963675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241259528086654:2102] [node 5] ICS09 handshake done sender: [6:7587241259528086643:2121] self: [6:7587241259528086647:0] peer: [5:7587241263654974264:0] socket: 428 qp: -1
2025-12-24T02:11:06.963679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241259528086654:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:06.963685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:06.964806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241260510756127:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:06.967633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241261744078576:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.967674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241261744078480:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:06.967910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241263698766867:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:06.967918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241263698766867:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:06.968353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241261744078578:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6470
2025-12-24T02:11:06.968438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241263698766867:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:06.968559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241261744078576:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:06.968610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241261744078480:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:06.968643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:06.968652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241261744078480:2050] [node 3] ICP22 created new session: [4:7587241261744078579:2050]
2025-12-24T02:11:06.968658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241261744078579:2050] [node 3] ICS09 handshake done sender: [4:7587241261744078576:2120] self: [4:7587241261744078577:0] peer: [3:7587241263698766866:0] socket: 424 qp: -1
2025-12-24T02:11:06.968664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241261744078579:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:06.968673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:06.969232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241263698766859:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:06.969246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:06.969250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241263698766859:2107] [node 4] ICP22 created new session: [3:7587241263698766894:2107]
2025-12-24T02:11:06.969253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241263698766894:2107] [node 4] ICS09 handshake done sender: [3:7587241263698766867:2116] self: [3:7587241263698766866:0] peer: [4:7587241261744078577:0] socket: 370 qp: -1
2025-12-24T02:11:06.969257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241263698766894:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:06.969262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:06.972610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:06.972683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241260510756211:2048] [node 1] ICS01 socket: 379 reason# EndOfStream
2025-12-24T02:11:06.972691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241260510756126:2048] [node 1] ICP30 unregister session Session# [2:7587241260510756211:2048] VirtualId# [2:7587241260510756207:0]
2025-12-24T02:11:06.972697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241260510756211:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.972724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.972743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.973589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:06.973641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241263698766888:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-12-24T02:11:06.973647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241263698766790:2048] [node 1] ICP30 unregister session Session# [3:7587241263698766888:2048] VirtualId# [3:7587241263698766874:0]
2025-12-24T02:11:06.973650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241263698766888:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.973685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.973701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.975367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:06.975434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259528086635:2048] [node 1] ICS01 socket: 378 reason# EndOfStream
2025-12-24T02:11:06.975439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259528086552:2048] [node 1] ICP30 unregister session Session# [6:7587241259528086635:2048] VirtualId# [6:7587241259528086633:0]
2025-12-24T02:11:06.975444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259528086635:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.975482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.975504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.975626Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:06.975704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241261744078568:2048] [node 1] ICS01 socket: 407 reason# EndOfStream
2025-12-24T02:11:06.975717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241261744078478:2048] [node 1] ICP30 unregister session Session# [4:7587241261744078568:2048] VirtualId# [4:7587241261744078560:0]
2025-12-24T02:11:06.975725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241261744078568:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.975758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.975783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.976609Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:06.976679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241263654974281:2048] [node 1] ICS01 socket: 412 reason# EndOfStream
2025-12-24T02:11:06.976684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241263654974187:2048] [node 1] ICP30 unregister session Session# [5:7587241263654974281:2048] VirtualId# [5:7587241263654974269:0]
2025-12-24T02:11:06.976690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241263654974281:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.976736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:06.976761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:06.982801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:06.982901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241263698766891:2049] [node 2] ICS07 socket disconnect 375 reason# EndOfStream
2025-12-24T02:11:06.982906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241263698766891:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.982946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.982978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241263698766891:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:06.985197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:06.985273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259528086651:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-12-24T02:11:06.985280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259528086553:2049] [node 2] ICP30 unregister session Session# [6:7587241259528086651:2049] VirtualId# [6:7587241259528086645:0]
2025-12-24T02:11:06.985286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259528086651:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.985327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.985347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.986808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:06.987020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:06.987081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241263654974275:2049] [node 2] ICS01 socket: 415 reason# EndOfStream
2025-12-24T02:11:06.987088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241263654974188:2049] [node 2] ICP30 unregister session Session# [5:7587241263654974275:2049] VirtualId# [5:7587241263654974271:0]
2025-12-24T02:11:06.987093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241263654974275:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.987116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.987138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.986895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241261744078563:2049] [node 2] ICS07 socket disconnect 376 reason# EndOfStream
2025-12-24T02:11:06.986906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241261744078563:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:06.986957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:06.986979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241261744078563:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:06.995066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241261744078479:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:06.995086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241261744078563:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:06.995094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241261744078479:2049] [node 2] ICP30 unregister session Session# [4:7587241261744078563:2049] VirtualId# [4:7587241261744078561:0]
2025-12-24T02:11:06.995113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:06.995120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241261744078479:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:06.995122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:07.010752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241261744078479:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:07.015619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:07.015720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241261744078579:2050] [node 3] ICS01 socket: 424 reason# EndOfStream
2025-12-24T02:11:07.015730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241261744078480:2050] [node 3] ICP30 unregister session Session# [4:7587241261744078579:2050] VirtualId# [4:7587241261744078577:0]
2025-12-24T02:11:07.015737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241261744078579:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.015804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.015832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.017302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:07.017394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259528086648:2050] [node 3] ICS01 socket: 387 reason# EndOfStream
2025-12-24T02:11:07.017403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259528086554:2050] [node 3] ICP30 unregister session Session# [6:7587241259528086648:2050] VirtualId# [6:7587241259528086644:0]
2025-12-24T02:11:07.017410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259528086648:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.017473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.017501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.020730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:07.020851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241263654974285:2050] [node 3] ICS01 socket: 385 reason# EndOfStream
2025-12-24T02:11:07.020861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241263654974189:2050] [node 3] ICP30 unregister session Session# [5:7587241263654974285:2050] VirtualId# [5:7587241263654974273:0]
2025-12-24T02:11:07.020867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241263654974285:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.020926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.020951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.025785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:07.025899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241259528086641:2108] [node 4] ICS01 socket: 398 reason# EndOfStream
2025-12-24T02:11:07.025910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241259528086623:2108] [node 4] ICP30 unregister session Session# [6:7587241259528086641:2108] VirtualId# [6:7587241259528086638:0]
2025-12-24T02:11:07.025916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241259528086641:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.025977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:07.026005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:07.075649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241266859714879:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:07.075673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241266859714880:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.081805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266817594446:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.081829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266817594447:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.082118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241265025891127:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.082133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241265025891128:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.082139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241265025891129:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.080577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241264834256336:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.080589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241264834256337:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.106749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267456033102:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.106779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267456033103:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.106790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267456033104:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.104978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265419331679:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.105022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265419331680:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.105028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265419331681:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.137835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241265025891189:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.139261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331743:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.139381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241266859714941:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55149
2025-12-24T02:11:07.139396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241266859714942:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55149
2025-12-24T02:11:07.139088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267456033161:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.139822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256396:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.140735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331744:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.143478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266817594511:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:07.143565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266817594512:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:07.143997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594506:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.144347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331745:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.144503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256397:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.146900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241266859714944:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55149
2025-12-24T02:11:07.151678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241264834256402:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.152253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891199:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.154284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594522:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
2025-12-24T02:11:07.151427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241266859714948:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:07.152136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241266859714953:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:07.152210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241266859714954:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.159120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241266859714961:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55149
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.162870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033168:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.163147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594523:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
2025-12-24T02:11:07.185866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241264834256410:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:07.186286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241264834256414:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:07.195856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267456033171:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:07.212043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241267456033103:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16127812ns) {Configure:104(16127129ns) {ProcessConfigured:135(15832257ns) {StartInitialHandshake:157(15832025ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(674ns) {SwitchToState:179(243ns)}}}}}
2025-12-24T02:11:07.211020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241265419331680:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16227900ns) {Configure:104(16224951ns) {ProcessConfigured:135(15827140ns) {StartInitialHandshake:157(15826107ns) {DropHandshakes:470(1072ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(316ns)} GenerateSessionVirtualId:988(316ns) PrepareNewSessionHandshake:477(1778ns) {SwitchToState:179(895ns)}}}}}
2025-12-24T02:11:07.211209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265419331751:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.215292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714941:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.215363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241266859714962:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55149
2025-12-24T02:11:07.214838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331744:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.222096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265419331749:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:07.238868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267456033168:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.238924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267456033102:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.239036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714962:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.246304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714950:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.246333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241266859714950:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.247341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266817594526:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:07.247510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594523:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.247585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
2025-12-24T02:11:07.248087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594522:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.248102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241266817594446:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.247612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256417:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
2025-12-24T02:11:07.247683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256418:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
2025-12-24T02:11:07.250801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331745:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.250910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265419331680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.250951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256417:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.251037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594525:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.251074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594516:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.251077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266817594516:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.251452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594506:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.251699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266817594516:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.251786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266817594511:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.251802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.251807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266817594511:2108] [node 5] ICP22 created new session: [2:7587241266817594534:2108]
2025-12-24T02:11:07.251812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266817594534:2108] [node 5] ICS09 handshake done sender: [2:7587241266817594516:2112] self: [2:7587241266817594515:0] peer: [5:7587241265419331761:0] socket: 388 qp: -1
2025-12-24T02:11:07.251818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266817594534:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:07.251826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.255062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256396:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.255115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241264834256336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.250799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241265025891201:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:07.251761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891199:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.251794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241265025891129:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.257589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256404:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.257614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241264834256404:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.258731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714952:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.258766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241266859714952:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.258839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331743:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.259236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265419331751:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.260244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714944:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.260748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241264834256404:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.260667Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891195:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.260682Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241265025891195:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.261653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241265025891195:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.261772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241265025891189:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.261782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.261790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241265025891189:2108] [node 5] ICP22 created new session: [4:7587241265025891208:2108]
2025-12-24T02:11:07.261796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241265025891208:2108] [node 5] ICS09 handshake done sender: [4:7587241265025891195:2111] self: [4:7587241265025891194:0] peer: [5:7587241265419331762:0] socket: 376 qp: -1
2025-12-24T02:11:07.261822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241265025891208:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:07.261843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.261115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241264834256402:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.261144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.261154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241264834256402:2110] [node 4] ICP22 created new session: [3:7587241264834256422:2110]
2025-12-24T02:11:07.261161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241264834256422:2110] [node 4] ICS09 handshake done sender: [3:7587241264834256404:2111] self: [3:7587241264834256403:0] peer: [4:7587241265025891206:0] socket: 397 qp: -1
2025-12-24T02:11:07.261169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241264834256422:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:07.261182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.262612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241266859714950:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.262807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256397:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.262848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241264834256337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.264157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256426:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.266153Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241265419331680:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17248) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241265419331745:2111] held: no
2025-12-24T02:11:07.266293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331765:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
2025-12-24T02:11:07.266332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331763:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
2025-12-24T02:11:07.270360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714940:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.270385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241266859714940:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.271262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256426:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.271312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256418:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.271431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241264834256402:2110] [node 4] ICP18 handshake (actor [3:7587241264834256426:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.274795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594514:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.274834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266817594514:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.274871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331764:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
2025-12-24T02:11:07.275333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331765:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.275376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265419331745:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.276011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714942:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.276037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241266859714948:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.276060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.276071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241266859714948:2111] [node 6] ICP22 created new session: [1:7587241266859714969:2111]
2025-12-24T02:11:07.276083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241266859714969:2111] [node 6] ICS09 handshake done sender: [1:7587241266859714950:2114] self: [1:7587241266859714949:0] peer: [6:7587241267456033181:0] socket: 400 qp: -1
2025-12-24T02:11:07.276093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241266859714969:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:07.276107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.276696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241266859714952:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.279134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266817594514:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.278916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265419331743:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.278990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265419331680:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.279004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.279011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265419331680:2049] [node 2] ICP22 created new session: [5:7587241265419331766:2049]
2025-12-24T02:11:07.279018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265419331766:2049] [node 2] ICS09 handshake done sender: [5:7587241265419331745:2111] self: [5:7587241265419331761:0] peer: [2:7587241266817594515:0] socket: 391 qp: -1
2025-12-24T02:11:07.279025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265419331766:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.279034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.279326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266817594447:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.279343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.279348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266817594447:2049] [node 3] ICP22 created new session: [2:7587241266817594538:2049]
2025-12-24T02:11:07.279352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266817594538:2049] [node 3] ICS09 handshake done sender: [2:7587241266817594514:2111] self: [2:7587241266817594513:0] peer: [3:7587241264834256425:0] socket: 390 qp: -1
2025-12-24T02:11:07.279356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266817594538:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.279362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.279493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241265025891129:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17538) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241265025891199:2112] held: no
2025-12-24T02:11:07.280081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256413:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.280091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241264834256413:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.279659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265419331681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.279687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265419331751:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.279694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.279697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265419331751:2112] [node 4] ICP22 created new session: [5:7587241265419331770:2112]
2025-12-24T02:11:07.279702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265419331770:2112] [node 4] ICS09 handshake done sender: [5:7587241265419331743:2109] self: [5:7587241265419331762:0] peer: [4:7587241265025891194:0] socket: 377 qp: -1
2025-12-24T02:11:07.279706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265419331770:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:07.279711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.282493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241264834256413:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.282948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241266859714940:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.283496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714961:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.283539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241266859714879:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.283561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.283569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241266859714879:2048] [node 2] ICP22 created new session: [1:7587241266859714973:2048]
2025-12-24T02:11:07.283576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241266859714973:2048] [node 2] ICS09 handshake done sender: [1:7587241266859714952:2115] self: [1:7587241266859714951:0] peer: [2:7587241266817594529:0] socket: 401 qp: -1
2025-12-24T02:11:07.283584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241266859714973:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:07.283595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.283751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241266859714880:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.283758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.283762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241266859714880:2049] [node 3] ICP22 created new session: [1:7587241266859714975:2049]
2025-12-24T02:11:07.283767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241266859714975:2049] [node 3] ICS09 handshake done sender: [1:7587241266859714940:2107] self: [1:7587241266859714939:0] peer: [3:7587241264834256420:0] socket: 380 qp: -1
2025-12-24T02:11:07.283772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241266859714975:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.283776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.284321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241266859714880:2049] [node 3] ICP18 handshake (actor [1:7587241266859714961:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.290947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241264834256410:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.290983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.290994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241264834256410:2113] [node 5] ICP22 created new session: [3:7587241264834256431:2113]
2025-12-24T02:11:07.291002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241264834256431:2113] [node 5] ICS09 handshake done sender: [3:7587241264834256413:2115] self: [3:7587241264834256412:0] peer: [5:7587241265419331769:0] socket: 421 qp: -1
2025-12-24T02:11:07.291012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241264834256431:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:07.291026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.291130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241266817594446:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241266817594523:2115] held: no
2025-12-24T02:11:07.291277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594541:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.293077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331773:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.293401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331774:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
2025-12-24T02:11:07.293417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265419331765:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.293518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241264834256336:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241264834256396:2107] held: no
2025-12-24T02:11:07.293601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241264834256337:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17248) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241264834256397:2108] held: no
2025-12-24T02:11:07.293718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256429:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.294254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241264834256430:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33348
2025-12-24T02:11:07.294377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241264834256396:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.292812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267456033102:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28119) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267456033168:2110] held: no
2025-12-24T02:11:07.292865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033182:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.292967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033183:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.294470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265419331681:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.294488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.294495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265419331681:2050] [node 3] ICP22 created new session: [5:7587241265419331775:2050]
2025-12-24T02:11:07.294501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265419331775:2050] [node 3] ICS09 handshake done sender: [5:7587241265419331765:2120] self: [5:7587241265419331769:0] peer: [3:7587241264834256412:0] socket: 425 qp: -1
2025-12-24T02:11:07.294508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265419331775:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.294514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.294600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241265419331681:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.296429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241264834256336:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.296442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.296448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241264834256336:2048] [node 1] ICP22 created new session: [3:7587241264834256434:2048]
2025-12-24T02:11:07.296454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241264834256434:2048] [node 1] ICS09 handshake done sender: [3:7587241264834256396:2107] self: [3:7587241264834256420:0] peer: [1:7587241266859714939:0] socket: 381 qp: -1
2025-12-24T02:11:07.296459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241264834256434:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.296464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.296565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241264834256397:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.296832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241264834256337:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.296843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.296846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241264834256337:2049] [node 2] ICP22 created new session: [3:7587241264834256436:2049]
2025-12-24T02:11:07.296851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241264834256436:2049] [node 2] ICS09 handshake done sender: [3:7587241264834256397:2108] self: [3:7587241264834256425:0] peer: [2:7587241266817594513:0] socket: 392 qp: -1
2025-12-24T02:11:07.296855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241264834256436:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.296858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.297117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241264834256416:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.297122Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241264834256416:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.297672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241264834256416:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.295537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.295600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033185:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.295653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267456033168:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.295712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267456033102:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.295724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.295730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267456033102:2048] [node 1] ICP22 created new session: [6:7587241267456033187:2048]
2025-12-24T02:11:07.295734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267456033187:2048] [node 1] ICS09 handshake done sender: [6:7587241267456033168:2110] self: [6:7587241267456033181:0] peer: [1:7587241266859714949:0] socket: 404 qp: -1
2025-12-24T02:11:07.295742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267456033187:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.295749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.295879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.297016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267456033186:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.297043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267456033104:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.297555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033190:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.297795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267456033186:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.297830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267456033104:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.297838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.297842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267456033104:2050] [node 3] ICP22 created new session: [6:7587241267456033191:2050]
2025-12-24T02:11:07.297846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267456033191:2050] [node 3] ICS09 handshake done sender: [6:7587241267456033186:2120] self: [6:7587241267456033189:0] peer: [3:7587241264834256415:0] socket: 409 qp: -1
2025-12-24T02:11:07.297850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267456033191:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.297855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.301342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266817594523:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.301452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241266817594446:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.301469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.301476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266817594446:2048] [node 1] ICP22 created new session: [2:7587241266817594542:2048]
2025-12-24T02:11:07.301482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266817594542:2048] [node 1] ICS09 handshake done sender: [2:7587241266817594523:2115] self: [2:7587241266817594529:0] peer: [1:7587241266859714951:0] socket: 405 qp: -1
2025-12-24T02:11:07.301490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266817594542:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.301499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.300955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891213:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.302353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891212:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.302629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267456033184:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.302668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267456033182:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.302718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267456033171:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.302731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267456033103:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.302937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891214:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.302982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891215:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.302995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891216:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.303178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891205:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.303182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241265025891205:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.303467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241265025891199:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.304683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891216:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.304817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241265025891205:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.304883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241265025891201:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.304899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.304905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241265025891201:2113] [node 6] ICP22 created new session: [4:7587241265025891219:2113]
2025-12-24T02:11:07.304909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241265025891219:2113] [node 6] ICS09 handshake done sender: [4:7587241265025891205:2115] self: [4:7587241265025891204:0] peer: [6:7587241267456033194:0] socket: 422 qp: -1
2025-12-24T02:11:07.304916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241265025891219:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:07.304922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.305193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241265025891129:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.305205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.305210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241265025891129:2050] [node 3] ICP22 created new session: [4:7587241265025891221:2050]
2025-12-24T02:11:07.305215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241265025891221:2050] [node 3] ICS09 handshake done sender: [4:7587241265025891199:2112] self: [4:7587241265025891206:0] peer: [3:7587241264834256403:0] socket: 398 qp: -1
2025-12-24T02:11:07.305219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241265025891221:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.305224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.305559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267456033171:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11699) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267456033184:2118] held: no
2025-12-24T02:11:07.307823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241264834256414:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.307861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.307873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241264834256414:2114] [node 6] ICP22 created new session: [3:7587241264834256439:2114]
2025-12-24T02:11:07.307882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241264834256439:2114] [node 6] ICS09 handshake done sender: [3:7587241264834256416:2116] self: [3:7587241264834256415:0] peer: [6:7587241267456033189:0] socket: 413 qp: -1
2025-12-24T02:11:07.307890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241264834256439:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:07.307904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.308671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891214:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.308714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241265025891128:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.308976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033197:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.309526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267456033185:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.309543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267456033184:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.309292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331774:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.309375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265419331679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.310755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594518:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.310777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266817594518:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.317114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241265025891215:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.317167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241265025891127:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.315719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714958:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.315740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241266859714958:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.318834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241266859714958:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.319051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241266859714956:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.319060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241266859714956:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.318971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267456033171:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.319008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.319017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267456033171:2111] [node 4] ICP22 created new session: [6:7587241267456033198:2111]
2025-12-24T02:11:07.319025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267456033198:2111] [node 4] ICS09 handshake done sender: [6:7587241267456033184:2118] self: [6:7587241267456033194:0] peer: [4:7587241265025891204:0] socket: 433 qp: -1
2025-12-24T02:11:07.319034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267456033198:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:07.319046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.319211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267456033161:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.322715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265419331759:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.322732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241265419331759:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.323803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265419331778:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40241
2025-12-24T02:11:07.323117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266817594528:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.323132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266817594528:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.322463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241266859714954:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.322487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.322498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241266859714954:2113] [node 5] ICP22 created new session: [1:7587241266859714979:2113]
2025-12-24T02:11:07.322505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241266859714979:2113] [node 5] ICS09 handshake done sender: [1:7587241266859714958:2117] self: [1:7587241266859714957:0] peer: [5:7587241265419331777:0] socket: 403 qp: -1
2025-12-24T02:11:07.322514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241266859714979:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:07.322529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.323991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266817594518:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.323999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265419331774:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.324118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266817594528:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.323743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033201:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.324196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266817594512:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.324288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267456033182:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.324210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.324216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266817594512:2109] [node 6] ICP22 created new session: [2:7587241266817594546:2109]
2025-12-24T02:11:07.324344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267456033103:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.324358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.324364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267456033103:2049] [node 2] ICP22 created new session: [6:7587241267456033202:2049]
2025-12-24T02:11:07.324371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267456033202:2049] [node 2] ICS09 handshake done sender: [6:7587241267456033182:2116] self: [6:7587241267456033195:0] peer: [2:7587241266817594517:0] socket: 416 qp: -1
2025-12-24T02:11:07.324379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267456033202:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.324388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.324223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266817594546:2109] [node 6] ICS09 handshake done sender: [2:7587241266817594518:2113] self: [2:7587241266817594517:0] peer: [6:7587241267456033195:0] socket: 411 qp: -1
2025-12-24T02:11:07.324229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266817594546:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:07.324237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.324377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266817594526:2116] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.324386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.324390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266817594526:2116] [node 4] ICP22 created new session: [2:7587241266817594548:2116]
2025-12-24T02:11:07.324394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266817594548:2116] [node 4] ICS09 handshake done sender: [2:7587241266817594528:2118] self: [2:7587241266817594527:0] peer: [4:7587241265025891223:0] socket: 399 qp: -1
2025-12-24T02:11:07.324397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266817594548:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:07.324402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.325276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265419331679:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.325293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.325299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265419331679:2048] [node 1] ICP22 created new session: [5:7587241265419331779:2048]
2025-12-24T02:11:07.325306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265419331779:2048] [node 1] ICS09 handshake done sender: [5:7587241265419331774:2122] self: [5:7587241265419331777:0] peer: [1:7587241266859714957:0] socket: 435 qp: -1
2025-12-24T02:11:07.325313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265419331779:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.325321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.326849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266817594550:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24643
2025-12-24T02:11:07.327378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265419331759:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.327480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241265419331749:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.327499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.327506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265419331749:2113] [node 6] ICP22 created new session: [5:7587241265419331782:2113]
2025-12-24T02:11:07.327512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265419331782:2113] [node 6] ICS09 handshake done sender: [5:7587241265419331759:2117] self: [5:7587241265419331758:0] peer: [6:7587241267456033200:0] socket: 395 qp: -1
2025-12-24T02:11:07.327517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265419331782:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:07.327524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.327442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267456033204:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14949
2025-12-24T02:11:07.327617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267456033185:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.327854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267456033161:2103] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.327863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.327868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267456033161:2103] [node 5] ICP22 created new session: [6:7587241267456033205:2103]
2025-12-24T02:11:07.327872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267456033205:2103] [node 5] ICS09 handshake done sender: [6:7587241267456033185:2119] self: [6:7587241267456033200:0] peer: [5:7587241265419331758:0] socket: 434 qp: -1
2025-12-24T02:11:07.327877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267456033205:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:07.327882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.331580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241266859714956:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.331610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241266817594512:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.331733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241266859714953:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.331771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.331781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241266859714953:2112] [node 4] ICP22 created new session: [1:7587241266859714982:2112]
2025-12-24T02:11:07.331786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241266859714982:2112] [node 4] ICS09 handshake done sender: [1:7587241266859714956:2116] self: [1:7587241266859714955:0] peer: [4:7587241265025891224:0] socket: 431 qp: -1
2025-12-24T02:11:07.331793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241266859714982:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:07.331802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.332227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241265025891128:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17248) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241265025891214:2118] held: no
2025-12-24T02:11:07.332309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891226:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.332486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241265025891215:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.332544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241265025891127:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.332546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241265025891225:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45869
2025-12-24T02:11:07.332554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.332561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241265025891127:2048] [node 1] ICP22 created new session: [4:7587241265025891227:2048]
2025-12-24T02:11:07.332566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241265025891227:2048] [node 1] ICS09 handshake done sender: [4:7587241265025891215:2119] self: [4:7587241265025891224:0] peer: [1:7587241266859714955:0] socket: 439 qp: -1
2025-12-24T02:11:07.332571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241265025891227:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.332579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.332919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241265025891214:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.334509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241265025891128:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.334529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.334537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241265025891128:2049] [node 2] ICP22 created new session: [4:7587241265025891229:2049]
2025-12-24T02:11:07.334543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241265025891229:2049] [node 2] ICS09 handshake done sender: [4:7587241265025891214:2118] self: [4:7587241265025891223:0] peer: [2:7587241266817594527:0] socket: 438 qp: -1
2025-12-24T02:11:07.334551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241265025891229:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.334559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.346799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:07.346926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241266817594542:2048] [node 1] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:07.346937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241266817594446:2048] [node 1] ICP30 unregister session Session# [2:7587241266817594542:2048] VirtualId# [2:7587241266817594529:0]
2025-12-24T02:11:07.346945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241266817594542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.347010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.347050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.346798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:07.346938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241264834256434:2048] [node 1] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:07.346953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241264834256336:2048] [node 1] ICP30 unregister session Session# [3:7587241264834256434:2048] VirtualId# [3:7587241264834256420:0]
2025-12-24T02:11:07.346962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241264834256434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.347026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.347067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.353318Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:07.353419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267456033187:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-12-24T02:11:07.353427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267456033102:2048] [node 1] ICP30 unregister session Session# [6:7587241267456033187:2048] VirtualId# [6:7587241267456033181:0]
2025-12-24T02:11:07.353435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267456033187:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.353497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.353523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.354779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:07.354922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241265025891227:2048] [node 1] ICS01 socket: 439 reason# EndOfStream
2025-12-24T02:11:07.354934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241265025891127:2048] [node 1] ICP30 unregister session Session# [4:7587241265025891227:2048] VirtualId# [4:7587241265025891224:0]
2025-12-24T02:11:07.354794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:07.354922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265419331779:2048] [node 1] ICS01 socket: 435 reason# EndOfStream
2025-12-24T02:11:07.354932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265419331679:2048] [node 1] ICP30 unregister session Session# [5:7587241265419331779:2048] VirtualId# [5:7587241265419331777:0]
2025-12-24T02:11:07.354941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241265025891227:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.355011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.355042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.354943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265419331779:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.355012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.355046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.370859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:07.370998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241264834256436:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:07.371010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241264834256337:2049] [node 2] ICP30 unregister session Session# [3:7587241264834256436:2049] VirtualId# [3:7587241264834256425:0]
2025-12-24T02:11:07.371023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241264834256436:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.371097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.371130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.374610Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:07.374727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265419331766:2049] [node 2] ICS01 socket: 391 reason# EndOfStream
2025-12-24T02:11:07.374739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265419331680:2049] [node 2] ICP30 unregister session Session# [5:7587241265419331766:2049] VirtualId# [5:7587241265419331761:0]
2025-12-24T02:11:07.374748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265419331766:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.374799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.374830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.376803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:07.376903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241265025891229:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:07.376912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241265025891128:2049] [node 2] ICP30 unregister session Session# [4:7587241265025891229:2049] VirtualId# [4:7587241265025891223:0]
2025-12-24T02:11:07.376920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241265025891229:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.376989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.377026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.374857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:07.374954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267456033202:2049] [node 2] ICS01 socket: 416 reason# EndOfStream
2025-12-24T02:11:07.374963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267456033103:2049] [node 2] ICP30 unregister session Session# [6:7587241267456033202:2049] VirtualId# [6:7587241267456033195:0]
2025-12-24T02:11:07.374970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267456033202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.375034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.375061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.398787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:07.398907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265419331775:2050] [node 3] ICS01 socket: 425 reason# EndOfStream
2025-12-24T02:11:07.398919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265419331681:2050] [node 3] ICP30 unregister session Session# [5:7587241265419331775:2050] VirtualId# [5:7587241265419331769:0]
2025-12-24T02:11:07.398929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265419331775:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.398999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.399031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.399007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:07.399081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267456033191:2050] [node 3] ICS01 socket: 409 reason# EndOfStream
2025-12-24T02:11:07.399087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267456033104:2050] [node 3] ICP30 unregister session Session# [6:7587241267456033191:2050] VirtualId# [6:7587241267456033189:0]
2025-12-24T02:11:07.399092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267456033191:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.399135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.399159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.416448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:07.416555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267456033198:2111] [node 4] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:07.416566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267456033171:2111] [node 4] ICP30 unregister session Session# [6:7587241267456033198:2111] VirtualId# [6:7587241267456033194:0]
2025-12-24T02:11:07.416574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267456033198:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.416643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:07.416676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:07.450403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266945530667:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.450416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266945530668:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.458806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268034561080:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:07.458830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268034561081:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.456040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267326645696:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.456048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267326645697:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.455214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263786665716:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.455224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263786665717:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.455229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263786665718:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.456695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267481135347:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.456704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267481135348:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.456708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267481135349:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.462951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265786902448:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.462960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265786902449:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.462964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265786902450:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.486638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267326645754:2103] [node 4] ICP01 ready to work
2025-12-24T02:11:07.487212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267326645760:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:07.488449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266945530728:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:07.488549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266945530729:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:07.488640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241266945530730:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:07.491205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268034561145:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:07.492171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268034561146:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:07.492251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268034561147:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:07.499900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267326645770:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.502076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268034561154:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-12-24T02:11:07.502099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268034561155:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-12-24T02:11:07.502864Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241268034561080:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16035486ns) {Configure:104(16034739ns) {ProcessConfigured:135(13463133ns) {StartInitialHandshake:157(13461851ns) {DropHandshakes:470(157ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(21ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(812ns) {SwitchToState:179(531ns)}}}}}
2025-12-24T02:11:07.504428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263786665786:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:07.504533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241263786665787:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:07.502927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530739:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-12-24T02:11:07.502948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530740:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-12-24T02:11:07.502960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265786902511:2081] [node 6] ICP01 ready to work
2025-12-24T02:11:07.503061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241265786902512:2091] [node 4] ICP01 ready to work
2025-12-24T02:11:07.506046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241265786902449:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18946915ns) {Configure:104(18946075ns) {ProcessConfigured:135(14965011ns) {StartInitialHandshake:157(14964371ns) {DropHandshakes:470(556ns) {DropIncomingHandshake:442(126ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(906ns) {SwitchToState:179(387ns)}}}}}
2025-12-24T02:11:07.505928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241267326645760:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18661198ns) {Configure:104(18660441ns) {ProcessConfigured:135(16325794ns) {StartInitialHandshake:157(16324846ns) {DropHandshakes:470(163ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(580ns) {SwitchToState:179(310ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.506940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665784:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.507015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665785:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.510356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665784:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.514809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530741:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.518618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645776:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
2025-12-24T02:11:07.518644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645777:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
2025-12-24T02:11:07.522839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241266945530728:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33107578ns) {Configure:104(28047730ns) {ProcessConfigured:135(27737721ns) {StartInitialHandshake:157(27737208ns) {DropHandshakes:470(712ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(143ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(1818ns) {SwitchToState:179(896ns)}}}}}
2025-12-24T02:11:07.523169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530740:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.528224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645777:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.528347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241267326645696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.529721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267481135391:2075] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.529881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241267481135392:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:07.527483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268034561160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-12-24T02:11:07.527503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268034561161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-12-24T02:11:07.527514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268034561162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:304
2025-12-24T02:11:07.531007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263786665718:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.531053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241263786665718:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20638025ns) {IncomingHandshake:261(20637393ns) {DropIncomingHandshake:442(281ns) PrepareNewSessionHandshake:477(649ns) {SwitchToState:179(224ns)} GenerateSessionVirtualId:988(287ns) IssueIncomingHandshakeReply:193(20631288ns) {SwitchToState:179(924ns)}}}
2025-12-24T02:11:07.531174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561154:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.529842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645775:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
2025-12-24T02:11:07.534763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
2025-12-24T02:11:07.535109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241265786902512:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31503936ns) {Configure:104(31503457ns) {ProcessConfigured:135(31252266ns) {StartInitialHandshake:157(31251711ns) {DropHandshakes:470(187ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(1521ns) {SwitchToState:179(641ns)}}}}}
2025-12-24T02:11:07.535231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241263786665787:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29941457ns) {Configure:104(29940028ns) {ProcessConfigured:135(29705585ns) {StartInitialHandshake:157(29705389ns) {DropHandshakes:470(232ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(746ns) {SwitchToState:179(250ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.538884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241265786902450:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51303475ns) {Configure:104(51292106ns) {ProcessConfigured:135(51007808ns) {StartInitialHandshake:157(51007281ns) {DropHandshakes:470(415ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1272ns) {SwitchToState:179(602ns)}}}}}
2025-12-24T02:11:07.539127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530749:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-12-24T02:11:07.547044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645775:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.551224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645759:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.551243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267326645759:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.551563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561149:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.551579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268034561149:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.551744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561161:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.551840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665785:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.551873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263786665716:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.554728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135413:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.555060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645776:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.555023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241268034561145:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(62714648ns) {Configure:104(62713938ns) {ProcessConfigured:135(62356344ns) {StartInitialHandshake:157(62355739ns) {DropHandshakes:470(713ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(102ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(439ns)}}}}}
2025-12-24T02:11:07.555353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561155:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.555865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135422:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.555940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135423:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.556564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241266945530729:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(65715276ns) {Configure:104(65714582ns) {ProcessConfigured:135(65420805ns) {StartInitialHandshake:157(65420547ns) {DropHandshakes:470(493ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:477(1129ns) {SwitchToState:179(75ns)}}}}}
2025-12-24T02:11:07.561129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267481135413:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.561179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267481135422:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.561215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267481135347:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.561227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267481135391:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.562919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267326645759:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.562802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561153:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.562824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268034561153:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.563184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.563053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267326645754:2103] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.563076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.563084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267326645754:2103] [node 4] ICP22 created new session: [3:7587241267326645783:2103]
2025-12-24T02:11:07.563090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267326645783:2103] [node 4] ICS09 handshake done sender: [3:7587241267326645759:2108] self: [3:7587241267326645758:0] peer: [4:7587241263786665794:0] socket: 378 qp: -1
2025-12-24T02:11:07.563097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267326645783:2103] [node 4] ICS10 traffic start
2025-12-24T02:11:07.563107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.564006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263786665784:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.564912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263786665718:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.564957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.564964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263786665718:2050] [node 3] ICP22 created new session: [4:7587241263786665799:2050]
2025-12-24T02:11:07.564970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263786665799:2050] [node 3] ICS09 handshake done sender: [4:7587241263786665784:2111] self: [4:7587241263786665794:0] peer: [3:7587241267326645758:0] socket: 381 qp: -1
2025-12-24T02:11:07.564977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263786665799:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.564986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.567735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561151:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.567749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268034561151:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.567077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902526:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.567794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902514:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.567800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241265786902514:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.571158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530749:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.571244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530739:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.571429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268034561153:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.571505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268034561149:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.571585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645778:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.571622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241267326645697:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.571934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.571795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530741:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.571885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530736:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.571888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266945530736:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.572589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561162:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.572620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268034561081:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.572642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.572649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268034561081:2049] [node 3] ICP22 created new session: [1:7587241268034561169:2049]
2025-12-24T02:11:07.572657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268034561169:2049] [node 3] ICS09 handshake done sender: [1:7587241268034561149:2110] self: [1:7587241268034561142:0] peer: [3:7587241267326645779:0] socket: 393 qp: -1
2025-12-24T02:11:07.572664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268034561169:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.572678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.573091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268034561147:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.573107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.573112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268034561147:2109] [node 4] ICP22 created new session: [1:7587241268034561171:2109]
2025-12-24T02:11:07.573117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268034561171:2109] [node 4] ICS09 handshake done sender: [1:7587241268034561153:2112] self: [1:7587241268034561152:0] peer: [4:7587241263786665797:0] socket: 395 qp: -1
2025-12-24T02:11:07.573122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268034561171:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:07.573149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.573513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266945530736:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.573618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266945530668:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.573636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.573642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266945530668:2049] [node 3] ICP22 created new session: [2:7587241266945530755:2049]
2025-12-24T02:11:07.573648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266945530755:2049] [node 3] ICS09 handshake done sender: [2:7587241266945530736:2110] self: [2:7587241266945530731:0] peer: [3:7587241267326645786:0] socket: 410 qp: -1
2025-12-24T02:11:07.573654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266945530755:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.573661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.574862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902527:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.571829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665802:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.572169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263786665785:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.580818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267481135430:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.580867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267481135349:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.581265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241267326645696:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241267326645777:2117] held: no
2025-12-24T02:11:07.581319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241267326645697:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241267326645778:2118] held: no
2025-12-24T02:11:07.581384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645773:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.581389Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267326645773:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.575916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-12-24T02:11:07.581988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268034561151:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.581748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267481135423:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.581779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267481135348:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.582118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268034561146:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.582137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.582145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268034561146:2108] [node 6] ICP22 created new session: [1:7587241268034561177:2108]
2025-12-24T02:11:07.582151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268034561177:2108] [node 6] ICS09 handshake done sender: [1:7587241268034561151:2111] self: [1:7587241268034561150:0] peer: [6:7587241267481135428:0] socket: 394 qp: -1
2025-12-24T02:11:07.582157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268034561177:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:07.582166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.582172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265786902514:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.582246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241265786902511:2081] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.582256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.582263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265786902511:2081] [node 6] ICP22 created new session: [5:7587241265786902529:2081]
2025-12-24T02:11:07.582269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265786902529:2081] [node 6] ICS09 handshake done sender: [5:7587241265786902514:2107] self: [5:7587241265786902513:0] peer: [6:7587241267481135429:0] socket: 404 qp: -1
2025-12-24T02:11:07.582275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265786902529:2081] [node 6] ICS10 traffic start
2025-12-24T02:11:07.582283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.582320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645787:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.583302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267326645788:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58742
2025-12-24T02:11:07.583553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267326645777:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.583625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267326645778:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.583699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241267326645696:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.583713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.583717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267326645696:2048] [node 1] ICP22 created new session: [3:7587241267326645791:2048]
2025-12-24T02:11:07.583723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267326645791:2048] [node 1] ICS09 handshake done sender: [3:7587241267326645777:2117] self: [3:7587241267326645779:0] peer: [1:7587241268034561142:0] socket: 409 qp: -1
2025-12-24T02:11:07.583729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267326645791:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.583736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.583833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241267326645697:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.583841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.583847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267326645697:2049] [node 2] ICP22 created new session: [3:7587241267326645793:2049]
2025-12-24T02:11:07.583853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267326645793:2049] [node 2] ICS09 handshake done sender: [3:7587241267326645778:2118] self: [3:7587241267326645786:0] peer: [2:7587241266945530731:0] socket: 413 qp: -1
2025-12-24T02:11:07.583857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267326645793:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.583862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.584194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267326645773:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.584283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267326645770:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.584298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.584303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267326645770:2112] [node 6] ICP22 created new session: [3:7587241267326645795:2112]
2025-12-24T02:11:07.584307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267326645795:2112] [node 6] ICS09 handshake done sender: [3:7587241267326645773:2113] self: [3:7587241267326645772:0] peer: [6:7587241267481135431:0] socket: 422 qp: -1
2025-12-24T02:11:07.584311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267326645795:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:07.584318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.582463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263786665716:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.582483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.586260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267481135349:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30437) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267481135430:2118] held: no
2025-12-24T02:11:07.582491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263786665716:2048] [node 1] ICP22 created new session: [4:7587241263786665803:2048]
2025-12-24T02:11:07.582496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263786665803:2048] [node 1] ICS09 handshake done sender: [4:7587241263786665785:2112] self: [4:7587241263786665797:0] peer: [1:7587241268034561152:0] socket: 401 qp: -1
2025-12-24T02:11:07.582501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263786665803:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.582508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.584970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241263786665716:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.585025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.589791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267481135347:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267481135413:2105] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.591581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902527:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.591731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265786902448:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.591805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902531:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.591849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902532:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.592038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135435:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.592332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665806:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.592453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665807:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.592492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665808:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.592505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241263786665717:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.592598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530757:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.592646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530747:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.592649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266945530747:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.592740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530743:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.592754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266945530743:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.592813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135438:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.593595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241266945530667:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.593740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267326645774:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.593749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267326645774:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.594386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267326645774:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.593012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135436:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.592995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665807:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.593112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902531:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.593151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902532:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.593248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265786902450:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.594805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267326645760:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.594822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.594828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267326645760:2106] [node 5] ICP22 created new session: [3:7587241267326645798:2106]
2025-12-24T02:11:07.594839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266945530747:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.594833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267326645798:2106] [node 5] ICS09 handshake done sender: [3:7587241267326645774:2114] self: [3:7587241267326645764:0] peer: [5:7587241265786902535:0] socket: 386 qp: -1
2025-12-24T02:11:07.594839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267326645798:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:07.594849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.595119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902526:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.595467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561160:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.595519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561159:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.595522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268034561159:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.595163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266945530728:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.595280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267481135422:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.595180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.595186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266945530728:2105] [node 4] ICP22 created new session: [2:7587241266945530763:2105]
2025-12-24T02:11:07.595536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267481135391:2075] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.595191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266945530763:2105] [node 4] ICS09 handshake done sender: [2:7587241266945530747:2116] self: [2:7587241266945530733:0] peer: [4:7587241263786665809:0] socket: 430 qp: -1
2025-12-24T02:11:07.595546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.595551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267481135391:2075] [node 5] ICP22 created new session: [6:7587241267481135440:2075]
2025-12-24T02:11:07.595196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266945530763:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:07.595202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.595556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267481135440:2075] [node 5] ICS09 handshake done sender: [6:7587241267481135422:2114] self: [6:7587241267481135429:0] peer: [5:7587241265786902513:0] socket: 418 qp: -1
2025-12-24T02:11:07.595561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267481135440:2075] [node 5] ICS10 traffic start
2025-12-24T02:11:07.595581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.599351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241268034561146:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(17195240ns) {HandleHandshakeStatus:329(17194541ns) {DropIncomingHandshake:442(3620ns) DropOutgoingHandshake:457(139ns) SwitchToState:179(264ns) ProcessPendingSessionEvents:495(6480131ns) {ForwardSessionEventToSession:612(6467246ns) ForwardSessionEventToSession:612(2188ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(591ns) ForwardSessionEventToSession:612(1172ns) ForwardSessionEventToSession:612(562ns) ForwardSessionEventToSession:612(492ns) ForwardSessionEventToSession:612(541ns) ForwardSessionEventToSession:612(850ns)}}}
2025-12-24T02:11:07.599570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268034561165:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.599581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268034561165:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.599687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.600063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265786902512:2091] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.600078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265786902531:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.598893Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267481135348:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267481135423:2115] held: no
2025-12-24T02:11:07.598946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267481135413:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.599555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135437:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.600210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241263786665717:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241263786665806:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.600255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265786902450:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.600275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.600281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265786902450:2050] [node 3] ICP22 created new session: [5:7587241265786902539:2050]
2025-12-24T02:11:07.600286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265786902539:2050] [node 3] ICS09 handshake done sender: [5:7587241265786902531:2117] self: [5:7587241265786902535:0] peer: [3:7587241267326645764:0] socket: 392 qp: -1
2025-12-24T02:11:07.600293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265786902539:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.600301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.600646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902537:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.600863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665808:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.600969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241265786902537:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.600995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241265786902449:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.595742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266945530743:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.595828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266945530730:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.595840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.595844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266945530730:2107] [node 6] ICP22 created new session: [2:7587241266945530765:2107]
2025-12-24T02:11:07.595848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266945530765:2107] [node 6] ICS09 handshake done sender: [2:7587241266945530743:2115] self: [2:7587241266945530742:0] peer: [6:7587241267481135432:0] socket: 412 qp: -1
2025-12-24T02:11:07.595852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266945530765:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:07.595864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.601088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241266945530750:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.601094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241266945530750:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.601842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266945530750:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.601946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241266945530729:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.601965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.601971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266945530729:2106] [node 5] ICP22 created new session: [2:7587241266945530768:2106]
2025-12-24T02:11:07.601977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266945530768:2106] [node 5] ICS09 handshake done sender: [2:7587241266945530750:2118] self: [2:7587241266945530732:0] peer: [5:7587241265786902541:0] socket: 447 qp: -1
2025-12-24T02:11:07.601983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266945530768:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:07.601991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.603562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241266945530770:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26969
2025-12-24T02:11:07.603700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241267481135438:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.603741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267481135347:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.603750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.603757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267481135347:2048] [node 1] ICP22 created new session: [6:7587241267481135442:2048]
2025-12-24T02:11:07.603763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267481135442:2048] [node 1] ICS09 handshake done sender: [6:7587241267481135413:2105] self: [6:7587241267481135428:0] peer: [1:7587241268034561150:0] socket: 400 qp: -1
2025-12-24T02:11:07.603769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267481135442:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.603776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.602910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241263786665812:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38161
2025-12-24T02:11:07.603125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263786665806:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.603796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665791:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.603804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241263786665791:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.611099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268034561159:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.611323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268034561080:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.611358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.611366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268034561080:2048] [node 2] ICP22 created new session: [1:7587241268034561181:2048]
2025-12-24T02:11:07.611373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268034561181:2048] [node 2] ICS09 handshake done sender: [1:7587241268034561159:2115] self: [1:7587241268034561144:0] peer: [2:7587241266945530760:0] socket: 415 qp: -1
2025-12-24T02:11:07.611381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268034561181:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:07.611396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.610910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241265786902511:2081] [node 6] ICP35 event processing took too much time PendingConnection:279(28620877ns) {HandleHandshakeStatus:329(28619866ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(199ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(17999987ns) {ForwardSessionEventToSession:612(17984036ns) ForwardSessionEventToSession:612(2753ns) ForwardSessionEventToSession:612(1216ns) ForwardSessionEventToSession:612(878ns) ForwardSessionEventToSession:612(466ns) ForwardSessionEventToSession:612(1633ns) ForwardSessionEventToSession:612(576ns) ForwardSessionEventToSession:612(564ns) ForwardSessionEventToSession:612(815ns) ForwardSessionEventToSession:612(897ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.611891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241267481135391:2075] [node 5] ICP35 event processing took too much time PendingConnection:279(16322819ns) {HandleHandshakeStatus:329(16322348ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(1306ns) SwitchToState:179(169ns) ProcessPendingSessionEvents:495(12238007ns) {ForwardSessionEventToSession:612(12224452ns) ForwardSessionEventToSession:612(2619ns) ForwardSessionEventToSession:612(947ns) ForwardSessionEventToSession:612(1063ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(851ns) ForwardSessionEventToSession:612(738ns) ForwardSessionEventToSession:612(805ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(1197ns)}}}
2025-12-24T02:11:07.611943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135439:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
2025-12-24T02:11:07.611979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241267326645760:2106] [node 5] ICP35 event processing took too much time PendingConnection:279(17138070ns) {HandleHandshakeStatus:329(17136864ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(164ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(12267722ns) {ForwardSessionEventToSession:612(12254757ns) ForwardSessionEventToSession:612(2412ns) ForwardSessionEventToSession:612(814ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(1132ns) ForwardSessionEventToSession:612(671ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(1182ns)}}}
2025-12-24T02:11:07.612440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267481135430:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.612537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267481135349:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.612555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.612563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267481135349:2050] [node 3] ICP22 created new session: [6:7587241267481135444:2050]
2025-12-24T02:11:07.612569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267481135444:2050] [node 3] ICS09 handshake done sender: [6:7587241267481135430:2118] self: [6:7587241267481135431:0] peer: [3:7587241267326645772:0] socket: 380 qp: -1
2025-12-24T02:11:07.612576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267481135444:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.612584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.612705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241267481135392:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.612718Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267481135423:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.612917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267481135348:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.612930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.612936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267481135348:2049] [node 2] ICP22 created new session: [6:7587241267481135447:2049]
2025-12-24T02:11:07.612941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267481135447:2049] [node 2] ICS09 handshake done sender: [6:7587241267481135423:2115] self: [6:7587241267481135432:0] peer: [2:7587241266945530742:0] socket: 419 qp: -1
2025-12-24T02:11:07.612947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267481135447:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.612954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.614851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268034561165:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.614831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241266945530757:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.614979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268034561145:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.614998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.615005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268034561145:2107] [node 5] ICP22 created new session: [1:7587241268034561184:2107]
2025-12-24T02:11:07.615011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268034561184:2107] [node 5] ICS09 handshake done sender: [1:7587241268034561165:2119] self: [1:7587241268034561148:0] peer: [5:7587241265786902534:0] socket: 429 qp: -1
2025-12-24T02:11:07.615016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268034561184:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:07.615024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.615164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241263786665717:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.615186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.615194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263786665717:2049] [node 2] ICP22 created new session: [4:7587241263786665813:2049]
2025-12-24T02:11:07.615201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263786665813:2049] [node 2] ICS09 handshake done sender: [4:7587241263786665806:2119] self: [4:7587241263786665809:0] peer: [2:7587241266945530733:0] socket: 408 qp: -1
2025-12-24T02:11:07.615209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263786665813:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.615217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.615374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241263786665796:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.615379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241263786665796:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.616323Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241265786902449:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22889) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241265786902537:2120] held: no
2025-12-24T02:11:07.616498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241265786902511:2081] [node 6] ICP18 handshake (actor [5:7587241265786902532:2118]) is too old
2025-12-24T02:11:07.618994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263786665796:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.619640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241263786665787:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.619662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.619670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263786665787:2110] [node 6] ICP22 created new session: [4:7587241263786665817:2110]
2025-12-24T02:11:07.619676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263786665817:2110] [node 6] ICS09 handshake done sender: [4:7587241263786665796:2116] self: [4:7587241263786665792:0] peer: [6:7587241267481135446:0] socket: 406 qp: -1
2025-12-24T02:11:07.619684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263786665817:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:07.619691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.622900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241263786665791:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.623005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241263786665786:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.623028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.623037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241263786665786:2109] [node 5] ICP22 created new session: [4:7587241263786665819:2109]
2025-12-24T02:11:07.623043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241263786665819:2109] [node 5] ICS09 handshake done sender: [4:7587241263786665791:2114] self: [4:7587241263786665790:0] peer: [5:7587241265786902538:0] socket: 423 qp: -1
2025-12-24T02:11:07.623056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241263786665819:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:07.623066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.624471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241266945530667:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241266945530757:2119] held: no
2025-12-24T02:11:07.624502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241266945530667:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.624513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.624518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241266945530667:2048] [node 1] ICP22 created new session: [2:7587241266945530771:2048]
2025-12-24T02:11:07.624525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241266945530771:2048] [node 1] ICS09 handshake done sender: [2:7587241266945530757:2119] self: [2:7587241266945530760:0] peer: [1:7587241268034561144:0] socket: 397 qp: -1
2025-12-24T02:11:07.624532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241266945530771:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.624541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.626894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241267481135392:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4501) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241267481135438:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.627083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241265786902512:2091] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4501) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241265786902526:2115] held: no
2025-12-24T02:11:07.627157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.627526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241267481135449:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16649
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.628580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265786902537:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.628663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902543:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.628799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241265786902544:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26617
2025-12-24T02:11:07.629725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265786902449:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.629739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.629746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265786902449:2049] [node 2] ICP22 created new session: [5:7587241265786902545:2049]
2025-12-24T02:11:07.629753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265786902545:2049] [node 2] ICS09 handshake done sender: [5:7587241265786902537:2120] self: [5:7587241265786902541:0] peer: [2:7587241266945530732:0] socket: 388 qp: -1
2025-12-24T02:11:07.629760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265786902545:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.629769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.629873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241265786902448:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12289) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241265786902527:2116] held: no
2025-12-24T02:11:07.629936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265786902527:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.630002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241265786902526:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.630062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265786902448:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.630069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.630076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265786902448:2048] [node 1] ICP22 created new session: [5:7587241265786902547:2048]
2025-12-24T02:11:07.630081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265786902547:2048] [node 1] ICS09 handshake done sender: [5:7587241265786902527:2116] self: [5:7587241265786902534:0] peer: [1:7587241268034561148:0] socket: 391 qp: -1
2025-12-24T02:11:07.630086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265786902547:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.630091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.630178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241265786902512:2091] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.630188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.630192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241265786902512:2091] [node 4] ICP22 created new session: [5:7587241265786902549:2091]
2025-12-24T02:11:07.630196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241265786902549:2091] [node 4] ICS09 handshake done sender: [5:7587241265786902526:2115] self: [5:7587241265786902538:0] peer: [4:7587241263786665790:0] socket: 424 qp: -1
2025-12-24T02:11:07.630200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241265786902549:2091] [node 4] ICS10 traffic start
2025-12-24T02:11:07.630205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.630057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241267481135438:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.630340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241267481135392:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.630351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.630358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241267481135392:2084] [node 4] ICP22 created new session: [6:7587241267481135450:2084]
2025-12-24T02:11:07.630364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241267481135450:2084] [node 4] ICS09 handshake done sender: [6:7587241267481135438:2122] self: [6:7587241267481135446:0] peer: [4:7587241263786665792:0] socket: 439 qp: -1
2025-12-24T02:11:07.630372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241267481135450:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:07.630379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.640737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:07.640837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241267326645791:2048] [node 1] ICS01 socket: 409 reason# EndOfStream
2025-12-24T02:11:07.640846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241267326645696:2048] [node 1] ICP30 unregister session Session# [3:7587241267326645791:2048] VirtualId# [3:7587241267326645779:0]
2025-12-24T02:11:07.640854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241267326645791:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.640923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.640949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.646161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:07.646221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267481135442:2048] [node 1] ICS01 socket: 400 reason# EndOfStream
2025-12-24T02:11:07.646226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267481135347:2048] [node 1] ICP30 unregister session Session# [6:7587241267481135442:2048] VirtualId# [6:7587241267481135428:0]
2025-12-24T02:11:07.646231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267481135442:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.646265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.646285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.645885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:07.645973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265786902547:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-12-24T02:11:07.645981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265786902448:2048] [node 1] ICP30 unregister session Session# [5:7587241265786902547:2048] VirtualId# [5:7587241265786902534:0]
2025-12-24T02:11:07.645988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265786902547:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.645999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.646045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.649345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:07.649435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241263786665803:2048] [node 1] ICS01 socket: 401 reason# EndOfStream
2025-12-24T02:11:07.649442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241263786665716:2048] [node 1] ICP30 unregister session Session# [4:7587241263786665803:2048] VirtualId# [4:7587241263786665797:0]
2025-12-24T02:11:07.649448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241263786665803:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.649509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.649534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.661562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:07.661660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241263786665813:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-12-24T02:11:07.661670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241263786665717:2049] [node 2] ICP30 unregister session Session# [4:7587241263786665813:2049] VirtualId# [4:7587241263786665809:0]
2025-12-24T02:11:07.661677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241263786665813:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.661728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.661759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.661797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:07.661858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265786902545:2049] [node 2] ICS01 socket: 388 reason# EndOfStream
2025-12-24T02:11:07.661864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265786902449:2049] [node 2] ICP30 unregister session Session# [5:7587241265786902545:2049] VirtualId# [5:7587241265786902541:0]
2025-12-24T02:11:07.661868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265786902545:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.661890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.661911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.663608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:07.663706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267481135447:2049] [node 2] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:07.663714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267481135348:2049] [node 2] ICP30 unregister session Session# [6:7587241267481135447:2049] VirtualId# [6:7587241267481135432:0]
2025-12-24T02:11:07.663721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267481135447:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.663774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.663826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.672005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:07.672107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241263786665799:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:07.672118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241263786665718:2050] [node 3] ICP30 unregister session Session# [4:7587241263786665799:2050] VirtualId# [4:7587241263786665794:0]
2025-12-24T02:11:07.672125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241263786665799:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.672190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.672219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.673814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:07.673892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241265786902539:2050] [node 3] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:07.673900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241265786902450:2050] [node 3] ICP30 unregister session Session# [5:7587241265786902539:2050] VirtualId# [5:7587241265786902535:0]
2025-12-24T02:11:07.673906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241265786902539:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.673954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.673975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.674773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:07.674851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267481135444:2050] [node 3] ICS01 socket: 380 reason# EndOfStream
2025-12-24T02:11:07.674857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267481135349:2050] [node 3] ICP30 unregister session Session# [6:7587241267481135444:2050] VirtualId# [6:7587241267481135431:0]
2025-12-24T02:11:07.674863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267481135444:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.674911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.674934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.687362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:07.687458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241267481135450:2084] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-12-24T02:11:07.687469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241267481135392:2084] [node 4] ICP30 unregister session Session# [6:7587241267481135450:2084] VirtualId# [6:7587241267481135446:0]
2025-12-24T02:11:07.687475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241267481135450:2084] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.687539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:07.687562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:07.724036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241265536353105:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:07.724044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241265536353106:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.726844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241267764605770:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.726857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241267764605771:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:07.727672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267358145977:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.727681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267358145978:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.736465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241263876239083:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.736474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241263876239084:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.736480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241263876239085:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.731860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241267116537821:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.731869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241267116537822:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.731873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241267116537823:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.729980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241267693021020:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:07.729997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241267693021021:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:07.730002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241267693021022:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:07.746995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241267116537877:2097] [node 6] ICP01 ready to work
2025-12-24T02:11:07.747074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241267116537880:2100] [node 4] ICP01 ready to work
2025-12-24T02:11:07.747347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241263876239136:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:07.747770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241265536353170:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:07.748162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241265536353179:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:07.747065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267358146038:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.748843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146041:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.748755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021081:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-12-24T02:11:07.749277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239140:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.749706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021085:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.750630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241263876239144:2103] [node 4] ICP01 ready to work
2025-12-24T02:11:07.747386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241267764605831:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:07.749100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241267764605841:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.751370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241267693021092:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:07.751373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021086:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.751546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241267693021095:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:07.752771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267358146047:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:07.750544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537886:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.750567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537887:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.750884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241267764605847:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.757901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239153:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.758926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021093:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-12-24T02:11:07.759561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241265536353174:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:07.759390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241267764605848:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36892
2025-12-24T02:11:07.762749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241267764605843:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:07.762611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241267358146046:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.763745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241265536353183:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55669
2025-12-24T02:11:07.763763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241265536353184:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55669
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.766835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241267358145977:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19666548ns) {Configure:104(19642120ns) {ProcessConfigured:135(17753574ns) {StartInitialHandshake:157(17753011ns) {DropHandshakes:470(314ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:477(1463ns) {SwitchToState:179(966ns)}}}}}
2025-12-24T02:11:07.767041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146056:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.767065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146057:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.767087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146058:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.767240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241267764605851:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36892
2025-12-24T02:11:07.767296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241267764605852:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36892
2025-12-24T02:11:07.783366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241267358146046:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20655425ns) {Configure:104(20654704ns) {ProcessConfigured:135(20351315ns) {StartInitialHandshake:157(20349977ns) {DropHandshakes:470(886ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(2127ns) {SwitchToState:179(1259ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.788598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146056:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.788648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241267358145978:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.789654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146061:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.791338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021085:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.791377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241267693021021:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.788782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605839:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.788796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241267764605839:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.789771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241267764605839:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.789877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241267764605771:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.789891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.789898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241267764605771:2049] [node 3] ICP22 created new session: [2:7587241267764605854:2049]
2025-12-24T02:11:07.789905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241267764605854:2049] [node 3] ICS09 handshake done sender: [2:7587241267764605839:2111] self: [2:7587241267764605837:0] peer: [3:7587241267358146060:0] socket: 397 qp: -1
2025-12-24T02:11:07.789913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241267764605854:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.789936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.794822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605845:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.791659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267358146056:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.791789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241267358145978:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.791802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.791810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267358145978:2049] [node 2] ICP22 created new session: [3:7587241267358146062:2049]
2025-12-24T02:11:07.791816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267358146062:2049] [node 2] ICS09 handshake done sender: [3:7587241267358146056:2115] self: [3:7587241267358146060:0] peer: [2:7587241267764605837:0] socket: 419 qp: -1
2025-12-24T02:11:07.791825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267358146062:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.791842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.794834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241267764605845:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.794853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241263876239140:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.794899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241263876239084:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.795449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605833:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.795457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241267764605833:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.798592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241267764605845:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.798627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241267764605833:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.798747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241267764605831:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.798760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.798767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241267764605831:2107] [node 6] ICP22 created new session: [2:7587241267764605859:2107]
2025-12-24T02:11:07.798774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241267764605859:2107] [node 6] ICS09 handshake done sender: [2:7587241267764605833:2108] self: [2:7587241267764605832:0] peer: [6:7587241263876239163:0] socket: 387 qp: -1
2025-12-24T02:11:07.798780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241267764605859:2107] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.800745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239164:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.798801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.804061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241263876239140:2101] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.804222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241263876239084:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.804236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.804247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241263876239084:2049] [node 2] ICP22 created new session: [6:7587241263876239165:2049]
2025-12-24T02:11:07.804253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241263876239165:2049] [node 2] ICS09 handshake done sender: [6:7587241263876239140:2101] self: [6:7587241263876239163:0] peer: [2:7587241267764605832:0] socket: 389 qp: -1
2025-12-24T02:11:07.804261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241263876239165:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.804280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.812785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241265536353186:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55669
2025-12-24T02:11:07.813028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241265536353187:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55669
2025-12-24T02:11:07.811096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537901:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.812633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537886:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.815563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146065:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.815029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241267764605841:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.815057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.815066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241267764605841:2109] [node 4] ICP22 created new session: [2:7587241267764605862:2109]
2025-12-24T02:11:07.815073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241267764605862:2109] [node 4] ICS09 handshake done sender: [2:7587241267764605845:2113] self: [2:7587241267764605844:0] peer: [4:7587241267693021102:0] socket: 398 qp: -1
2025-12-24T02:11:07.815081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241267764605862:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:07.815091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.815108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605852:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.815724Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241267764605841:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(16980126ns) {HandleHandshakeStatus:329(16979312ns) {DropIncomingHandshake:442(594ns) DropOutgoingHandshake:457(432ns) SwitchToState:179(561ns) ProcessPendingSessionEvents:495(556830ns) {ForwardSessionEventToSession:612(545860ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(1103ns) ForwardSessionEventToSession:612(919ns) ForwardSessionEventToSession:612(609ns) ForwardSessionEventToSession:612(696ns) ForwardSessionEventToSession:612(484ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(448ns) ForwardSessionEventToSession:612(966ns)}}}
2025-12-24T02:11:07.815880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241267764605831:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(17110710ns) {HandleHandshakeStatus:329(17109904ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(605ns) SwitchToState:179(191ns) ProcessPendingSessionEvents:495(400071ns) {ForwardSessionEventToSession:612(390596ns) ForwardSessionEventToSession:612(973ns) ForwardSessionEventToSession:612(969ns) ForwardSessionEventToSession:612(513ns) ForwardSessionEventToSession:612(550ns) ForwardSessionEventToSession:612(824ns) ForwardSessionEventToSession:612(693ns) ForwardSessionEventToSession:612(467ns) ForwardSessionEventToSession:612(951ns) ForwardSessionEventToSession:612(789ns)}}}
2025-12-24T02:11:07.814914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021086:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.818786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021103:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-12-24T02:11:07.818844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353186:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.820255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605847:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.820311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241267764605770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.822445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353166:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.822457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241265536353166:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.823050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021093:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.823082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241267693021085:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.823275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241267693021021:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.823293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.823301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241267693021021:2049] [node 2] ICP22 created new session: [4:7587241267693021105:2049]
2025-12-24T02:11:07.823308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241267693021105:2049] [node 2] ICS09 handshake done sender: [4:7587241267693021085:2110] self: [4:7587241267693021102:0] peer: [2:7587241267764605844:0] socket: 399 qp: -1
2025-12-24T02:11:07.823318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241267693021105:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.823339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.823576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241267764605850:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.823586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241267764605850:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.824390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241265536353166:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.824520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241265536353105:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.824539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.824545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241265536353105:2048] [node 2] ICP22 created new session: [1:7587241265536353190:2048]
2025-12-24T02:11:07.824551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241265536353190:2048] [node 2] ICS09 handshake done sender: [1:7587241265536353166:2107] self: [1:7587241265536353165:0] peer: [2:7587241267764605865:0] socket: 384 qp: -1
2025-12-24T02:11:07.824557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241265536353190:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:07.824565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:07.824684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241267764605850:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.823306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537901:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.824794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241267764605843:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.824806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.824813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241267764605843:2112] [node 5] ICP22 created new session: [2:7587241267764605867:2112]
2025-12-24T02:11:07.823356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241267116537822:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.824818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241267764605867:2112] [node 5] ICS09 handshake done sender: [2:7587241267764605850:2116] self: [2:7587241267764605849:0] peer: [5:7587241267116537903:0] socket: 420 qp: -1
2025-12-24T02:11:07.824823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241267764605867:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:07.824843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.827722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241267764605770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30169) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241267764605847:2114] held: no
2025-12-24T02:11:07.829731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241263876239153:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.829853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239168:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.829879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241263876239085:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.830045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353184:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.827117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241267116537822:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241267116537901:2117] held: no
2025-12-24T02:11:07.827163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537904:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.827932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241267116537901:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:07.828046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241267116537822:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.828055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:07.828061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241267116537822:2049] [node 2] ICP22 created new session: [5:7587241267116537905:2049]
2025-12-24T02:11:07.828067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241267116537905:2049] [node 2] ICS09 handshake done sender: [5:7587241267116537901:2117] self: [5:7587241267116537903:0] peer: [2:7587241267764605849:0] socket: 434 qp: -1
2025-12-24T02:11:07.828073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241267116537905:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:07.828090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.830951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241267764605869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36892
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.831177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239170:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.831534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241263876239153:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.831642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241263876239085:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.831659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.831673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241263876239085:2050] [node 3] ICP22 created new session: [6:7587241263876239171:2050]
2025-12-24T02:11:07.831685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241263876239171:2050] [node 3] ICS09 handshake done sender: [6:7587241263876239153:2107] self: [6:7587241263876239169:0] peer: [3:7587241267358146048:0] socket: 418 qp: -1
2025-12-24T02:11:07.831697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241263876239171:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.827180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146057:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.830190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146049:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.831709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.830196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267358146049:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.830407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146041:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.830439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241267358145977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.831340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267358146049:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.831999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267358146047:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.832015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.832022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267358146047:2111] [node 6] ICP22 created new session: [3:7587241267358146069:2111]
2025-12-24T02:11:07.832029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267358146069:2111] [node 6] ICS09 handshake done sender: [3:7587241267358146049:2112] self: [3:7587241267358146048:0] peer: [6:7587241263876239169:0] socket: 415 qp: -1
2025-12-24T02:11:07.832034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267358146069:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:07.832040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.832856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.833291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241267764605847:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.833584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241267764605770:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.833601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.833609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241267764605770:2048] [node 1] ICP22 created new session: [2:7587241267764605870:2048]
2025-12-24T02:11:07.833622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241267764605870:2048] [node 1] ICS09 handshake done sender: [2:7587241267764605847:2114] self: [2:7587241267764605865:0] peer: [1:7587241265536353165:0] socket: 405 qp: -1
2025-12-24T02:11:07.833629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241267764605870:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.833670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.834337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241267764605841:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.834226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021108:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.834275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241267693021022:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.834450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146059:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.834460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267358146059:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.835144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267358146059:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.835241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267358146046:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.835255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.835276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267358146046:2110] [node 4] ICP22 created new session: [3:7587241267358146073:2110]
2025-12-24T02:11:07.835282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267358146073:2110] [node 4] ICS09 handshake done sender: [3:7587241267358146059:2118] self: [3:7587241267358146054:0] peer: [4:7587241267693021110:0] socket: 443 qp: -1
2025-12-24T02:11:07.835288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267358146073:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:07.835294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.836095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353187:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.833090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537907:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.834512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537907:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.834548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241267116537880:2100] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.839205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021097:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.839218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241267693021097:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.842731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353171:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.842755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241265536353171:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.844621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239173:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.846399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241265536353171:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.846581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241265536353106:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.846606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.846615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241265536353106:2049] [node 3] ICP22 created new session: [1:7587241265536353196:2049]
2025-12-24T02:11:07.846622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241265536353196:2049] [node 3] ICS09 handshake done sender: [1:7587241265536353171:2109] self: [1:7587241265536353169:0] peer: [3:7587241267358146067:0] socket: 385 qp: -1
2025-12-24T02:11:07.846628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241265536353196:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:07.846635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.847449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241263876239173:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.847504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241263876239136:2099] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.846274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021111:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-12-24T02:11:07.847086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241267358146047:2111] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.847119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241267358146075:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24673
2025-12-24T02:11:07.847778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267358146041:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.847859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021081:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.847906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241267693021097:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.847098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537910:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.847639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537896:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.847644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241267116537896:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.848395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241267116537896:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.848478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241267116537877:2097] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.848493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.848499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241267116537877:2097] [node 6] ICP22 created new session: [5:7587241267116537912:2097]
2025-12-24T02:11:07.848505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241267116537912:2097] [node 6] ICS09 handshake done sender: [5:7587241267116537896:2113] self: [5:7587241267116537895:0] peer: [6:7587241263876239174:0] socket: 417 qp: -1
2025-12-24T02:11:07.848511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241267116537912:2097] [node 6] ICS10 traffic start
2025-12-24T02:11:07.848520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.854878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241267693021092:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.854911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.854922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241267693021092:2113] [node 5] ICP22 created new session: [4:7587241267693021113:2113]
2025-12-24T02:11:07.854930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241267693021113:2113] [node 5] ICS09 handshake done sender: [4:7587241267693021097:2117] self: [4:7587241267693021096:0] peer: [5:7587241267116537908:0] socket: 445 qp: -1
2025-12-24T02:11:07.854939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241267693021113:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:07.854963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.855408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241265536353199:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55669
2025-12-24T02:11:07.855361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241267358145977:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.855381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.855391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267358145977:2048] [node 1] ICP22 created new session: [3:7587241267358146076:2048]
2025-12-24T02:11:07.855398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267358146076:2048] [node 1] ICS09 handshake done sender: [3:7587241267358146041:2109] self: [3:7587241267358146067:0] peer: [1:7587241265536353169:0] socket: 393 qp: -1
2025-12-24T02:11:07.855404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267358146076:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.855422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.856673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241267358145977:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.856798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241267693021020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.856816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241267693021108:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.857013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241267693021022:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.857027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.857033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241267693021022:2050] [node 3] ICP22 created new session: [4:7587241267693021117:2050]
2025-12-24T02:11:07.857038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241267693021117:2050] [node 3] ICS09 handshake done sender: [4:7587241267693021108:2120] self: [4:7587241267693021110:0] peer: [3:7587241267358146054:0] socket: 444 qp: -1
2025-12-24T02:11:07.857044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241267693021117:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.857050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.857459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353176:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.857470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241265536353176:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.858802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241265536353176:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.859689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353183:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.859715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241265536353170:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.859726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.859734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241265536353170:2108] [node 4] ICP22 created new session: [1:7587241265536353201:2108]
2025-12-24T02:11:07.859740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241265536353201:2108] [node 4] ICS09 handshake done sender: [1:7587241265536353176:2111] self: [1:7587241265536353175:0] peer: [4:7587241267693021115:0] socket: 388 qp: -1
2025-12-24T02:11:07.859747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241265536353201:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:07.859772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.859855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241265536353170:2108] [node 4] ICP18 handshake (actor [1:7587241265536353183:2114]) is too old
2025-12-24T02:11:07.863014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537887:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.863068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241267116537823:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.864035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241267693021020:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30169) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241267693021081:2108] held: no
2025-12-24T02:11:07.867742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537915:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.866961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241267358146040:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.870615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241267116537910:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.866975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241267358146040:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.867537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241267358146046:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:07.870686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241263876239168:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.870977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241267693021119:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18754
2025-12-24T02:11:07.871201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241267693021081:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.870815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241267358146040:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.870981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241267358146038:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.871006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.871015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241267358146038:2106] [node 5] ICP22 created new session: [3:7587241267358146079:2106]
2025-12-24T02:11:07.871022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241267358146079:2106] [node 5] ICS09 handshake done sender: [3:7587241267358146040:2108] self: [3:7587241267358146039:0] peer: [5:7587241267116537916:0] socket: 394 qp: -1
2025-12-24T02:11:07.871030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241267358146079:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:07.871040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.871326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241267693021020:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.871341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.871348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241267693021020:2048] [node 1] ICP22 created new session: [4:7587241267693021120:2048]
2025-12-24T02:11:07.871354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241267693021120:2048] [node 1] ICS09 handshake done sender: [4:7587241267693021081:2108] self: [4:7587241267693021115:0] peer: [1:7587241265536353175:0] socket: 392 qp: -1
2025-12-24T02:11:07.871361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241267693021120:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.871380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.875207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241263876239136:2099] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241263876239173:2120] held: no
2025-12-24T02:11:07.875282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241263876239136:2099] [node 5] ICP35 event processing took too much time PendingConnection:279(19984059ns) {HandleHandshakeStatus:395(19983482ns) {LogHandshakeFail:472(3233ns) {UpdateErrorStateLog:545(1377ns)} DropOutgoingHandshake:457(960ns) ProcessPendingSessionEvents:495(16997ns) {DropSessionEvent:512(4147ns) DropSessionEvent:512(1201ns) DropSessionEvent:512(1111ns) DropSessionEvent:512(2893ns) DropSessionEvent:512(933ns) DropSessionEvent:512(616ns) DropSessionEvent:512(1305ns) DropSessionEvent:512(862ns) DropSessionEvent:512(1254ns) DropSessionEvent:512(1109ns)}}}
2025-12-24T02:11:07.875163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241267116537877:2097] [node 6] ICP35 event processing took too much time PendingConnection:279(26639029ns) {HandleHandshakeStatus:329(26638438ns) {DropIncomingHandshake:442(3776ns) DropOutgoingHandshake:457(125ns) SwitchToState:179(94ns) ProcessPendingSessionEvents:495(23739270ns) {ForwardSessionEventToSession:612(23721572ns) ForwardSessionEventToSession:612(3387ns) ForwardSessionEventToSession:612(1004ns) ForwardSessionEventToSession:612(1081ns) ForwardSessionEventToSession:612(848ns) ForwardSessionEventToSession:612(1148ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(782ns) ForwardSessionEventToSession:612(767ns) ForwardSessionEventToSession:612(986ns)}}}
2025-12-24T02:11:07.878908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241267116537907:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.878959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353181:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.878974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241265536353181:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.882406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239176:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.882569Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241267693021092:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(27663770ns) {HandleHandshakeStatus:329(27662861ns) {DropIncomingHandshake:442(5102ns) DropOutgoingHandshake:457(224ns) SwitchToState:179(164ns) ProcessPendingSessionEvents:495(11689048ns) {ForwardSessionEventToSession:612(11678114ns) ForwardSessionEventToSession:612(1593ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(554ns) ForwardSessionEventToSession:612(448ns) ForwardSessionEventToSession:612(1034ns) ForwardSessionEventToSession:612(1029ns) ForwardSessionEventToSession:612(437ns) ForwardSessionEventToSession:612(515ns) ForwardSessionEventToSession:612(737ns)}}}
2025-12-24T02:11:07.882343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241267116537880:2100] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16969) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241267116537907:2119] held: no
2025-12-24T02:11:07.882910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353199:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.882372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241267116537880:2100] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.882382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.882389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241267116537880:2100] [node 4] ICP22 created new session: [5:7587241267116537918:2100]
2025-12-24T02:11:07.882396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241267116537918:2100] [node 4] ICS09 handshake done sender: [5:7587241267116537907:2119] self: [5:7587241267116537908:0] peer: [4:7587241267693021096:0] socket: 446 qp: -1
2025-12-24T02:11:07.882402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241267116537918:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:07.882408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.882485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241267116537823:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24928) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241267116537887:2109] held: no
2025-12-24T02:11:07.882589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241267116537821:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.882646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537917:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.882759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239177:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.882824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241263876239173:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:07.883159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241263876239177:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.883182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241263876239144:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.883304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241263876239136:2099] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.883315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.883321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241263876239136:2099] [node 5] ICP22 created new session: [6:7587241263876239179:2099]
2025-12-24T02:11:07.883325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241263876239179:2099] [node 5] ICS09 handshake done sender: [6:7587241263876239173:2120] self: [6:7587241263876239174:0] peer: [5:7587241267116537895:0] socket: 433 qp: -1
2025-12-24T02:11:07.883330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241263876239179:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:07.883345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:07.887014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241263876239083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:07.887061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241263876239083:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16295319ns) {IncomingHandshake:261(16294496ns) {DropIncomingHandshake:442(129ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(210ns)} GenerateSessionVirtualId:988(210ns) IssueIncomingHandshakeReply:193(16287128ns) {SwitchToState:179(518ns)}}}
2025-12-24T02:11:07.887851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241265536353185:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.887864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241265536353185:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.888334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241265536353181:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.888628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241265536353185:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.888768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239181:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.888893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241265536353174:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.888919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:07.888927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241265536353174:2110] [node 5] ICP22 created new session: [1:7587241265536353206:2110]
2025-12-24T02:11:07.888933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241265536353206:2110] [node 5] ICS09 handshake done sender: [1:7587241265536353185:2116] self: [1:7587241265536353182:0] peer: [5:7587241267116537920:0] socket: 449 qp: -1
2025-12-24T02:11:07.888941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241265536353206:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:07.888948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.889771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241263876239168:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.887378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241267116537887:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:07.889866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241263876239083:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.889880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.889886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241263876239083:2048] [node 1] ICP22 created new session: [6:7587241263876239182:2048]
2025-12-24T02:11:07.889891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241263876239182:2048] [node 1] ICS09 handshake done sender: [6:7587241263876239168:2118] self: [6:7587241263876239175:0] peer: [1:7587241265536353180:0] socket: 416 qp: -1
2025-12-24T02:11:07.889896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241263876239182:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.889966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:07.890956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241265536353179:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.890976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.890983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241265536353179:2112] [node 6] ICP22 created new session: [1:7587241265536353208:2112]
2025-12-24T02:11:07.890989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241265536353208:2112] [node 6] ICS09 handshake done sender: [1:7587241265536353181:2113] self: [1:7587241265536353180:0] peer: [6:7587241263876239175:0] socket: 425 qp: -1
2025-12-24T02:11:07.890995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241265536353208:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:07.891002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.889418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241267116537823:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.889429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:07.889435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241267116537823:2050] [node 3] ICP22 created new session: [5:7587241267116537922:2050]
2025-12-24T02:11:07.889441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241267116537922:2050] [node 3] ICS09 handshake done sender: [5:7587241267116537887:2109] self: [5:7587241267116537916:0] peer: [3:7587241267358146039:0] socket: 401 qp: -1
2025-12-24T02:11:07.889446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241267116537922:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:07.889453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:07.889613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241267116537921:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16416
2025-12-24T02:11:07.894816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241267693021101:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:07.894834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241267693021101:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:07.899064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241267116537821:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30169) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241267116537910:2120] held: no
2025-12-24T02:11:07.899171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241267116537910:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:07.899952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241267116537821:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.899966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:07.899973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241267116537821:2048] [node 1] ICP22 created new session: [5:7587241267116537924:2048]
2025-12-24T02:11:07.899981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241267116537924:2048] [node 1] ICS09 handshake done sender: [5:7587241267116537910:2120] self: [5:7587241267116537920:0] peer: [1:7587241265536353182:0] socket: 450 qp: -1
2025-12-24T02:11:07.899989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241267116537924:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:07.899996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:07.903511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241263876239184:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8757
2025-12-24T02:11:07.906895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241267693021101:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:07.907014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241267693021095:2116] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:07.907038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:07.907046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241267693021095:2116] [node 6] ICP22 created new session: [4:7587241267693021123:2116]
2025-12-24T02:11:07.907052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241267693021123:2116] [node 6] ICS09 handshake done sender: [4:7587241267693021101:2118] self: [4:7587241267693021100:0] peer: [6:7587241263876239178:0] socket: 436 qp: -1
2025-12-24T02:11:07.907059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241267693021123:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:07.907067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:07.908196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241263876239144:2103] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16969) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241263876239177:2122] held: no
2025-12-24T02:11:07.908440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241263876239177:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:07.908557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241263876239144:2103] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:07.908571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:07.908576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241263876239144:2103] [node 4] ICP22 created new session: [6:7587241263876239185:2103]
2025-12-24T02:11:07.908582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241263876239185:2103] [node 4] ICS09 handshake done sender: [6:7587241263876239177:2122] self: [6:7587241263876239178:0] peer: [4:7587241267693021100:0] socket: 404 qp: -1
2025-12-24T02:11:07.908590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241263876239185:2103] [node 4] ICS10 traffic start
2025-12-24T02:11:07.908601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:07.918779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:07.918867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241267764605870:2048] [node 1] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:07.918874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241267764605770:2048] [node 1] ICP30 unregister session Session# [2:7587241267764605870:2048] VirtualId# [2:7587241267764605865:0]
2025-12-24T02:11:07.918881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241267764605870:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.918771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:07.918928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.918955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.918867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241267358146076:2048] [node 1] ICS01 socket: 393 reason# EndOfStream
2025-12-24T02:11:07.918875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241267358145977:2048] [node 1] ICP30 unregister session Session# [3:7587241267358146076:2048] VirtualId# [3:7587241267358146067:0]
2025-12-24T02:11:07.918881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241267358146076:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.918928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.918954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.922931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:07.922998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241263876239182:2048] [node 1] ICS01 socket: 416 reason# EndOfStream
2025-12-24T02:11:07.923004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241263876239083:2048] [node 1] ICP30 unregister session Session# [6:7587241263876239182:2048] VirtualId# [6:7587241263876239175:0]
2025-12-24T02:11:07.923009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241263876239182:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.923050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.923069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.922845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:07.922923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241267693021120:2048] [node 1] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:07.922930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241267693021020:2048] [node 1] ICP30 unregister session Session# [4:7587241267693021120:2048] VirtualId# [4:7587241267693021115:0]
2025-12-24T02:11:07.922936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241267693021120:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.922984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.923013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.923048Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:07.923112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241267116537924:2048] [node 1] ICS01 socket: 450 reason# EndOfStream
2025-12-24T02:11:07.923116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241267116537821:2048] [node 1] ICP30 unregister session Session# [5:7587241267116537924:2048] VirtualId# [5:7587241267116537920:0]
2025-12-24T02:11:07.923120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241267116537924:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.923160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:07.923179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:07.926975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:07.927052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241267358146062:2049] [node 2] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:07.927058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241267358145978:2049] [node 2] ICP30 unregister session Session# [3:7587241267358146062:2049] VirtualId# [3:7587241267358146060:0]
2025-12-24T02:11:07.927062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241267358146062:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.927088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.927106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.938785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:07.938878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241263876239165:2049] [node 2] ICS01 socket: 389 reason# EndOfStream
2025-12-24T02:11:07.938886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241263876239084:2049] [node 2] ICP30 unregister session Session# [6:7587241263876239165:2049] VirtualId# [6:7587241263876239163:0]
2025-12-24T02:11:07.938890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241263876239165:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.938942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.938964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.938889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:07.938981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241267693021105:2049] [node 2] ICS01 socket: 399 reason# EndOfStream
2025-12-24T02:11:07.938988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241267693021021:2049] [node 2] ICP30 unregister session Session# [4:7587241267693021105:2049] VirtualId# [4:7587241267693021102:0]
2025-12-24T02:11:07.938994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241267693021105:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.939043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.939067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.939113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:07.939182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241267116537905:2049] [node 2] ICS01 socket: 434 reason# EndOfStream
2025-12-24T02:11:07.939188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241267116537822:2049] [node 2] ICP30 unregister session Session# [5:7587241267116537905:2049] VirtualId# [5:7587241267116537903:0]
2025-12-24T02:11:07.939193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241267116537905:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.939225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:07.939247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:07.954848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:07.954932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241263876239171:2050] [node 3] ICS01 socket: 418 reason# EndOfStream
2025-12-24T02:11:07.954940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241263876239085:2050] [node 3] ICP30 unregister session Session# [6:7587241263876239171:2050] VirtualId# [6:7587241263876239169:0]
2025-12-24T02:11:07.954946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241263876239171:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.954996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.955019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.955333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:07.955421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241267116537922:2050] [node 3] ICS01 socket: 401 reason# EndOfStream
2025-12-24T02:11:07.955428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241267116537823:2050] [node 3] ICP30 unregister session Session# [5:7587241267116537922:2050] VirtualId# [5:7587241267116537916:0]
2025-12-24T02:11:07.955436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241267116537922:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.955488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:07.955515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:07.974849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:07.974936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241263876239185:2103] [node 4] ICS01 socket: 404 reason# EndOfStream
2025-12-24T02:11:07.974948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241263876239144:2103] [node 4] ICP30 unregister session Session# [6:7587241263876239185:2103] VirtualId# [6:7587241263876239178:0]
2025-12-24T02:11:07.974954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241263876239185:2103] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.975015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:07.975037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:07.987950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:07.988032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241263876239179:2099] [node 5] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:07.988039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241263876239136:2099] [node 5] ICP30 unregister session Session# [6:7587241263876239179:2099] VirtualId# [6:7587241263876239174:0]
2025-12-24T02:11:07.988045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241263876239179:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:07.988080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:07.988098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:08.008691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241269367567682:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.008709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241269367567683:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.006751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270851989643:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:08.006767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270851989644:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.010481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271467274422:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.010489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271467274423:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.021784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268568698589:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.021800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268568698590:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.021805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268568698591:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.014606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241270018171531:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.014617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241270018171532:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.014622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241270018171533:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.019408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270595672756:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.019418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270595672757:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.019425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270595672758:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.071826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270851989704:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:08.074062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270851989711:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.074880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274482:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.077050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270595672822:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.078800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672816:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.078897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241269367567749:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:08.078991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241269367567750:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.079226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270851989716:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13662
2025-12-24T02:11:08.086820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270595672811:2100] [node 5] ICP01 ready to work
2025-12-24T02:11:08.094327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241269367567754:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.099823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698653:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.099845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698654:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.095109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241270851989644:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23228990ns) {Configure:104(23227735ns) {ProcessConfigured:135(22945989ns) {StartInitialHandshake:157(22945621ns) {DropHandshakes:470(437ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(949ns) {SwitchToState:179(438ns)}}}}}
2025-12-24T02:11:08.103420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270851989719:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.134741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274482:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.135196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274490:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.135728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241270851989719:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28856744ns) {Configure:104(28855446ns) {ProcessConfigured:135(28476810ns) {StartInitialHandshake:157(28476334ns) {DropHandshakes:470(723ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(129ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:477(574ns) {SwitchToState:179(259ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.136362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567760:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
2025-12-24T02:11:08.135447Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241268568698590:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28511851ns) {Configure:104(28510047ns) {ProcessConfigured:135(28232574ns) {StartInitialHandshake:157(28232103ns) {DropHandshakes:470(1101ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(1267ns) {SwitchToState:179(737ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.138877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171598:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.139506Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241268568698589:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32301808ns) {Configure:104(32300861ns) {ProcessConfigured:135(32046773ns) {StartInitialHandshake:157(32046463ns) {DropHandshakes:470(334ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(825ns) {SwitchToState:179(457ns)}}}}}
2025-12-24T02:11:08.139564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698654:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.141558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268568698660:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:08.142881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567760:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.142919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241269367567682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.142828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268568698659:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:08.142870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271467274489:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:08.142947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271467274488:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:08.150394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271467274487:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:08.151519Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241271467274422:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61020452ns) {Configure:104(61018897ns) {ProcessConfigured:135(60613652ns) {StartInitialHandshake:157(60613226ns) {DropHandshakes:470(700ns) {DropIncomingHandshake:442(172ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(449ns) PrepareNewSessionHandshake:477(902ns) {SwitchToState:179(355ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.151863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241270018171532:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41239392ns) {Configure:104(28244939ns) {ProcessConfigured:135(27931081ns) {StartInitialHandshake:157(27930742ns) {DropHandshakes:470(1220ns) {DropIncomingHandshake:442(147ns) DropOutgoingHandshake:457(168ns)} GenerateSessionVirtualId:988(517ns) PrepareNewSessionHandshake:477(707ns) {SwitchToState:179(125ns)}}}}}
2025-12-24T02:11:08.151695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672834:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.151768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672835:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.152093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270595672834:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.152120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270595672757:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.154883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567763:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
No RDMA context found for GID: 2025-12-24T02:11:08.154908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567764:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.154926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
2025-12-24T02:11:08.154965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567762:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
2025-12-24T02:11:08.154973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567755:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.154976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241269367567755:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.157284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241269367567755:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.157410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241269367567750:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.157431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.157439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241269367567750:2110] [node 6] ICP22 created new session: [2:7587241269367567768:2110]
2025-12-24T02:11:08.157446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241269367567768:2110] [node 6] ICS09 handshake done sender: [2:7587241269367567755:2112] self: [2:7587241269367567753:0] peer: [6:7587241270595672836:0] socket: 406 qp: -1
2025-12-24T02:11:08.157456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241269367567768:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.157475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.158892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171604:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.159091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672837:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.159301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270595672834:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.159372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171598:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.159400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241270018171532:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.161254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567762:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.161293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567752:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.161297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241269367567752:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.161679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241269367567750:2110] [node 6] ICP18 handshake (actor [2:7587241269367567762:2115]) is too old
2025-12-24T02:11:08.159500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270595672757:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.159517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.159522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270595672757:2049] [node 2] ICP22 created new session: [6:7587241270595672838:2049]
2025-12-24T02:11:08.159527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270595672838:2049] [node 2] ICS09 handshake done sender: [6:7587241270595672834:2115] self: [6:7587241270595672836:0] peer: [2:7587241269367567753:0] socket: 407 qp: -1
2025-12-24T02:11:08.159531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270595672838:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.159537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.162782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989708:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.162798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270851989708:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.164736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241270018171596:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:08.161945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241269367567752:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.162024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241269367567749:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.162033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.162039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241269367567749:2109] [node 5] ICP22 created new session: [2:7587241269367567771:2109]
2025-12-24T02:11:08.162045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241269367567771:2109] [node 5] ICS09 handshake done sender: [2:7587241269367567752:2111] self: [2:7587241269367567751:0] peer: [5:7587241270018171605:0] socket: 405 qp: -1
2025-12-24T02:11:08.162051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241269367567771:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:08.162057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.163194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270595672816:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.166718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241270018171597:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:08.170835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241268568698660:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29187772ns) {Configure:104(29186949ns) {ProcessConfigured:135(28882706ns) {StartInitialHandshake:157(28882217ns) {DropHandshakes:470(1040ns) {DropIncomingHandshake:442(126ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(1149ns) {SwitchToState:179(768ns)}}}}}
2025-12-24T02:11:08.170909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698653:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.170943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268568698589:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.173640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274496:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.173653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271467274496:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.174483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271467274496:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.173423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270595672835:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.173479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270595672758:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.174344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672842:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.174735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270595672835:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.174861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270595672758:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.174888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.174894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270595672758:2050] [node 3] ICP22 created new session: [6:7587241270595672843:2050]
2025-12-24T02:11:08.174900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270595672843:2050] [node 3] ICS09 handshake done sender: [6:7587241270595672835:2116] self: [6:7587241270595672841:0] peer: [3:7587241271467274495:0] socket: 424 qp: -1
2025-12-24T02:11:08.174907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270595672843:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.174917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.171637Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241269367567754:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(77224030ns) {Configure:104(77222978ns) {ProcessConfigured:135(76818421ns) {StartInitialHandshake:157(76818001ns) {DropHandshakes:470(794ns) {DropIncomingHandshake:442(191ns) DropOutgoingHandshake:457(123ns)} GenerateSessionVirtualId:988(293ns) PrepareNewSessionHandshake:477(822ns) {SwitchToState:179(99ns)}}}}}
2025-12-24T02:11:08.174822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989717:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.174842Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270851989717:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.178868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271467274488:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.178894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.178902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271467274488:2110] [node 6] ICP22 created new session: [3:7587241271467274506:2110]
2025-12-24T02:11:08.178908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271467274506:2110] [node 6] ICS09 handshake done sender: [3:7587241271467274496:2113] self: [3:7587241271467274495:0] peer: [6:7587241270595672841:0] socket: 422 qp: -1
2025-12-24T02:11:08.178915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271467274506:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.178936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.179170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241271467274489:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36168593ns) {Configure:104(36167773ns) {ProcessConfigured:135(35853873ns) {StartInitialHandshake:157(35853286ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(550ns) {SwitchToState:179(261ns)}}}}}
2025-12-24T02:11:08.179297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241270595672756:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(64213976ns) {Configure:104(64212930ns) {ProcessConfigured:135(63859631ns) {StartInitialHandshake:157(63859420ns) {DropHandshakes:470(599ns) {DropIncomingHandshake:442(99ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(677ns) {SwitchToState:179(96ns)}}}}}
2025-12-24T02:11:08.179332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270595672756:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.183314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989716:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.182819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241270018171531:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(72323917ns) {Configure:104(72323105ns) {ProcessConfigured:135(71931297ns) {StartInitialHandshake:157(71930680ns) {DropHandshakes:470(845ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(584ns) PrepareNewSessionHandshake:477(1515ns) {SwitchToState:179(937ns)}}}}}
2025-12-24T02:11:08.186893Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241268568698659:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43981411ns) {Configure:104(43981010ns) {ProcessConfigured:135(43741977ns) {StartInitialHandshake:157(43724031ns) {DropHandshakes:470(523ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(114ns)} GenerateSessionVirtualId:988(644ns) PrepareNewSessionHandshake:477(810ns) {SwitchToState:179(542ns)}} IncomingHandshake:261(7351ns) {DropIncomingHandshake:442(138ns) PrepareNewSessionHandshake:477(126ns) {SwitchToState:179(75ns)} GenerateSessionVirtualId:988(173ns) IssueIncomingHandshakeReply:193(350ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.187748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274504:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.187777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.189166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274505:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.189197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241271467274423:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.191290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567758:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.191304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241269367567758:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.191872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274504:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.193085Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241270018171596:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28273684ns) {Configure:104(28273137ns) {ProcessConfigured:135(28008261ns) {StartInitialHandshake:157(28007534ns) {DropHandshakes:470(586ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(784ns) {SwitchToState:179(491ns)}}}}}
2025-12-24T02:11:08.193190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270851989708:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.193247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241269367567758:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.197449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567765:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.197534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241269367567749:2109] [node 5] ICP18 handshake (actor [2:7587241269367567765:2118]) is too old
2025-12-24T02:11:08.194826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241270595672758:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19927511ns) {HandleHandshakeStatus:329(19926916ns) {DropIncomingHandshake:442(215ns) DropOutgoingHandshake:457(3788ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(19822449ns) {ForwardSessionEventToSession:612(19805484ns) ForwardSessionEventToSession:612(2810ns) ForwardSessionEventToSession:612(1090ns) ForwardSessionEventToSession:612(1401ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(911ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(1253ns)}}}
2025-12-24T02:11:08.198361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241270018171532:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15369) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241270018171598:2110] held: no
2025-12-24T02:11:08.198485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171604:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.198513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241270018171533:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.198621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989714:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.198637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270851989714:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.194872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274512:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.198937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270851989717:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.198943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270851989726:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13662
2025-12-24T02:11:08.195539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698671:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.195609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698672:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.198874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698671:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.198907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268568698590:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.198928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698673:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.199069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567764:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.199115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241269367567773:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.199119Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241269367567773:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.198820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271467274505:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.199074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241271467274423:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.199087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.199094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271467274423:2049] [node 2] ICP22 created new session: [3:7587241271467274513:2049]
2025-12-24T02:11:08.199096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241271467274488:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(20199877ns) {HandleHandshakeStatus:329(20199082ns) {DropIncomingHandshake:442(4740ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(12280994ns) {ForwardSessionEventToSession:612(12268593ns) ForwardSessionEventToSession:612(1874ns) ForwardSessionEventToSession:612(1354ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(480ns) ForwardSessionEventToSession:612(1196ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(533ns) ForwardSessionEventToSession:612(567ns) ForwardSessionEventToSession:612(1244ns)}}}
2025-12-24T02:11:08.199100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271467274513:2049] [node 2] ICS09 handshake done sender: [3:7587241271467274505:2117] self: [3:7587241271467274510:0] peer: [2:7587241269367567745:0] socket: 438 qp: -1
2025-12-24T02:11:08.199108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271467274513:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.199127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.199974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268568698653:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.199648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270851989643:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.199666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.199672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270851989643:2048] [node 2] ICP22 created new session: [1:7587241270851989728:2048]
2025-12-24T02:11:08.199679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270851989728:2048] [node 2] ICS09 handshake done sender: [1:7587241270851989708:2108] self: [1:7587241270851989705:0] peer: [2:7587241269367567761:0] socket: 392 qp: -1
2025-12-24T02:11:08.199685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270851989728:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:08.199690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.200123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241269367567777:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2364
2025-12-24T02:11:08.203204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241269367567683:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.203224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.203233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241269367567683:2049] [node 3] ICP22 created new session: [2:7587241269367567778:2049]
2025-12-24T02:11:08.203240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241269367567778:2049] [node 3] ICS09 handshake done sender: [2:7587241269367567758:2113] self: [2:7587241269367567745:0] peer: [3:7587241271467274510:0] socket: 430 qp: -1
2025-12-24T02:11:08.203247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241269367567778:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.203261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.203540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989726:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.203708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270851989704:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.203716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.203722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270851989704:2107] [node 4] ICP22 created new session: [1:7587241270851989734:2107]
2025-12-24T02:11:08.203728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270851989734:2107] [node 4] ICS09 handshake done sender: [1:7587241270851989717:2113] self: [1:7587241270851989706:0] peer: [4:7587241268568698668:0] socket: 400 qp: -1
2025-12-24T02:11:08.203733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270851989734:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:08.203749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.203827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241270851989704:2107] [node 4] ICP18 handshake (actor [1:7587241270851989726:2116]) is too old
2025-12-24T02:11:08.203900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270851989730:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13662
2025-12-24T02:11:08.203947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270851989731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13662
2025-12-24T02:11:08.206763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268568698589:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.206783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.206791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268568698589:2048] [node 1] ICP22 created new session: [4:7587241268568698675:2048]
2025-12-24T02:11:08.206796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268568698675:2048] [node 1] ICS09 handshake done sender: [4:7587241268568698653:2109] self: [4:7587241268568698668:0] peer: [1:7587241270851989706:0] socket: 403 qp: -1
2025-12-24T02:11:08.206802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268568698675:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.206832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.202955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274515:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.203151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274499:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.203158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271467274499:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.207281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274490:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.208993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989730:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.210439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270851989732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13662
2025-12-24T02:11:08.211309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241269367567773:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.211527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698677:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.211787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268568698671:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.211415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241269367567754:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.211436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.211442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241269367567754:2108] [node 4] ICP22 created new session: [2:7587241269367567781:2108]
2025-12-24T02:11:08.211447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241269367567781:2108] [node 4] ICS09 handshake done sender: [2:7587241269367567773:2119] self: [2:7587241269367567759:0] peer: [4:7587241268568698674:0] socket: 434 qp: -1
2025-12-24T02:11:08.211454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241269367567781:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:08.211471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.211931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268568698590:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.211946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.211951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268568698590:2049] [node 2] ICP22 created new session: [4:7587241268568698678:2049]
2025-12-24T02:11:08.211956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268568698678:2049] [node 2] ICS09 handshake done sender: [4:7587241268568698671:2117] self: [4:7587241268568698674:0] peer: [2:7587241269367567759:0] socket: 437 qp: -1
2025-12-24T02:11:08.211961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268568698678:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.211966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.213395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270851989714:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.213498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989732:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.213560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270851989711:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.213567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.213573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270851989711:2110] [node 6] ICP22 created new session: [1:7587241270851989738:2110]
2025-12-24T02:11:08.213577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270851989738:2110] [node 6] ICS09 handshake done sender: [1:7587241270851989714:2111] self: [1:7587241270851989713:0] peer: [6:7587241270595672846:0] socket: 394 qp: -1
2025-12-24T02:11:08.213581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270851989738:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.214756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698673:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.213587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.213687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241270851989711:2110] [node 6] ICP18 handshake (actor [1:7587241270851989732:2119]) is too old
2025-12-24T02:11:08.214987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274515:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.215103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241271467274422:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.215301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698682:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.217256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171614:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.217862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271467274499:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.217947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171614:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.217669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698682:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.217701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268568698591:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.218859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241269367567683:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(25466382ns) {HandleHandshakeStatus:329(25465692ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(191ns) ProcessPendingSessionEvents:495(42537ns) {ForwardSessionEventToSession:612(31478ns) ForwardSessionEventToSession:612(725ns) ForwardSessionEventToSession:612(1144ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(619ns) ForwardSessionEventToSession:612(856ns) ForwardSessionEventToSession:612(522ns) ForwardSessionEventToSession:612(662ns) ForwardSessionEventToSession:612(913ns) ForwardSessionEventToSession:612(1095ns)}}}
2025-12-24T02:11:08.218951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241269367567683:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:08.219245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989731:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.222146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241270595672756:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241270595672816:2104] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.222934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171616:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.223069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171617:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.223106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171618:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.223134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171619:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.223041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672848:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.223574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271467274508:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.223584Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271467274508:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.223356Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241270851989643:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23679204ns) {HandleHandshakeStatus:329(23678665ns) {DropIncomingHandshake:442(3852ns) DropOutgoingHandshake:457(157ns) SwitchToState:179(121ns) ProcessPendingSessionEvents:495(20091857ns) {ForwardSessionEventToSession:612(20081587ns) ForwardSessionEventToSession:612(1577ns) ForwardSessionEventToSession:612(1477ns) ForwardSessionEventToSession:612(415ns) ForwardSessionEventToSession:612(489ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(498ns) ForwardSessionEventToSession:612(1238ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(1016ns)}}}
2025-12-24T02:11:08.223524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989720:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.223530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270851989720:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.223459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241269367567682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241269367567760:2114] held: no
2025-12-24T02:11:08.223512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241269367567760:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.226910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241269367567682:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.226926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.226933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241269367567682:2048] [node 1] ICP22 created new session: [2:7587241269367567783:2048]
2025-12-24T02:11:08.226938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241269367567783:2048] [node 1] ICS09 handshake done sender: [2:7587241269367567760:2114] self: [2:7587241269367567761:0] peer: [1:7587241270851989705:0] socket: 393 qp: -1
2025-12-24T02:11:08.226945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241269367567783:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.226965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.226772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271467274487:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.226810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.226819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271467274487:2108] [node 5] ICP22 created new session: [3:7587241271467274520:2108]
2025-12-24T02:11:08.226826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271467274520:2108] [node 5] ICS09 handshake done sender: [3:7587241271467274499:2114] self: [3:7587241271467274498:0] peer: [5:7587241270018171612:0] socket: 423 qp: -1
2025-12-24T02:11:08.226834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271467274520:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:08.226871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.230871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672849:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.230937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270595672816:2104] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.230856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171617:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.230882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171618:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.230950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241270018171533:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241270018171604:2113] held: no
2025-12-24T02:11:08.230998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241270018171531:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.231010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241270018171596:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.231302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241270018171598:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.231327Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241270018171604:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.233578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241270018171532:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.233594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.233601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241270018171532:2049] [node 2] ICP22 created new session: [5:7587241270018171622:2049]
2025-12-24T02:11:08.233607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241270018171622:2049] [node 2] ICS09 handshake done sender: [5:7587241270018171598:2110] self: [5:7587241270018171605:0] peer: [2:7587241269367567751:0] socket: 408 qp: -1
2025-12-24T02:11:08.233612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241270018171622:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.233619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.233803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241270018171533:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.233810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.233814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241270018171533:2050] [node 3] ICP22 created new session: [5:7587241270018171624:2050]
2025-12-24T02:11:08.233819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241270018171624:2050] [node 3] ICS09 handshake done sender: [5:7587241270018171604:2113] self: [5:7587241270018171612:0] peer: [3:7587241271467274498:0] socket: 431 qp: -1
2025-12-24T02:11:08.233823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241270018171624:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.233840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.234898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271467274508:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.235098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271467274523:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6258
2025-12-24T02:11:08.235226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271467274489:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.235244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.235250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271467274489:2109] [node 4] ICP22 created new session: [3:7587241271467274524:2109]
2025-12-24T02:11:08.235256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271467274524:2109] [node 4] ICS09 handshake done sender: [3:7587241271467274508:2118] self: [3:7587241271467274494:0] peer: [4:7587241268568698683:0] socket: 415 qp: -1
2025-12-24T02:11:08.235275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271467274524:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:08.235282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.235259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270595672849:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.235294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270595672756:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.235304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.235311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270595672756:2048] [node 1] ICP22 created new session: [6:7587241270595672850:2048]
2025-12-24T02:11:08.235319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270595672850:2048] [node 1] ICS09 handshake done sender: [6:7587241270595672816:2104] self: [6:7587241270595672846:0] peer: [1:7587241270851989713:0] socket: 395 qp: -1
2025-12-24T02:11:08.235326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270595672850:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.235334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.235485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270595672822:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.235822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270851989720:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.236195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270851989644:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.236213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.236218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270851989644:2049] [node 3] ICP22 created new session: [1:7587241270851989742:2049]
2025-12-24T02:11:08.236224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270851989742:2049] [node 3] ICS09 handshake done sender: [1:7587241270851989720:2114] self: [1:7587241270851989707:0] peer: [3:7587241271467274518:0] socket: 447 qp: -1
2025-12-24T02:11:08.236229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270851989742:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.236235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.238244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270851989722:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.238262Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270851989722:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.238642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698667:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.238656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241268568698667:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.238920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241268568698591:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29208) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241268568698682:2121] held: no
2025-12-24T02:11:08.236042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271467274515:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.236558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241271467274422:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.236569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.236577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271467274422:2048] [node 1] ICP22 created new session: [3:7587241271467274526:2048]
2025-12-24T02:11:08.236582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271467274526:2048] [node 1] ICS09 handshake done sender: [3:7587241271467274515:2120] self: [3:7587241271467274518:0] peer: [1:7587241270851989707:0] socket: 448 qp: -1
2025-12-24T02:11:08.236588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271467274526:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.236593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.245088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270851989722:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.245260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270851989719:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.245286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.245295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270851989719:2109] [node 5] ICP22 created new session: [1:7587241270851989745:2109]
2025-12-24T02:11:08.245302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270851989745:2109] [node 5] ICS09 handshake done sender: [1:7587241270851989722:2115] self: [1:7587241270851989721:0] peer: [5:7587241270018171620:0] socket: 451 qp: -1
2025-12-24T02:11:08.245310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270851989745:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:08.245319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.246837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268568698669:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.246854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241268568698669:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.250958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268568698667:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.250999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241270018171531:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241270018171617:2119] held: no
2025-12-24T02:11:08.251244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241268568698660:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.251271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.251279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268568698660:2112] [node 5] ICP22 created new session: [4:7587241268568698686:2112]
2025-12-24T02:11:08.251285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268568698686:2112] [node 5] ICS09 handshake done sender: [4:7587241268568698667:2115] self: [4:7587241268568698665:0] peer: [5:7587241270018171621:0] socket: 427 qp: -1
2025-12-24T02:11:08.251292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268568698686:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:08.251301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.255408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241270018171596:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241270018171618:2120] held: no
2025-12-24T02:11:08.255462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171626:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.255677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241270018171627:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23372
2025-12-24T02:11:08.256710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241270018171617:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.257603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241270018171618:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.257845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268568698669:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.258379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268568698689:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59925
2025-12-24T02:11:08.258173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241270018171531:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.258188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.258193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241270018171531:2048] [node 1] ICP22 created new session: [5:7587241270018171629:2048]
2025-12-24T02:11:08.258197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241270018171629:2048] [node 1] ICS09 handshake done sender: [5:7587241270018171617:2119] self: [5:7587241270018171620:0] peer: [1:7587241270851989721:0] socket: 418 qp: -1
2025-12-24T02:11:08.258202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241270018171629:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.258207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.258518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268568698682:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.258597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241268568698659:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.258611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.258616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268568698659:2111] [node 6] ICP22 created new session: [4:7587241268568698690:2111]
2025-12-24T02:11:08.258620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268568698690:2111] [node 6] ICS09 handshake done sender: [4:7587241268568698669:2116] self: [4:7587241268568698666:0] peer: [6:7587241270595672852:0] socket: 426 qp: -1
2025-12-24T02:11:08.258627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268568698690:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:08.258633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.256429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672853:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.256450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672854:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.257959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270595672849:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.258030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270595672822:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.258042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.258048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270595672822:2108] [node 4] ICP22 created new session: [6:7587241270595672855:2108]
2025-12-24T02:11:08.258053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270595672855:2108] [node 4] ICS09 handshake done sender: [6:7587241270595672849:2121] self: [6:7587241270595672852:0] peer: [4:7587241268568698666:0] socket: 457 qp: -1
2025-12-24T02:11:08.258059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270595672855:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:08.258069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.258668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241270018171596:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.258679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.258682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241270018171596:2108] [node 4] ICP22 created new session: [5:7587241270018171631:2108]
2025-12-24T02:11:08.258686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241270018171631:2108] [node 4] ICS09 handshake done sender: [5:7587241270018171618:2120] self: [5:7587241270018171621:0] peer: [4:7587241268568698665:0] socket: 450 qp: -1
2025-12-24T02:11:08.258690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241270018171631:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:08.258708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.267020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268568698591:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.267036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.267042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268568698591:2050] [node 3] ICP22 created new session: [4:7587241268568698692:2050]
2025-12-24T02:11:08.267048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268568698692:2050] [node 3] ICS09 handshake done sender: [4:7587241268568698682:2121] self: [4:7587241268568698683:0] peer: [3:7587241271467274494:0] socket: 455 qp: -1
2025-12-24T02:11:08.267056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268568698692:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.267062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.270874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270595672853:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.270923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270595672811:2100] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.271102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241270018171611:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.271112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241270018171611:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.271902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270595672858:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26427
2025-12-24T02:11:08.271997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241270018171611:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.272179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270595672853:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.272280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270595672811:2100] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.272295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.272300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270595672811:2100] [node 5] ICP22 created new session: [6:7587241270595672859:2100]
2025-12-24T02:11:08.272305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270595672859:2100] [node 5] ICS09 handshake done sender: [6:7587241270595672853:2122] self: [6:7587241270595672857:0] peer: [5:7587241270018171608:0] socket: 449 qp: -1
2025-12-24T02:11:08.272310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270595672859:2100] [node 5] ICS10 traffic start
2025-12-24T02:11:08.272324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.273050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241270018171597:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.273067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.273072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241270018171597:2109] [node 6] ICP22 created new session: [5:7587241270018171634:2109]
2025-12-24T02:11:08.273076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241270018171634:2109] [node 6] ICS09 handshake done sender: [5:7587241270018171611:2116] self: [5:7587241270018171608:0] peer: [6:7587241270595672857:0] socket: 417 qp: -1
2025-12-24T02:11:08.273081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241270018171634:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:08.273086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.286160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:08.286219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241271467274526:2048] [node 1] ICS01 socket: 448 reason# EndOfStream
2025-12-24T02:11:08.286224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241271467274422:2048] [node 1] ICP30 unregister session Session# [3:7587241271467274526:2048] VirtualId# [3:7587241271467274518:0]
2025-12-24T02:11:08.286229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241271467274526:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.286252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.286268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.285423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:08.285516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241269367567783:2048] [node 1] ICS01 socket: 393 reason# EndOfStream
2025-12-24T02:11:08.285524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241269367567682:2048] [node 1] ICP30 unregister session Session# [2:7587241269367567783:2048] VirtualId# [2:7587241269367567761:0]
2025-12-24T02:11:08.285531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241269367567783:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.285589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.285611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.292090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:08.292174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241270018171629:2048] [node 1] ICS01 socket: 418 reason# EndOfStream
2025-12-24T02:11:08.292181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241270018171531:2048] [node 1] ICP30 unregister session Session# [5:7587241270018171629:2048] VirtualId# [5:7587241270018171620:0]
2025-12-24T02:11:08.292188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241270018171629:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.292235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.292260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.292295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:08.292356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270595672850:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-12-24T02:11:08.292360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270595672756:2048] [node 1] ICP30 unregister session Session# [6:7587241270595672850:2048] VirtualId# [6:7587241270595672846:0]
2025-12-24T02:11:08.292365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270595672850:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.292390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.292411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.295828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:08.295940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241268568698675:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-12-24T02:11:08.295948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241268568698589:2048] [node 1] ICP30 unregister session Session# [4:7587241268568698675:2048] VirtualId# [4:7587241268568698668:0]
2025-12-24T02:11:08.295955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241268568698675:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.296026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.296056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.303072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:08.303153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241271467274513:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:08.303163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241271467274423:2049] [node 2] ICP30 unregister session Session# [3:7587241271467274513:2049] VirtualId# [3:7587241271467274510:0]
2025-12-24T02:11:08.303170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241271467274513:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.303206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.303224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.310752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:08.310848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241270018171622:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-12-24T02:11:08.310857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241270018171532:2049] [node 2] ICP30 unregister session Session# [5:7587241270018171622:2049] VirtualId# [5:7587241270018171605:0]
2025-12-24T02:11:08.310864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241270018171622:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.310924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.310948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.310987Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:08.311045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270595672838:2049] [node 2] ICS01 socket: 407 reason# EndOfStream
2025-12-24T02:11:08.311049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270595672757:2049] [node 2] ICP30 unregister session Session# [6:7587241270595672838:2049] VirtualId# [6:7587241270595672836:0]
2025-12-24T02:11:08.311054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270595672838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.311079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.311101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.313633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:08.313720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241268568698678:2049] [node 2] ICS01 socket: 437 reason# EndOfStream
2025-12-24T02:11:08.313727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241268568698590:2049] [node 2] ICP30 unregister session Session# [4:7587241268568698678:2049] VirtualId# [4:7587241268568698674:0]
2025-12-24T02:11:08.313733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241268568698678:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.313793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.313819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.333429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:08.333529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270595672843:2050] [node 3] ICS01 socket: 424 reason# EndOfStream
2025-12-24T02:11:08.333538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270595672758:2050] [node 3] ICP30 unregister session Session# [6:7587241270595672843:2050] VirtualId# [6:7587241270595672841:0]
2025-12-24T02:11:08.333545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270595672843:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.333613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.333640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.333739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:08.333808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241270018171624:2050] [node 3] ICS01 socket: 431 reason# EndOfStream
2025-12-24T02:11:08.333814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241270018171533:2050] [node 3] ICP30 unregister session Session# [5:7587241270018171624:2050] VirtualId# [5:7587241270018171612:0]
2025-12-24T02:11:08.333819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241270018171624:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.333850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.333875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.352750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:08.352850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270595672855:2108] [node 4] ICS01 socket: 457 reason# EndOfStream
2025-12-24T02:11:08.352859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270595672822:2108] [node 4] ICP30 unregister session Session# [6:7587241270595672855:2108] VirtualId# [6:7587241270595672852:0]
2025-12-24T02:11:08.352866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270595672855:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.352923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:08.352951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:08.360712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:08.360814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270595672859:2100] [node 5] ICS01 socket: 449 reason# EndOfStream
2025-12-24T02:11:08.360826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270595672811:2100] [node 5] ICP30 unregister session Session# [6:7587241270595672859:2100] VirtualId# [6:7587241270595672857:0]
2025-12-24T02:11:08.360834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270595672859:2100] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.360890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:08.360918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:08.392856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270039722937:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:08.392864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270039722938:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.398424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241270497838596:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.398448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241270497838597:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.398887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241272273059519:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.398903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241272273059520:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.411583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269633598409:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.411595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269633598410:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.411599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269633598411:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.386845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241271395943231:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.386855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241271395943232:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.386860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241271395943233:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.432285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241272191338310:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.432332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241272191338311:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.432337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241272191338312:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.433138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270039722998:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:08.438938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270039723004:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:08.439028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241270039723005:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:08.438947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241272273059581:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:08.439034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241272273059582:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:08.442946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241270497838658:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:08.443050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241270497838659:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:08.443014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241272191338368:2103] [node 5] ICP01 ready to work
2025-12-24T02:11:08.443104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241272191338369:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:08.459786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241270039723004:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20703561ns) {Configure:104(20702518ns) {ProcessConfigured:135(20277677ns) {StartInitialHandshake:157(20277236ns) {DropHandshakes:470(729ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(105ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:477(1376ns) {SwitchToState:179(817ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.468184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241272273059590:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:08.469160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059591:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.468799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241271395943231:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28936766ns) {Configure:104(28935283ns) {ProcessConfigured:135(28686877ns) {StartInitialHandshake:157(28686658ns) {DropHandshakes:470(353ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(817ns) {SwitchToState:179(656ns)}}}}}
2025-12-24T02:11:08.468994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241271395943296:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:08.469071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241271395943297:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:08.469921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943298:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.469936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943299:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.469950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943300:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.471626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269633598476:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:08.471714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269633598477:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:08.471759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598472:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.471820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598473:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.471830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598474:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.471840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598475:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.471186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270039723012:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:336
2025-12-24T02:11:08.471208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270039723013:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:336
2025-12-24T02:11:08.471220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270039723014:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:336
2025-12-24T02:11:08.506876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598472:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.531019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241270497838671:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.554258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241270497838670:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38930
2025-12-24T02:11:08.571089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269633598409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.571598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838670:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.571889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943310:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.571934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943311:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.572207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943299:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.572065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723000:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.572077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270039723000:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.572237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241271395943232:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.572392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943311:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.572414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241271395943297:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.577236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598483:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.577256Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241269633598483:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.577555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598475:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.578082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943316:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.578829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838663:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.578849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241270497838663:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.579911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723014:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.580203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270039723000:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.580307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943310:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.580295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270039722998:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.580309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.580316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270039722998:2107] [node 4] ICP22 created new session: [1:7587241270039723021:2107]
2025-12-24T02:11:08.580322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270039723021:2107] [node 4] ICS09 handshake done sender: [1:7587241270039723000:2108] self: [1:7587241270039722999:0] peer: [4:7587241269633598491:0] socket: 396 qp: -1
2025-12-24T02:11:08.580329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270039723021:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:08.580338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.582828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269633598483:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.583047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241269633598476:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.583069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.583076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269633598476:2108] [node 5] ICP22 created new session: [4:7587241269633598495:2108]
2025-12-24T02:11:08.583083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269633598495:2108] [node 5] ICS09 handshake done sender: [4:7587241269633598483:2116] self: [4:7587241269633598482:0] peer: [5:7587241271395943315:0] socket: 432 qp: -1
2025-12-24T02:11:08.583090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269633598495:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:08.583100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.583554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.583800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269633598472:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.583900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269633598409:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.583913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.583919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269633598409:2048] [node 1] ICP22 created new session: [4:7587241269633598498:2048]
2025-12-24T02:11:08.583924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269633598498:2048] [node 1] ICS09 handshake done sender: [4:7587241269633598472:2110] self: [4:7587241269633598491:0] peer: [1:7587241270039722999:0] socket: 398 qp: -1
2025-12-24T02:11:08.583931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269633598498:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.583936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.584703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241270497838663:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.585002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241270497838658:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.585013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.585019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241270497838658:2107] [node 5] ICP22 created new session: [2:7587241270497838679:2107]
2025-12-24T02:11:08.585025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241270497838679:2107] [node 5] ICS09 handshake done sender: [2:7587241270497838663:2110] self: [2:7587241270497838662:0] peer: [5:7587241271395943314:0] socket: 403 qp: -1
2025-12-24T02:11:08.585030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241270497838679:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:08.585039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.590733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943317:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.592644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241270497838678:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38930
2025-12-24T02:11:08.592814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241271395943299:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.594838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241271395943232:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.594871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.594880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241271395943232:2049] [node 2] ICP22 created new session: [5:7587241271395943318:2049]
2025-12-24T02:11:08.594887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241271395943318:2049] [node 2] ICS09 handshake done sender: [5:7587241271395943299:2112] self: [5:7587241271395943314:0] peer: [2:7587241270497838662:0] socket: 413 qp: -1
2025-12-24T02:11:08.594896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241271395943318:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.594906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.595463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241270039723024:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:336
2025-12-24T02:11:08.598955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241271395943231:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.599009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241271395943297:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24039) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241271395943311:2118] held: no
2025-12-24T02:11:08.599106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241271395943311:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.602345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723007:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.602359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270039723007:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.614340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270039723007:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.614488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270039723004:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.614512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.614521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270039723004:2109] [node 5] ICP22 created new session: [1:7587241270039723026:2109]
2025-12-24T02:11:08.614527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270039723026:2109] [node 5] ICS09 handshake done sender: [1:7587241270039723007:2111] self: [1:7587241270039723006:0] peer: [5:7587241271395943321:0] socket: 424 qp: -1
2025-12-24T02:11:08.614533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270039723026:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:08.614543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.624506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838678:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.624574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241270497838596:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.628012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338388:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.628060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338389:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.628091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.627704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241271395943231:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241271395943310:2117] held: no
2025-12-24T02:11:08.628193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338391:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.627824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241271395943297:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.627835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.627842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241271395943297:2109] [node 4] ICP22 created new session: [5:7587241271395943322:2109]
2025-12-24T02:11:08.627848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241271395943322:2109] [node 4] ICS09 handshake done sender: [5:7587241271395943311:2118] self: [5:7587241271395943315:0] peer: [4:7587241269633598482:0] socket: 434 qp: -1
2025-12-24T02:11:08.627856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241271395943322:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:08.627863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.628028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943298:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943324:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.628753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241272191338391:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241272191338369:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.628297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598474:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598473:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269633598411:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.627792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059599:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
2025-12-24T02:11:08.627852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059600:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
2025-12-24T02:11:08.627891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059601:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
2025-12-24T02:11:08.628608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059600:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059599:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059587:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.628650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241272273059587:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.629257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059591:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.629302Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241272273059587:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.629382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241272273059581:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.629394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.629401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241272273059581:2107] [node 4] ICP22 created new session: [3:7587241272273059605:2107]
2025-12-24T02:11:08.629407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241272273059605:2107] [node 4] ICS09 handshake done sender: [3:7587241272273059587:2111] self: [3:7587241272273059585:0] peer: [4:7587241269633598500:0] socket: 410 qp: -1
2025-12-24T02:11:08.629414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241272273059605:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:08.629421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.629740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241272273059520:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.631264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943300:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.631305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241271395943310:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.632596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598485:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.632608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241269633598485:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.632412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241272191338388:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.632436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241272191338389:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.632465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241272191338311:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.632476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241272191338368:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.632425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838661:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.632439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241270497838661:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.633358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838665:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.633361Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241270497838665:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.633883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241270497838683:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38930
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.635831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059609:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.637292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338395:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.637384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241270497838661:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.637543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241270497838665:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.637707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241272191338388:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.637794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838683:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.637806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241270497838597:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.637820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.637828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241270497838597:2049] [node 3] ICP22 created new session: [2:7587241270497838686:2049]
2025-12-24T02:11:08.637834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241270497838686:2049] [node 3] ICS09 handshake done sender: [2:7587241270497838661:2109] self: [2:7587241270497838660:0] peer: [3:7587241272273059607:0] socket: 402 qp: -1
2025-12-24T02:11:08.637840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241270497838686:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.637847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.637955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241270497838659:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.637960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.637964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241270497838659:2108] [node 6] ICP22 created new session: [2:7587241270497838688:2108]
2025-12-24T02:11:08.637968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241270497838688:2108] [node 6] ICS09 handshake done sender: [2:7587241270497838665:2111] self: [2:7587241270497838664:0] peer: [6:7587241272191338393:0] socket: 404 qp: -1
2025-12-24T02:11:08.637971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241270497838688:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:08.637975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.638064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241270497838597:2049] [node 3] ICP18 handshake (actor [2:7587241270497838683:2117]) is too old
2025-12-24T02:11:08.638265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241272191338311:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.638281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.638285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241272191338311:2049] [node 2] ICP22 created new session: [6:7587241272191338396:2049]
2025-12-24T02:11:08.638289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241272191338396:2049] [node 2] ICS09 handshake done sender: [6:7587241272191338388:2115] self: [6:7587241272191338393:0] peer: [2:7587241270497838664:0] socket: 406 qp: -1
2025-12-24T02:11:08.638294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241272191338396:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.638299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.638912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723011:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.638926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270039723011:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.641017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241269633598411:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241269633598474:2112] held: no
2025-12-24T02:11:08.641157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598502:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.642348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598503:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.642418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269633598485:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.642515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241269633598477:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.642533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.642538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269633598477:2109] [node 6] ICP22 created new session: [4:7587241269633598505:2109]
2025-12-24T02:11:08.642545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269633598505:2109] [node 6] ICS09 handshake done sender: [4:7587241269633598485:2117] self: [4:7587241269633598484:0] peer: [6:7587241272191338392:0] socket: 427 qp: -1
2025-12-24T02:11:08.642551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269633598505:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:08.642560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.642660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269633598474:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.642984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723013:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.643061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723012:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.643084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723024:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.643523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241271395943302:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.643540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241271395943302:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.643844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241271395943231:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.643857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.643865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241271395943231:2048] [node 1] ICP22 created new session: [5:7587241271395943326:2048]
2025-12-24T02:11:08.643871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241271395943326:2048] [node 1] ICS09 handshake done sender: [5:7587241271395943310:2117] self: [5:7587241271395943321:0] peer: [1:7587241270039723006:0] socket: 428 qp: -1
2025-12-24T02:11:08.643879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241271395943326:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.643888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.644011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241271395943233:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.646822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270039723011:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.647097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270039722937:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.647116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.647123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270039722937:2048] [node 2] ICP22 created new session: [1:7587241270039723033:2048]
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:08.647128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270039723033:2048] [node 2] ICS09 handshake done sender: [1:7587241270039723011:2113] self: [1:7587241270039723010:0] peer: [2:7587241270497838682:0] socket: 443 qp: -1
2025-12-24T02:11:08.647133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270039723033:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:08.647141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.647056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241272191338390:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.647094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241272191338310:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.647250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.647667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269633598411:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.647684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.647690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269633598411:2050] [node 3] ICP22 created new session: [4:7587241269633598507:2050]
2025-12-24T02:11:08.647695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269633598507:2050] [node 3] ICS09 handshake done sender: [4:7587241269633598474:2112] self: [4:7587241269633598500:0] peer: [3:7587241272273059585:0] socket: 426 qp: -1
2025-12-24T02:11:08.647700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269633598507:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.647707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.647817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269633598503:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.647857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269633598410:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.648078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723009:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.648087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270039723009:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.648691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241272191338391:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.648758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241272191338369:2104] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.648773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.648778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241272191338369:2104] [node 4] ICP22 created new session: [6:7587241272191338402:2104]
2025-12-24T02:11:08.648782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241272191338402:2104] [node 4] ICS09 handshake done sender: [6:7587241272191338391:2118] self: [6:7587241272191338392:0] peer: [4:7587241269633598484:0] socket: 451 qp: -1
2025-12-24T02:11:08.648788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241272191338402:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:08.648794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.648956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338401:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.649691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270039723009:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.649774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270039723005:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.649790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.649795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270039723005:2110] [node 6] ICP22 created new session: [1:7587241270039723036:2110]
2025-12-24T02:11:08.649800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270039723036:2110] [node 6] ICS09 handshake done sender: [1:7587241270039723009:2112] self: [1:7587241270039723008:0] peer: [6:7587241272191338399:0] socket: 425 qp: -1
2025-12-24T02:11:08.649804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270039723036:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.649810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.650225Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241272191338369:2104] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.650823Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241272191338310:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241272191338390:2117] held: no
2025-12-24T02:11:08.650863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338404:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.650978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241272191338390:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.650956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241271395943302:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.652454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241270497838691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38930
2025-12-24T02:11:08.652709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241270497838675:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.652718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241270497838675:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.653819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241271395943296:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.653843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.653851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241271395943296:2108] [node 6] ICP22 created new session: [5:7587241271395943330:2108]
2025-12-24T02:11:08.653858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241271395943330:2108] [node 6] ICS09 handshake done sender: [5:7587241271395943302:2114] self: [5:7587241271395943301:0] peer: [6:7587241272191338394:0] socket: 417 qp: -1
2025-12-24T02:11:08.653866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241271395943330:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:08.653875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.655426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241270497838596:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241270497838678:2116] held: no
2025-12-24T02:11:08.655469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241270497838678:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.655660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241270497838596:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.655673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.655678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241270497838596:2048] [node 1] ICP22 created new session: [2:7587241270497838693:2048]
2025-12-24T02:11:08.655684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241270497838693:2048] [node 1] ICS09 handshake done sender: [2:7587241270497838678:2116] self: [2:7587241270497838682:0] peer: [1:7587241270039723010:0] socket: 445 qp: -1
2025-12-24T02:11:08.655691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241270497838693:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.655697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.655950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241270497838675:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.656017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241270497838671:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.656030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.656034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241270497838671:2112] [node 4] ICP22 created new session: [2:7587241270497838695:2112]
2025-12-24T02:11:08.656038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241270497838695:2112] [node 4] ICS09 handshake done sender: [2:7587241270497838675:2115] self: [2:7587241270497838674:0] peer: [4:7587241269633598509:0] socket: 453 qp: -1
2025-12-24T02:11:08.656043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241270497838695:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:08.656049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.658879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241272273059520:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241272273059591:2113] held: no
2025-12-24T02:11:08.658936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059589:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.658941Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241272273059589:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.659477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241272273059591:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.660530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241272273059520:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.660545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.660550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241272273059520:2049] [node 2] ICP22 created new session: [3:7587241272273059612:2049]
2025-12-24T02:11:08.660556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241272273059612:2049] [node 2] ICS09 handshake done sender: [3:7587241272273059591:2113] self: [3:7587241272273059607:0] peer: [2:7587241270497838660:0] socket: 408 qp: -1
2025-12-24T02:11:08.660561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241272273059612:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.660567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.660808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059610:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.661441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059610:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.662681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241272273059589:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.659539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241272191338310:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.659557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.659565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241272191338310:2048] [node 1] ICP22 created new session: [6:7587241272191338405:2048]
2025-12-24T02:11:08.659572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241272191338405:2048] [node 1] ICS09 handshake done sender: [6:7587241272191338390:2117] self: [6:7587241272191338399:0] peer: [1:7587241270039723008:0] socket: 436 qp: -1
2025-12-24T02:11:08.659578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241272191338405:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.659588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.660428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241272191338389:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.661057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241272191338368:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16626) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241272191338389:2116] held: no
2025-12-24T02:11:08.661074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241272191338368:2103] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.661081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.661086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241272191338368:2103] [node 5] ICP22 created new session: [6:7587241272191338407:2103]
2025-12-24T02:11:08.661092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241272191338407:2103] [node 5] ICS09 handshake done sender: [6:7587241272191338389:2116] self: [6:7587241272191338394:0] peer: [5:7587241271395943301:0] socket: 422 qp: -1
2025-12-24T02:11:08.661098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241272191338407:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:08.661104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.661359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338409:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.662912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241269633598410:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241269633598503:2121] held: no
2025-12-24T02:11:08.662994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269633598510:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59229
2025-12-24T02:11:08.662968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241272273059582:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.662986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.662991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241272273059582:2108] [node 5] ICP22 created new session: [3:7587241272273059616:2108]
2025-12-24T02:11:08.662996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241272273059616:2108] [node 5] ICS09 handshake done sender: [3:7587241272273059589:2112] self: [3:7587241272273059588:0] peer: [5:7587241271395943328:0] socket: 411 qp: -1
2025-12-24T02:11:08.663002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241272273059616:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:08.663009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.663142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269633598503:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.663196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269633598410:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.663203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.663208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269633598410:2049] [node 2] ICP22 created new session: [4:7587241269633598511:2049]
2025-12-24T02:11:08.663213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269633598511:2049] [node 2] ICS09 handshake done sender: [4:7587241269633598503:2121] self: [4:7587241269633598509:0] peer: [2:7587241270497838674:0] socket: 460 qp: -1
2025-12-24T02:11:08.663218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269633598511:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.663225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.666089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241271395943332:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62016
2025-12-24T02:11:08.666292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241271395943300:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.666333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241271395943233:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.666343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.666349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241271395943233:2050] [node 3] ICP22 created new session: [5:7587241271395943333:2050]
2025-12-24T02:11:08.666353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241271395943333:2050] [node 3] ICS09 handshake done sender: [5:7587241271395943300:2113] self: [5:7587241271395943328:0] peer: [3:7587241272273059588:0] socket: 414 qp: -1
2025-12-24T02:11:08.666359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241271395943333:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.666367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.670857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059601:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.671034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241272273059519:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.671248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241272191338409:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.671292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241272191338312:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.671614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241270039723016:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.671626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241270039723016:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.672362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241272273059595:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.672372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241272273059595:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.672846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241272273059619:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18475
2025-12-24T02:11:08.673002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241272273059595:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.673074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241270039723016:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.673192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241272191338411:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12865
2025-12-24T02:11:08.673350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241272191338409:2123] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.673228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241270039722938:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.673281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.673288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241270039722938:2049] [node 3] ICP22 created new session: [1:7587241270039723039:2049]
2025-12-24T02:11:08.673295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241270039723039:2049] [node 3] ICS09 handshake done sender: [1:7587241270039723016:2117] self: [1:7587241270039723015:0] peer: [3:7587241272273059618:0] socket: 421 qp: -1
2025-12-24T02:11:08.673301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241270039723039:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.673312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.673406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241272191338312:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.673420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.673425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241272191338312:2050] [node 3] ICP22 created new session: [6:7587241272191338412:2050]
2025-12-24T02:11:08.673431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241272191338412:2050] [node 3] ICS09 handshake done sender: [6:7587241272191338409:2123] self: [6:7587241272191338410:0] peer: [3:7587241272273059594:0] socket: 437 qp: -1
2025-12-24T02:11:08.673437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241272191338412:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.673444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.674303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241272273059601:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.674361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241272273059519:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.674375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.674380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241272273059519:2048] [node 1] ICP22 created new session: [3:7587241272273059621:2048]
2025-12-24T02:11:08.674386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241272273059621:2048] [node 1] ICS09 handshake done sender: [3:7587241272273059601:2118] self: [3:7587241272273059618:0] peer: [1:7587241270039723015:0] socket: 449 qp: -1
2025-12-24T02:11:08.674392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241272273059621:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.674397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.674544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241272273059590:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.674552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.674555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241272273059590:2110] [node 6] ICP22 created new session: [3:7587241272273059623:2110]
2025-12-24T02:11:08.674559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241272273059623:2110] [node 6] ICS09 handshake done sender: [3:7587241272273059595:2115] self: [3:7587241272273059594:0] peer: [6:7587241272191338410:0] socket: 435 qp: -1
2025-12-24T02:11:08.674562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241272273059623:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.674566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.677605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241272273059519:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:08.686773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:08.686888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241272273059621:2048] [node 1] ICS01 socket: 449 reason# EndOfStream
2025-12-24T02:11:08.686898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241272273059519:2048] [node 1] ICP30 unregister session Session# [3:7587241272273059621:2048] VirtualId# [3:7587241272273059618:0]
2025-12-24T02:11:08.686904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241272273059621:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.686957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.686981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.686787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:08.686883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241270497838693:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:08.686892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241270497838596:2048] [node 1] ICP30 unregister session Session# [2:7587241270497838693:2048] VirtualId# [2:7587241270497838682:0]
2025-12-24T02:11:08.686899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241270497838693:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.686973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.686998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.691096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:08.691147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241272191338405:2048] [node 1] ICS01 socket: 436 reason# EndOfStream
2025-12-24T02:11:08.691152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241272191338310:2048] [node 1] ICP30 unregister session Session# [6:7587241272191338405:2048] VirtualId# [6:7587241272191338399:0]
2025-12-24T02:11:08.691155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241272191338405:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.691181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.691198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.691699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:08.691779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241269633598498:2048] [node 1] ICS01 socket: 398 reason# EndOfStream
2025-12-24T02:11:08.691787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241269633598409:2048] [node 1] ICP30 unregister session Session# [4:7587241269633598498:2048] VirtualId# [4:7587241269633598491:0]
2025-12-24T02:11:08.691793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241269633598498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.691845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.691868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.690871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:08.690952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241271395943326:2048] [node 1] ICS01 socket: 428 reason# EndOfStream
2025-12-24T02:11:08.690961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241271395943231:2048] [node 1] ICP30 unregister session Session# [5:7587241271395943326:2048] VirtualId# [5:7587241271395943321:0]
2025-12-24T02:11:08.690967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241271395943326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.691011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.691051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.700588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:08.700668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241272273059612:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-12-24T02:11:08.700676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241272273059520:2049] [node 2] ICP30 unregister session Session# [3:7587241272273059612:2049] VirtualId# [3:7587241272273059607:0]
2025-12-24T02:11:08.700682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241272273059612:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.700714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.700732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.714453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:08.714554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241269633598511:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-12-24T02:11:08.714564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241269633598410:2049] [node 2] ICP30 unregister session Session# [4:7587241269633598511:2049] VirtualId# [4:7587241269633598509:0]
2025-12-24T02:11:08.714570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241269633598511:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.714629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.714653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.717521Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:08.717607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241271395943318:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-12-24T02:11:08.717615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241271395943232:2049] [node 2] ICP30 unregister session Session# [5:7587241271395943318:2049] VirtualId# [5:7587241271395943314:0]
2025-12-24T02:11:08.717623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241271395943318:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.717675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.717697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.722759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:08.722860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241272191338396:2049] [node 2] ICS01 socket: 406 reason# EndOfStream
2025-12-24T02:11:08.722868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241272191338311:2049] [node 2] ICP30 unregister session Session# [6:7587241272191338396:2049] VirtualId# [6:7587241272191338393:0]
2025-12-24T02:11:08.722874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241272191338396:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.722927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.722951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.730766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:08.730986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:08.731035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241272191338412:2050] [node 3] ICS01 socket: 437 reason# EndOfStream
2025-12-24T02:11:08.731039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241272191338312:2050] [node 3] ICP30 unregister session Session# [6:7587241272191338412:2050] VirtualId# [6:7587241272191338410:0]
2025-12-24T02:11:08.731043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241272191338412:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.731064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.731083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.730859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241271395943333:2050] [node 3] ICS01 socket: 414 reason# EndOfStream
2025-12-24T02:11:08.730871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241271395943233:2050] [node 3] ICP30 unregister session Session# [5:7587241271395943333:2050] VirtualId# [5:7587241271395943328:0]
2025-12-24T02:11:08.730876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241271395943333:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.730920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.730941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.738777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:08.738855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241272191338402:2104] [node 4] ICS01 socket: 451 reason# EndOfStream
2025-12-24T02:11:08.738862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241272191338369:2104] [node 4] ICP30 unregister session Session# [6:7587241272191338402:2104] VirtualId# [6:7587241272191338392:0]
2025-12-24T02:11:08.738869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241272191338402:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.738933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:08.738955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:08.782562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268864389831:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:08.782572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268864389832:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.787782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271812366065:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.787811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271812366066:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.787789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268213109275:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.787816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268213109276:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.787822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268213109277:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.787731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241271148492613:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.787777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241271148492614:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.789945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268254068285:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.789960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268254068286:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.789973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268254068284:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.793102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270801821214:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.793120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270801821215:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.793129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270801821216:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.804647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268213109309:2080] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.805514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241268213109315:2083] [node 5] ICP01 ready to work
2025-12-24T02:11:08.805953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109318:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.804206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270801821239:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:08.804267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268254068302:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:08.804330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241270801821242:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:08.805571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821254:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.804376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268254068305:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:08.806042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821260:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.805786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068317:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.806410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068318:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.808431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268864389895:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:08.820706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271812366131:2100] [node 4] ICP01 ready to work
2025-12-24T02:11:08.820803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271812366132:2103] [node 5] ICP01 ready to work
2025-12-24T02:11:08.820789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109311:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.820810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241268213109311:2081] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.820879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241271812366133:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:08.820564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270801821254:2081] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.820611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270801821242:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.821506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068318:2072] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.822598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268254068302:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.822656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068317:2071] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.822803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109318:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.822846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109317:2084] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.822850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241268213109317:2084] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.822867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270801821260:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.823006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270801821239:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.821450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241271148492678:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:08.821560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241271148492679:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:08.821654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241271148492680:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:08.828528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068311:2069] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.828559Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241268254068311:2069] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.829038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241268864389895:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20498745ns) {Configure:104(20497869ns) {ProcessConfigured:135(20130457ns) {StartInitialHandshake:157(20129783ns) {DropHandshakes:470(875ns) {DropIncomingHandshake:442(157ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(1698ns) {SwitchToState:179(831ns)}}}}}
2025-12-24T02:11:08.829261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268864389901:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:08.829266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241268254068285:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22170276ns) {Configure:104(22169400ns) {ProcessConfigured:135(21672874ns) {StartInitialHandshake:157(21672048ns) {DropHandshakes:470(467ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1945ns) {SwitchToState:179(413ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.829370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241268864389902:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:08.829672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.829724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268213109317:2084] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.829743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268213109311:2081] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.829812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241271812366065:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19223429ns) {Configure:104(19222133ns) {ProcessConfigured:135(18796199ns) {StartInitialHandshake:157(18795540ns) {DropHandshakes:470(423ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(388ns) PrepareNewSessionHandshake:477(1267ns) {SwitchToState:179(359ns)}}}}}
2025-12-24T02:11:08.829878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241268213109309:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.829899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.829908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268213109309:2080] [node 6] ICP22 created new session: [4:7587241268213109358:2080]
2025-12-24T02:11:08.829914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268213109358:2080] [node 6] ICS09 handshake done sender: [4:7587241268213109311:2081] self: [4:7587241268213109310:0] peer: [6:7587241270801821294:0] socket: 404 qp: -1
2025-12-24T02:11:08.829924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268213109358:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:08.829935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.830093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241268213109315:2083] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.830108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.830111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268213109315:2083] [node 5] ICP22 created new session: [4:7587241268213109360:2083]
2025-12-24T02:11:08.830115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268213109360:2083] [node 5] ICS09 handshake done sender: [4:7587241268213109317:2084] self: [4:7587241268213109316:0] peer: [5:7587241268254068363:0] socket: 417 qp: -1
2025-12-24T02:11:08.830118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268213109360:2083] [node 5] ICS10 traffic start
2025-12-24T02:11:08.830122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.832534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241268864389832:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18225397ns) {Configure:104(18224027ns) {ProcessConfigured:135(17650959ns) {StartInitialHandshake:157(17649917ns) {DropHandshakes:470(1211ns) {DropIncomingHandshake:442(213ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(514ns) PrepareNewSessionHandshake:477(1181ns) {SwitchToState:179(887ns)}}}}}
No RDMA context found for GID: 2025-12-24T02:11:08.834351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366129:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:08.834362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492692:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.835104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821296:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.835145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366130:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.835109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492693:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.835126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268864389910:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-12-24T02:11:08.835264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.835375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109355:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.835420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268213109276:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.835512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268254068302:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23685) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241268254068318:2072] held: no
2025-12-24T02:11:08.835564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492684:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.835560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389910:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.835575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241271148492684:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.835643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268864389912:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-12-24T02:11:08.835710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268254068318:2072] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.835791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270801821254:2081] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.835815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268254068302:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.835832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.835840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268254068302:2060] [node 4] ICP22 created new session: [5:7587241268254068369:2060]
2025-12-24T02:11:08.835850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268254068369:2060] [node 4] ICS09 handshake done sender: [5:7587241268254068318:2072] self: [5:7587241268254068363:0] peer: [4:7587241268213109316:0] socket: 419 qp: -1
2025-12-24T02:11:08.835858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268254068369:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:08.835871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.836205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366134:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
2025-12-24T02:11:08.836207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366136:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
2025-12-24T02:11:08.836208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268864389911:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-12-24T02:11:08.836281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492692:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270801821242:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.836317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.836322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270801821242:2069] [node 4] ICP22 created new session: [6:7587241270801821299:2069]
2025-12-24T02:11:08.836329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270801821299:2069] [node 4] ICS09 handshake done sender: [6:7587241270801821254:2081] self: [6:7587241270801821294:0] peer: [4:7587241268213109310:0] socket: 405 qp: -1
2025-12-24T02:11:08.836323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492693:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366129:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270801821299:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:08.836331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366130:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.836351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.836334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268254068311:2069] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.836440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241271148492613:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.836586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366134:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366136:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241271812366066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.836515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389911:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389891:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.836626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268864389891:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.837349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268864389891:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.837941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366145:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
2025-12-24T02:11:08.837951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821297:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.837983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492694:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
2025-12-24T02:11:08.838076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492682:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.838087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241271148492682:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.838190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366145:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.838220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241271812366065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.838727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241271148492684:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.838794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241268254068305:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.838838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.838857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268254068305:2064] [node 6] ICP22 created new session: [5:7587241268254068372:2064]
2025-12-24T02:11:08.838865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268254068372:2064] [node 6] ICS09 handshake done sender: [5:7587241268254068311:2069] self: [5:7587241268254068310:0] peer: [6:7587241270801821295:0] socket: 412 qp: -1
2025-12-24T02:11:08.838882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268254068372:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:08.838907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.838879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268864389831:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.838899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.838904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268864389831:2048] [node 2] ICP22 created new session: [1:7587241268864389918:2048]
2025-12-24T02:11:08.838908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268864389918:2048] [node 2] ICS09 handshake done sender: [1:7587241268864389891:2106] self: [1:7587241268864389890:0] peer: [2:7587241271148492695:0] socket: 402 qp: -1
2025-12-24T02:11:08.838913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268864389918:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:08.838929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.839096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389903:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.839111Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268864389903:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.839176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821298:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.839260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241271148492678:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:08.839290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.839298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241271148492678:2107] [node 4] ICP22 created new session: [2:7587241271148492699:2107]
2025-12-24T02:11:08.839305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241271148492699:2107] [node 4] ICS09 handshake done sender: [2:7587241271148492684:2112] self: [2:7587241271148492683:0] peer: [4:7587241268213109364:0] socket: 427 qp: -1
2025-12-24T02:11:08.839314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241271148492699:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:08.839322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.839767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268864389916:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-12-24T02:11:08.839758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109365:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.839760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109366:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.839855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268213109355:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.839870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241270801821239:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5597) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241270801821260:2083] held: no
2025-12-24T02:11:08.839946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270801821297:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.839968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270801821260:2083] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.839984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268213109276:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.840000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.840006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268213109276:2049] [node 2] ICP22 created new session: [4:7587241268213109367:2049]
2025-12-24T02:11:08.840017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268213109367:2049] [node 2] ICS09 handshake done sender: [4:7587241268213109355:2116] self: [4:7587241268213109364:0] peer: [2:7587241271148492683:0] socket: 428 qp: -1
2025-12-24T02:11:08.840024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268213109367:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.840035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.840136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389912:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.840227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268864389903:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.840564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268864389832:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.840581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.840587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268864389832:2049] [node 3] ICP22 created new session: [1:7587241268864389922:2049]
2025-12-24T02:11:08.840593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268864389922:2049] [node 3] ICS09 handshake done sender: [1:7587241268864389903:2112] self: [1:7587241268864389899:0] peer: [3:7587241271812366153:0] socket: 440 qp: -1
2025-12-24T02:11:08.840599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268864389922:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.840606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.840622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492694:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.839760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068371:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.841028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270801821239:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.841044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.841051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270801821239:2064] [node 5] ICP22 created new session: [6:7587241270801821302:2064]
2025-12-24T02:11:08.841057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270801821302:2064] [node 5] ICS09 handshake done sender: [6:7587241270801821260:2083] self: [6:7587241270801821295:0] peer: [5:7587241268254068310:0] socket: 416 qp: -1
2025-12-24T02:11:08.841063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270801821302:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:08.841070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.841260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241271148492682:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.841269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270801821214:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.842089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492700:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:08.842088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068375:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.842128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241268864389924:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21364
2025-12-24T02:11:08.842140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241271148492613:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241271148492692:2115] held: no
2025-12-24T02:11:08.842178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241271148492614:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.842189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492702:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
2025-12-24T02:11:08.842191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.842196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241271148492614:2049] [node 3] ICP22 created new session: [2:7587241271148492705:2049]
2025-12-24T02:11:08.842202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241271148492705:2049] [node 3] ICS09 handshake done sender: [2:7587241271148492682:2111] self: [2:7587241271148492681:0] peer: [3:7587241271812366152:0] socket: 426 qp: -1
2025-12-24T02:11:08.842207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241271148492705:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.842212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.842302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366154:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.842508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241271148492678:2107] [node 4] ICP18 handshake (actor [2:7587241271148492694:2117]) is too old
2025-12-24T02:11:08.842557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241271148492703:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3944
2025-12-24T02:11:08.842563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068376:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.842571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389900:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.842585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268864389900:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.843253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241271812366155:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7446
2025-12-24T02:11:08.843306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109366:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.843366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268213109275:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.843411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389924:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.843627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389905:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.843637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268864389905:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.843796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241271148492692:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.843908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492703:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.843925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271812366145:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.843938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241271148492613:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.843953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.843964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241271148492613:2048] [node 1] ICP22 created new session: [2:7587241271148492708:2048]
2025-12-24T02:11:08.843969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241271148492708:2048] [node 1] ICS09 handshake done sender: [2:7587241271148492692:2115] self: [2:7587241271148492695:0] peer: [1:7587241268864389890:0] socket: 420 qp: -1
2025-12-24T02:11:08.843976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241271148492708:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.843983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.844034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271812366136:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.844093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268864389900:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.844367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821305:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.844424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.844242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268864389895:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.844254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.844262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268864389895:2108] [node 6] ICP22 created new session: [1:7587241268864389927:2108]
2025-12-24T02:11:08.844267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268864389927:2108] [node 6] ICS09 handshake done sender: [1:7587241268864389900:2111] self: [1:7587241268864389898:0] peer: [6:7587241270801821304:0] socket: 439 qp: -1
2025-12-24T02:11:08.844273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268864389927:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:08.844281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.844343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241268864389895:2108] [node 6] ICP18 handshake (actor [1:7587241268864389924:2119]) is too old
2025-12-24T02:11:08.844572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270801821297:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.844556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068375:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.844595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268254068285:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.844896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268864389905:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.844936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109370:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.844977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068376:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.845017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268254068284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.845023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268213109366:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.845044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492686:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.845058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241271148492686:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.845042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268864389901:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.845061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.845067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268864389901:2109] [node 4] ICP22 created new session: [1:7587241268864389930:2109]
2025-12-24T02:11:08.845072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268864389930:2109] [node 4] ICS09 handshake done sender: [1:7587241268864389905:2113] self: [1:7587241268864389904:0] peer: [4:7587241268213109369:0] socket: 457 qp: -1
2025-12-24T02:11:08.845123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268213109275:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.845077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268864389930:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:08.845084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.845147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.845154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268213109275:2048] [node 1] ICP22 created new session: [4:7587241268213109371:2048]
2025-12-24T02:11:08.845160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268213109371:2048] [node 1] ICS09 handshake done sender: [4:7587241268213109366:2118] self: [4:7587241268213109369:0] peer: [1:7587241268864389904:0] socket: 458 qp: -1
2025-12-24T02:11:08.845166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268213109371:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.845163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241271812366065:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.845174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.845186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.845203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271812366065:2048] [node 1] ICP22 created new session: [3:7587241271812366156:2048]
2025-12-24T02:11:08.845214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271812366156:2048] [node 1] ICS09 handshake done sender: [3:7587241271812366145:2119] self: [3:7587241271812366153:0] peer: [1:7587241268864389899:0] socket: 443 qp: -1
2025-12-24T02:11:08.845225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271812366156:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.845259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.845792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270801821214:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.845809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.845815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270801821214:2048] [node 1] ICP22 created new session: [6:7587241270801821307:2048]
2025-12-24T02:11:08.845821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270801821307:2048] [node 1] ICS09 handshake done sender: [6:7587241270801821297:2118] self: [6:7587241270801821304:0] peer: [1:7587241268864389898:0] socket: 442 qp: -1
2025-12-24T02:11:08.845827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270801821307:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.845883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241271148492686:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.845833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:08.846295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241271812366066:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.846310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.846316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271812366066:2049] [node 2] ICP22 created new session: [3:7587241271812366158:2049]
2025-12-24T02:11:08.846321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271812366158:2049] [node 2] ICS09 handshake done sender: [3:7587241271812366136:2114] self: [3:7587241271812366152:0] peer: [2:7587241271148492681:0] socket: 436 qp: -1
2025-12-24T02:11:08.846326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271812366158:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.846332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.846442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241271148492679:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.846465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.846472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241271148492679:2108] [node 5] ICP22 created new session: [2:7587241271148492712:2108]
2025-12-24T02:11:08.846478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241271148492712:2108] [node 5] ICS09 handshake done sender: [2:7587241271148492686:2113] self: [2:7587241271148492685:0] peer: [5:7587241268254068377:0] socket: 463 qp: -1
2025-12-24T02:11:08.846450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241268864389907:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.846460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241268864389907:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.846483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241271148492712:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:08.846495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.848363Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241270801821214:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:08.848730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270801821306:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.848761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270801821215:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.848886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268254068285:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26639) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241268254068375:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.849121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068379:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.849445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241268864389907:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.850413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241271148492688:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.850428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241271148492688:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.850561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241268864389902:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.850590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.850600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241268864389902:2110] [node 5] ICP22 created new session: [1:7587241268864389933:2110]
2025-12-24T02:11:08.850606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241268864389933:2110] [node 5] ICS09 handshake done sender: [1:7587241268864389907:2114] self: [1:7587241268864389906:0] peer: [5:7587241268254068378:0] socket: 465 qp: -1
2025-12-24T02:11:08.850612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241268864389933:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:08.850619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.850599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268254068375:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.851774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241271148492688:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.851879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241271148492680:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.851897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.851925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241271148492680:2109] [node 6] ICP22 created new session: [2:7587241271148492715:2109]
2025-12-24T02:11:08.851935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241271148492715:2109] [node 6] ICS09 handshake done sender: [2:7587241271148492688:2114] self: [2:7587241271148492687:0] peer: [6:7587241270801821309:0] socket: 468 qp: -1
2025-12-24T02:11:08.851940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241271148492715:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:08.851951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.852824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268254068285:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.852843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.852851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268254068285:2049] [node 2] ICP22 created new session: [5:7587241268254068381:2049]
2025-12-24T02:11:08.852858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268254068381:2049] [node 2] ICS09 handshake done sender: [5:7587241268254068375:2119] self: [5:7587241268254068377:0] peer: [2:7587241271148492685:0] socket: 464 qp: -1
2025-12-24T02:11:08.852864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268254068381:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.852873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.852950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241270801821215:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26639) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241270801821306:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.853111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821310:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.853102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109373:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.853239Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268254068284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241268254068376:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.853828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068380:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.853898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241268213109373:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.853910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241270801821310:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.853940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241270801821216:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.853941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241268213109277:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.854191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821311:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.854203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366144:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.854201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366140:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.854273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270801821306:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.854212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271812366140:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.854216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271812366144:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.854839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271812366140:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.855116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271812366131:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.855144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.855156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271812366131:2100] [node 4] ICP22 created new session: [3:7587241271812366163:2100]
2025-12-24T02:11:08.855167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271812366163:2100] [node 4] ICS09 handshake done sender: [3:7587241271812366140:2116] self: [3:7587241271812366139:0] peer: [4:7587241268213109374:0] socket: 437 qp: -1
2025-12-24T02:11:08.855180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271812366163:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:08.855170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268254068376:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.855189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.855188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270801821215:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.855205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.855211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270801821215:2049] [node 2] ICP22 created new session: [6:7587241270801821313:2049]
2025-12-24T02:11:08.855217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270801821313:2049] [node 2] ICS09 handshake done sender: [6:7587241270801821306:2121] self: [6:7587241270801821309:0] peer: [2:7587241271148492687:0] socket: 469 qp: -1
2025-12-24T02:11:08.855223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270801821313:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.855236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.855233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268254068284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.855244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.855248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268254068284:2048] [node 1] ICP22 created new session: [5:7587241268254068383:2048]
2025-12-24T02:11:08.855253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268254068383:2048] [node 1] ICS09 handshake done sender: [5:7587241268254068376:2120] self: [5:7587241268254068378:0] peer: [1:7587241268864389906:0] socket: 467 qp: -1
2025-12-24T02:11:08.855257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268254068383:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.855268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.855887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241268213109375:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34140
2025-12-24T02:11:08.856081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241268213109277:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5661) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241268213109373:2120] held: no
2025-12-24T02:11:08.856075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268254068371:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.856119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241268213109373:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.856136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268254068286:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.856198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271812366144:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.856546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241268213109277:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.856566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.856577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241268213109277:2050] [node 3] ICP22 created new session: [4:7587241268213109376:2050]
2025-12-24T02:11:08.856575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241271812366142:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.856588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241271812366142:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.856588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241268213109376:2050] [node 3] ICS09 handshake done sender: [4:7587241268213109373:2120] self: [4:7587241268213109374:0] peer: [3:7587241271812366139:0] socket: 448 qp: -1
2025-12-24T02:11:08.856602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241268213109376:2050] [node 3] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:08.856622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.856625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271812366133:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.856652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.856659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271812366133:2106] [node 6] ICP22 created new session: [3:7587241271812366166:2106]
2025-12-24T02:11:08.856666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271812366166:2106] [node 6] ICS09 handshake done sender: [3:7587241271812366144:2118] self: [3:7587241271812366143:0] peer: [6:7587241270801821312:0] socket: 447 qp: -1
2025-12-24T02:11:08.856672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271812366166:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:08.856683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.857083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241270801821315:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17521
2025-12-24T02:11:08.857188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241270801821310:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.857622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241270801821216:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.857650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.857664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241270801821216:2050] [node 3] ICP22 created new session: [6:7587241270801821316:2050]
2025-12-24T02:11:08.857684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241270801821316:2050] [node 3] ICS09 handshake done sender: [6:7587241270801821310:2122] self: [6:7587241270801821312:0] peer: [3:7587241271812366143:0] socket: 450 qp: -1
2025-12-24T02:11:08.857708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241270801821316:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.857722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.857840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241271812366142:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.858406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241271812366132:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.858436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.858442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241271812366132:2103] [node 5] ICP22 created new session: [3:7587241271812366169:2103]
2025-12-24T02:11:08.858448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241271812366169:2103] [node 5] ICS09 handshake done sender: [3:7587241271812366142:2117] self: [3:7587241271812366141:0] peer: [5:7587241268254068385:0] socket: 449 qp: -1
2025-12-24T02:11:08.858454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241271812366169:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:08.858472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.858560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268254068386:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56597
2025-12-24T02:11:08.858656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268254068371:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.858913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268254068286:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.858951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.858960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268254068286:2050] [node 3] ICP22 created new session: [5:7587241268254068387:2050]
2025-12-24T02:11:08.858966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268254068387:2050] [node 3] ICS09 handshake done sender: [5:7587241268254068371:2118] self: [5:7587241268254068385:0] peer: [3:7587241271812366141:0] socket: 456 qp: -1
2025-12-24T02:11:08.858980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268254068387:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.858990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.859770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241268254068286:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:08.863294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:08.863393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241271148492708:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-12-24T02:11:08.863407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241271148492613:2048] [node 1] ICP30 unregister session Session# [2:7587241271148492708:2048] VirtualId# [2:7587241271148492695:0]
2025-12-24T02:11:08.863418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241271148492708:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.863474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.863513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.863570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:08.863647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241271812366156:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-12-24T02:11:08.863657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241271812366065:2048] [node 1] ICP30 unregister session Session# [3:7587241271812366156:2048] VirtualId# [3:7587241271812366153:0]
2025-12-24T02:11:08.863662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241271812366156:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.863706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.863733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.864785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:08.864868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241270801821307:2048] [node 1] ICS07 socket disconnect 442 reason# EndOfStream
2025-12-24T02:11:08.864879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270801821307:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.864923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.864960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241270801821307:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:08.864985Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:08.865065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241268213109371:2048] [node 1] ICS07 socket disconnect 458 reason# EndOfStream
2025-12-24T02:11:08.865075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241268213109371:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.865123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.865159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241268213109371:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:08.865158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:08.865251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268254068383:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-12-24T02:11:08.865263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268254068284:2048] [node 1] ICP30 unregister session Session# [5:7587241268254068383:2048] VirtualId# [5:7587241268254068378:0]
2025-12-24T02:11:08.865269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268254068383:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.865312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241270801821214:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:08.865325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270801821307:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:08.865319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:08.865330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270801821214:2048] [node 1] ICP30 unregister session Session# [6:7587241270801821307:2048] VirtualId# [6:7587241270801821304:0]
2025-12-24T02:11:08.865357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.865353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.865374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241270801821214:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.865377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.865537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241268213109275:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:08.865551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241268213109371:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:08.865557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241268213109275:2048] [node 1] ICP30 unregister session Session# [4:7587241268213109371:2048] VirtualId# [4:7587241268213109369:0]
2025-12-24T02:11:08.865579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:08.865590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241268213109275:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.865593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.872469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:08.872564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241271812366158:2049] [node 2] ICS01 socket: 436 reason# EndOfStream
2025-12-24T02:11:08.872576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241271812366066:2049] [node 2] ICP30 unregister session Session# [3:7587241271812366158:2049] VirtualId# [3:7587241271812366152:0]
2025-12-24T02:11:08.872584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241271812366158:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.872632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.872672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.873471Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:08.873567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241268213109367:2049] [node 2] ICS01 socket: 428 reason# EndOfStream
2025-12-24T02:11:08.873582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241268213109276:2049] [node 2] ICP30 unregister session Session# [4:7587241268213109367:2049] VirtualId# [4:7587241268213109364:0]
2025-12-24T02:11:08.873593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241268213109367:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.873654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.873683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.873605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:08.873700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268254068381:2049] [node 2] ICS01 socket: 464 reason# EndOfStream
2025-12-24T02:11:08.873706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268254068285:2049] [node 2] ICP30 unregister session Session# [5:7587241268254068381:2049] VirtualId# [5:7587241268254068377:0]
2025-12-24T02:11:08.873712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268254068381:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.873762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.873796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.873796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:08.873874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270801821313:2049] [node 2] ICS01 socket: 469 reason# EndOfStream
2025-12-24T02:11:08.873888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270801821215:2049] [node 2] ICP30 unregister session Session# [6:7587241270801821313:2049] VirtualId# [6:7587241270801821309:0]
2025-12-24T02:11:08.873901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270801821313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.873952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:08.873985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:08.875369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241270801821214:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:08.875378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241268213109275:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:08.884501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:08.884605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241268213109376:2050] [node 3] ICS01 socket: 448 reason# EndOfStream
2025-12-24T02:11:08.884623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241268213109277:2050] [node 3] ICP30 unregister session Session# [4:7587241268213109376:2050] VirtualId# [4:7587241268213109374:0]
2025-12-24T02:11:08.884632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241268213109376:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.884724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.884764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.884848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:08.884921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270801821316:2050] [node 3] ICS01 socket: 450 reason# EndOfStream
2025-12-24T02:11:08.884933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270801821216:2050] [node 3] ICP30 unregister session Session# [6:7587241270801821316:2050] VirtualId# [6:7587241270801821312:0]
2025-12-24T02:11:08.884939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270801821316:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.884970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.884998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.884701Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:08.884788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241268254068387:2050] [node 3] ICS07 socket disconnect 456 reason# EndOfStream
2025-12-24T02:11:08.884795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268254068387:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.884858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:08.884890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241268254068387:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:08.885206Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268254068286:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5661) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:08.885218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268254068387:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:08.885223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268254068286:2050] [node 3] ICP30 unregister session Session# [5:7587241268254068387:2050] VirtualId# [5:7587241268254068385:0]
2025-12-24T02:11:08.885246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:08.885253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241268254068286:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5661) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.885255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5661) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.895192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241268254068286:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:08.895504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:08.895595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241270801821299:2069] [node 4] ICS07 socket disconnect 405 reason# EndOfStream
2025-12-24T02:11:08.895607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270801821299:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.895662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:08.895687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241270801821299:2069] [node 4] ICS15 start handshake
2025-12-24T02:11:08.895707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:08.895836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268254068369:2060] [node 4] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:08.895853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268254068302:2060] [node 4] ICP30 unregister session Session# [5:7587241268254068369:2060] VirtualId# [5:7587241268254068363:0]
2025-12-24T02:11:08.895862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268254068369:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.895984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241270801821242:2069] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23685) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:08.895968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:08.896010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:08.895996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270801821299:2069] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:08.896001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270801821242:2069] [node 4] ICP30 unregister session Session# [6:7587241270801821299:2069] VirtualId# [6:7587241270801821294:0]
2025-12-24T02:11:08.896017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:08.896023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241270801821242:2069] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23685) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.896025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23685) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:08.900907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:08.901005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241270801821302:2064] [node 5] ICS01 socket: 416 reason# EndOfStream
2025-12-24T02:11:08.901019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241270801821239:2064] [node 5] ICP30 unregister session Session# [6:7587241270801821302:2064] VirtualId# [6:7587241270801821295:0]
2025-12-24T02:11:08.901027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241270801821302:2064] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:08.901089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:08.901118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:08.916740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241268073940718:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.916752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241268073940719:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.915387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272014288496:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:08.915394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272014288497:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:08.920416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269519452474:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.920429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269519452476:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.920441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269519452475:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.922140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268356098089:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.922150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268356098091:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.922180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268356098090:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.923874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241268851867337:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.923881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241268851867338:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.923915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241268851867339:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:08.918606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241270520912568:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:08.918612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241270520912569:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:08.936728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241268851867395:2104] [node 5] ICP01 ready to work
2025-12-24T02:11:08.936855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241268073940779:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.937375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940781:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.938096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098150:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.938225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272014288565:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:08.938279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272014288563:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:08.938369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272014288567:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.940349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912628:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.940482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241270520912633:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:08.940592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241270520912634:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:08.941123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241268073940786:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:08.941246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241268073940788:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:08.941922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269519452547:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.942524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912643:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.942575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867410:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:08.943137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241270520912636:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:08.943473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912642:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.943829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241269519452544:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:08.944082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268356098153:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:08.944225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241268356098154:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:08.944439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241268851867413:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.944638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940795:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.944763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272014288574:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27145
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.945042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269519452551:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.950603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272014288576:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27145
2025-12-24T02:11:08.950713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272014288575:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27145
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.950920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940796:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
2025-12-24T02:11:08.955028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940797:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
2025-12-24T02:11:08.958744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272014288577:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27145
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.959334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098167:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.962702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241268073940719:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21383811ns) {Configure:104(21383281ns) {ProcessConfigured:135(21172308ns) {StartInitialHandshake:157(21171918ns) {DropHandshakes:470(310ns) {DropIncomingHandshake:442(38ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(959ns) {SwitchToState:179(835ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.962993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272014288579:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27145
2025-12-24T02:11:08.963004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241270520912636:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19805166ns) {Configure:104(19804500ns) {ProcessConfigured:135(19514225ns) {StartInitialHandshake:157(19513980ns) {DropHandshakes:470(378ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(120ns) PrepareNewSessionHandshake:477(1019ns) {SwitchToState:179(244ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.963087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912650:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.963091Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241269519452544:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19186856ns) {Configure:104(19186307ns) {ProcessConfigured:135(18877248ns) {StartInitialHandshake:157(18876779ns) {DropHandshakes:470(357ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(1754ns) {SwitchToState:179(1203ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.963247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940801:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
2025-12-24T02:11:08.972935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098150:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.972987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268356098090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.973169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269519452551:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288577:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269519452475:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.973312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940782:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973333Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241268073940782:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.973364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912642:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940796:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940781:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912628:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940795:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241270520912568:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.973487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940792:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241268073940792:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.973631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912650:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241268073940718:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.973705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912643:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241268851867410:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241268851867339:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.973853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912639:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241270520912639:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.973856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288560:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.973865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272014288560:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.974105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241268073940782:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.974218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241268073940779:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.974236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.974247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241268073940779:2107] [node 5] ICP22 created new session: [2:7587241268073940806:2107]
2025-12-24T02:11:08.974255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241268073940806:2107] [node 5] ICS09 handshake done sender: [2:7587241268073940782:2109] self: [2:7587241268073940780:0] peer: [5:7587241268356098169:0] socket: 411 qp: -1
2025-12-24T02:11:08.974267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241268073940806:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:08.974283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.974487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241268073940792:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:08.974595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288579:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.974633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288575:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.974903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288574:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.974982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098170:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.975021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241270520912639:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.975129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272014288560:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.975149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241268073940786:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.975168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.975174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241268073940786:2110] [node 4] ICP22 created new session: [2:7587241268073940810:2110]
2025-12-24T02:11:08.975180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241268073940810:2110] [node 4] ICS09 handshake done sender: [2:7587241268073940792:2113] self: [2:7587241268073940791:0] peer: [4:7587241269519452553:0] socket: 436 qp: -1
2025-12-24T02:11:08.975185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241268073940810:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:08.975220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.975325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268356098150:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.975398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288557:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.975415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272014288557:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.975878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272014288497:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.975895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.975904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272014288497:2049] [node 3] ICP22 created new session: [1:7587241272014288585:2049]
2025-12-24T02:11:08.975910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272014288585:2049] [node 3] ICS09 handshake done sender: [1:7587241272014288560:2108] self: [1:7587241272014288559:0] peer: [3:7587241270520912652:0] socket: 412 qp: -1
2025-12-24T02:11:08.975919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272014288585:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.975927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.976084Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241269519452475:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241269519452551:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.976402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269519452555:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:08.976307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241268073940813:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37628
2025-12-24T02:11:08.976504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272014288557:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.976696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269519452554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:08.976797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269519452551:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.976817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272014288496:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.976870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.976878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272014288496:2048] [node 2] ICP22 created new session: [1:7587241272014288589:2048]
2025-12-24T02:11:08.976828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268356098090:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.976889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272014288589:2048] [node 2] ICS09 handshake done sender: [1:7587241272014288557:2107] self: [1:7587241272014288556:0] peer: [2:7587241268073940802:0] socket: 408 qp: -1
2025-12-24T02:11:08.976895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272014288589:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:08.976852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.976859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268356098090:2049] [node 2] ICP22 created new session: [5:7587241268356098171:2049]
2025-12-24T02:11:08.976865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268356098171:2049] [node 2] ICS09 handshake done sender: [5:7587241268356098150:2108] self: [5:7587241268356098169:0] peer: [2:7587241268073940780:0] socket: 413 qp: -1
2025-12-24T02:11:08.976872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268356098171:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.976908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.976878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:08.976882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241268073940781:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.976924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241270520912634:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.976946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.976952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241270520912634:2110] [node 6] ICP22 created new session: [3:7587241270520912656:2110]
2025-12-24T02:11:08.976958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241270520912656:2110] [node 6] ICS09 handshake done sender: [3:7587241270520912639:2112] self: [3:7587241270520912638:0] peer: [6:7587241268851867416:0] socket: 426 qp: -1
2025-12-24T02:11:08.976963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241270520912656:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:08.976970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:08.977065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241268073940718:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.977084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.977090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241268073940718:2048] [node 1] ICP22 created new session: [2:7587241268073940814:2048]
2025-12-24T02:11:08.977096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241268073940814:2048] [node 1] ICS09 handshake done sender: [2:7587241268073940781:2108] self: [2:7587241268073940802:0] peer: [1:7587241272014288556:0] socket: 409 qp: -1
2025-12-24T02:11:08.977103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241268073940814:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.977121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.977980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:08.978063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269519452475:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.978078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:08.978083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269519452475:2049] [node 2] ICP22 created new session: [4:7587241269519452556:2049]
2025-12-24T02:11:08.978088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241269519452556:2049] [node 2] ICS09 handshake done sender: [4:7587241269519452551:2114] self: [4:7587241269519452553:0] peer: [2:7587241268073940791:0] socket: 440 qp: -1
2025-12-24T02:11:08.978094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241269519452556:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:08.978108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.979236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.983007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098167:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.983060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268356098089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.983191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288570:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.983205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272014288570:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.983504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867420:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:08.983507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867419:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:08.983540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867418:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:08.983622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241268851867410:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.987903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241268073940779:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:08.988514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269519452555:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.988556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269519452474:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.988697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272014288570:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:08.990365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867421:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.990584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912659:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.990620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241268851867417:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.990652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241268851867419:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.990676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241268851867338:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.990748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241268851867395:2104] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.990792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241268851867339:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.990772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241270520912628:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.990817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.990829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241268851867339:2050] [node 3] ICP22 created new session: [6:7587241268851867424:2050]
2025-12-24T02:11:08.990835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241268851867424:2050] [node 3] ICS09 handshake done sender: [6:7587241268851867410:2112] self: [6:7587241268851867416:0] peer: [3:7587241270520912638:0] socket: 429 qp: -1
2025-12-24T02:11:08.990850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241268851867424:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:08.990859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.990849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241270520912568:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.990869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.990878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241270520912568:2048] [node 1] ICP22 created new session: [3:7587241270520912661:2048]
2025-12-24T02:11:08.990884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241270520912661:2048] [node 1] ICS09 handshake done sender: [3:7587241270520912628:2107] self: [3:7587241270520912652:0] peer: [1:7587241272014288559:0] socket: 415 qp: -1
2025-12-24T02:11:08.990889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241270520912661:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.990895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: 2025-12-24T02:11:08.990949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241268851867339:2050] [node 3] ICP27 obsolete handshake fail ignored
::ffff:127.1.0.0
2025-12-24T02:11:08.991185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940797:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.991573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098176:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.991642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098166:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.991657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241268356098166:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.991658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272014288565:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.991684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:08.991690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272014288565:2110] [node 5] ICP22 created new session: [1:7587241272014288593:2110]
2025-12-24T02:11:08.991696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272014288593:2110] [node 5] ICS09 handshake done sender: [1:7587241272014288570:2112] self: [1:7587241272014288568:0] peer: [5:7587241268356098175:0] socket: 445 qp: -1
2025-12-24T02:11:08.991637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940794:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.991701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272014288593:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:08.991648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241268073940794:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.991709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:08.991763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288572:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.991776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272014288572:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.991802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912660:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.991914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268356098167:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:08.992026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912660:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.992051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241270520912569:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.992148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241268073940800:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.992158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241268073940800:2118] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.992610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241268073940800:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:08.992669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241268073940719:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.992688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:08.992703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241268073940719:2049] [node 3] ICP22 created new session: [2:7587241268073940818:2049]
2025-12-24T02:11:08.992715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241268073940818:2049] [node 3] ICS09 handshake done sender: [2:7587241268073940800:2118] self: [2:7587241268073940789:0] peer: [3:7587241270520912664:0] socket: 459 qp: -1
2025-12-24T02:11:08.992721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241268073940818:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:08.992729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:08.993148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098174:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.993182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268356098091:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.993326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288576:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.993874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241268851867420:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.993914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241268851867413:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.993960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241270520912665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14352
2025-12-24T02:11:08.993990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241268073940794:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.994078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241268073940788:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.994099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.994105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241268073940788:2111] [node 6] ICP22 created new session: [2:7587241268073940821:2111]
2025-12-24T02:11:08.994109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241268073940821:2111] [node 6] ICS09 handshake done sender: [2:7587241268073940794:2114] self: [2:7587241268073940793:0] peer: [6:7587241268851867422:0] socket: 453 qp: -1
2025-12-24T02:11:08.994114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241268073940821:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:08.994121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.994831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241269519452560:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:08.995755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269519452549:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.995769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241269519452549:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.995806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241272014288497:2049] [node 3] ICP18 handshake (actor [1:7587241272014288576:2117]) is too old
2025-12-24T02:11:08.995948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268356098089:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:08.995964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:08.995968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268356098089:2048] [node 1] ICP22 created new session: [5:7587241268356098178:2048]
2025-12-24T02:11:08.995974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241268356098178:2048] [node 1] ICS09 handshake done sender: [5:7587241268356098167:2115] self: [5:7587241268356098175:0] peer: [1:7587241272014288568:0] socket: 449 qp: -1
2025-12-24T02:11:08.995978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241268356098178:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:08.995984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.996087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241268851867421:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.996117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241268851867337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:08.996455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272014288572:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.996589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098181:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.996599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272014288573:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:08.996608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272014288573:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:08.997123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272014288563:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.997147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:08.997158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272014288563:2109] [node 4] ICP22 created new session: [1:7587241272014288596:2109]
2025-12-24T02:11:08.997165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272014288596:2109] [node 4] ICS09 handshake done sender: [1:7587241272014288572:2113] self: [1:7587241272014288569:0] peer: [4:7587241269519452559:0] socket: 460 qp: -1
2025-12-24T02:11:08.997176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272014288596:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:08.997190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:08.998035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272014288573:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.998072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241269519452474:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241269519452555:2117] held: no
2025-12-24T02:11:08.998168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272014288567:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:08.998189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:08.998202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272014288567:2111] [node 6] ICP22 created new session: [1:7587241272014288599:2111]
2025-12-24T02:11:08.998213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272014288599:2111] [node 6] ICS09 handshake done sender: [1:7587241272014288573:2114] self: [1:7587241272014288571:0] peer: [6:7587241268851867429:0] socket: 439 qp: -1
2025-12-24T02:11:08.998224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272014288599:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:08.998237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:08.999690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241268356098182:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:08.999846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268356098166:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:08.999828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241270520912569:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241270520912660:2120] held: no
2025-12-24T02:11:08.999875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241270520912660:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:08.999938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241268851867338:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241268851867417:2116] held: no
2025-12-24T02:11:09.000619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241268356098154:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.000637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.000643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268356098154:2110] [node 6] ICP22 created new session: [5:7587241272651065480:2110]
2025-12-24T02:11:09.000650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241272651065480:2110] [node 6] ICS09 handshake done sender: [5:7587241268356098166:2114] self: [5:7587241268356098164:0] peer: [6:7587241268851867423:0] socket: 464 qp: -1
2025-12-24T02:11:09.000656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241272651065480:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:09.000664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.000657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912647:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.000666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241270520912647:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.001165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241270520912647:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.000737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241268851867337:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241268851867421:2120] held: no
2025-12-24T02:11:09.001213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867428:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:09.001821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867431:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:09.001776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241270520912633:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.001798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.001805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241270520912633:2109] [node 5] ICP22 created new session: [3:7587241274815879963:2109]
2025-12-24T02:11:09.001811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274815879963:2109] [node 5] ICS09 handshake done sender: [3:7587241270520912647:2116] self: [3:7587241270520912635:0] peer: [5:7587241268356098177:0] socket: 467 qp: -1
2025-12-24T02:11:09.001817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274815879963:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:09.001825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.002040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241268851867419:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.002119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241270520912569:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.002138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.002145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241270520912569:2049] [node 2] ICP22 created new session: [3:7587241274815879965:2049]
2025-12-24T02:11:09.002150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274815879965:2049] [node 2] ICS09 handshake done sender: [3:7587241270520912660:2120] self: [3:7587241270520912664:0] peer: [2:7587241268073940789:0] socket: 469 qp: -1
2025-12-24T02:11:09.002157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274815879965:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.002164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.002556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273814419858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.002762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273814419859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:09.002784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269519452555:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.002850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269519452474:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.002865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.002871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269519452474:2048] [node 1] ICP22 created new session: [4:7587241273814419860:2048]
2025-12-24T02:11:09.002877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273814419860:2048] [node 1] ICS09 handshake done sender: [4:7587241269519452555:2117] self: [4:7587241269519452559:0] peer: [1:7587241272014288569:0] socket: 462 qp: -1
2025-12-24T02:11:09.002884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273814419860:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.002892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.003319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269519452560:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.003347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241269519452476:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.003440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241270520912649:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.003448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241270520912649:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.003951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273814419863:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:09.004045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241270520912649:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.004210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269519452560:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.004277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241269519452476:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.004291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.004297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269519452476:2050] [node 3] ICP22 created new session: [4:7587241273814419864:2050]
2025-12-24T02:11:09.004303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273814419864:2050] [node 3] ICS09 handshake done sender: [4:7587241269519452560:2118] self: [4:7587241273814419862:0] peer: [3:7587241270520912648:0] socket: 471 qp: -1
2025-12-24T02:11:09.004309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273814419864:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.004316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.005047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241270520912636:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.005071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.005077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241270520912636:2108] [node 4] ICP22 created new session: [3:7587241274815879968:2108]
2025-12-24T02:11:09.005082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274815879968:2108] [node 4] ICS09 handshake done sender: [3:7587241270520912649:2117] self: [3:7587241270520912648:0] peer: [4:7587241273814419862:0] socket: 428 qp: -1
2025-12-24T02:11:09.005090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274815879968:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:09.005097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.005996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241268356098089:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:09.006263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268356098091:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4152) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241268356098174:2118] held: no
2025-12-24T02:11:09.006374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273814419859:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.006546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269519452549:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.006661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241269519452547:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.006676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.006682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269519452547:2112] [node 6] ICP22 created new session: [4:7587241273814419868:2112]
2025-12-24T02:11:09.006688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273814419868:2112] [node 6] ICS09 handshake done sender: [4:7587241269519452549:2113] self: [4:7587241269519452548:0] peer: [6:7587241268851867426:0] socket: 417 qp: -1
2025-12-24T02:11:09.006706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273814419868:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:09.006712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.007257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241268851867395:2104] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.007286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.007292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241268851867395:2104] [node 5] ICP22 created new session: [6:7587241273146834728:2104]
2025-12-24T02:11:09.007298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241273146834728:2104] [node 5] ICS09 handshake done sender: [6:7587241268851867419:2118] self: [6:7587241268851867423:0] peer: [5:7587241268356098164:0] socket: 443 qp: -1
2025-12-24T02:11:09.007304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241273146834728:2104] [node 5] ICS10 traffic start
2025-12-24T02:11:09.007326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.007659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241272651065482:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:09.007679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241268851867430:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:09.007739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098181:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.007780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268356098174:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.008098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241268851867417:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.008188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241268356098182:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.008227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268356098091:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.008243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.008254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268356098091:2050] [node 3] ICP22 created new session: [5:7587241272651065483:2050]
2025-12-24T02:11:09.008259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241272651065483:2050] [node 3] ICS09 handshake done sender: [5:7587241268356098174:2118] self: [5:7587241268356098177:0] peer: [3:7587241270520912635:0] socket: 468 qp: -1
2025-12-24T02:11:09.008269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241272651065483:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.008276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.008303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241268851867338:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.008318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.008323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241268851867338:2049] [node 2] ICP22 created new session: [6:7587241273146834732:2049]
2025-12-24T02:11:09.008329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241273146834732:2049] [node 2] ICS09 handshake done sender: [6:7587241268851867417:2116] self: [6:7587241268851867422:0] peer: [2:7587241268073940793:0] socket: 455 qp: -1
2025-12-24T02:11:09.008335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241273146834732:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.008341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.008370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241268356098154:2110] [node 6] ICP18 handshake (actor [5:7587241268356098181:2120]) is too old
2025-12-24T02:11:09.008419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241268851867421:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.008603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241268356098153:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.008565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241268851867337:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.008577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.008582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241268851867337:2048] [node 1] ICP22 created new session: [6:7587241273146834734:2048]
2025-12-24T02:11:09.008587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241273146834734:2048] [node 1] ICS09 handshake done sender: [6:7587241268851867421:2120] self: [6:7587241268851867429:0] peer: [1:7587241272014288571:0] socket: 466 qp: -1
2025-12-24T02:11:09.008591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241273146834734:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.008597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.008902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241273146834730:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51317
2025-12-24T02:11:09.009107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241269519452552:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.009118Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241269519452552:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.009374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241268851867420:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.011369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273814419870:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8759
2025-12-24T02:11:09.011478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241268851867413:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.011494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.011502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241268851867413:2114] [node 4] ICP22 created new session: [6:7587241273146834736:2114]
2025-12-24T02:11:09.011508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241273146834736:2114] [node 4] ICS09 handshake done sender: [6:7587241268851867420:2119] self: [6:7587241268851867426:0] peer: [4:7587241269519452548:0] socket: 465 qp: -1
2025-12-24T02:11:09.011516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241273146834736:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:09.011523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.012430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241269519452547:2112] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.013020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241272651065486:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32544
2025-12-24T02:11:09.013105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241268356098182:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.013043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241269519452552:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.013280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241268356098153:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.013298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.013302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241268356098153:2109] [node 4] ICP22 created new session: [5:7587241272651065487:2109]
2025-12-24T02:11:09.013307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241272651065487:2109] [node 4] ICS09 handshake done sender: [5:7587241268356098182:2121] self: [5:7587241272651065485:0] peer: [4:7587241269519452550:0] socket: 437 qp: -1
2025-12-24T02:11:09.013311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241272651065487:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:09.013317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.013427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241269519452544:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.013444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.013450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241269519452544:2108] [node 5] ICP22 created new session: [4:7587241273814419872:2108]
2025-12-24T02:11:09.013456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273814419872:2108] [node 5] ICS09 handshake done sender: [4:7587241269519452552:2115] self: [4:7587241269519452550:0] peer: [5:7587241272651065485:0] socket: 416 qp: -1
2025-12-24T02:11:09.013461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273814419872:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:09.013468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.024000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:09.024103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241268073940814:2048] [node 1] ICS07 socket disconnect 409 reason# EndOfStream
2025-12-24T02:11:09.024116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241268073940814:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.024178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.024206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241268073940814:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:09.024225Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:09.024297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241270520912661:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-12-24T02:11:09.024308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241270520912568:2048] [node 1] ICP30 unregister session Session# [3:7587241270520912661:2048] VirtualId# [3:7587241270520912652:0]
2025-12-24T02:11:09.024315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241270520912661:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.024346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.024371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.024622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241268073940718:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.024640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241268073940814:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.024646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241268073940718:2048] [node 1] ICP30 unregister session Session# [2:7587241268073940814:2048] VirtualId# [2:7587241268073940802:0]
2025-12-24T02:11:09.024669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.024679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241268073940718:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.024682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.025581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:09.025632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241273814419860:2048] [node 1] ICS01 socket: 462 reason# EndOfStream
2025-12-24T02:11:09.025641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241269519452474:2048] [node 1] ICP30 unregister session Session# [4:7587241273814419860:2048] VirtualId# [4:7587241269519452559:0]
2025-12-24T02:11:09.025646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241273814419860:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.025683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.025710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.025746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:09.025817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241268356098178:2048] [node 1] ICS07 socket disconnect 449 reason# EndOfStream
2025-12-24T02:11:09.025828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268356098178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.025867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.025895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241268356098178:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:09.025930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:09.025997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241273146834734:2048] [node 1] ICS01 socket: 466 reason# EndOfStream
2025-12-24T02:11:09.026010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241268851867337:2048] [node 1] ICP30 unregister session Session# [6:7587241273146834734:2048] VirtualId# [6:7587241268851867429:0]
2025-12-24T02:11:09.026017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241273146834734:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.026066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.026104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.026168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241268356098089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.026181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268356098178:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.026186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268356098089:2048] [node 1] ICP30 unregister session Session# [5:7587241268356098178:2048] VirtualId# [5:7587241268356098175:0]
2025-12-24T02:11:09.026202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.026208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241268356098089:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.026215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2410) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.035484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:09.035595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241274815879965:2049] [node 2] ICS01 socket: 469 reason# EndOfStream
2025-12-24T02:11:09.035609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241270520912569:2049] [node 2] ICP30 unregister session Session# [3:7587241274815879965:2049] VirtualId# [3:7587241270520912664:0]
2025-12-24T02:11:09.035618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241274815879965:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.035678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.035712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.036152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241268356098089:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:09.036501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:09.036599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241268356098171:2049] [node 2] ICS01 socket: 413 reason# EndOfStream
2025-12-24T02:11:09.036618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268356098090:2049] [node 2] ICP30 unregister session Session# [5:7587241268356098171:2049] VirtualId# [5:7587241268356098169:0]
2025-12-24T02:11:09.036634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241268356098171:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.036679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.036713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.036730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:09.036827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241269519452556:2049] [node 2] ICS01 socket: 440 reason# EndOfStream
2025-12-24T02:11:09.036843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241269519452475:2049] [node 2] ICP30 unregister session Session# [4:7587241269519452556:2049] VirtualId# [4:7587241269519452553:0]
2025-12-24T02:11:09.036852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241269519452556:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.036929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.036969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.036972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:09.037050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241273146834732:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-12-24T02:11:09.037061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241268851867338:2049] [node 2] ICP30 unregister session Session# [6:7587241273146834732:2049] VirtualId# [6:7587241268851867422:0]
2025-12-24T02:11:09.037069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241273146834732:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.037107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.037137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.055874Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:09.055989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241272651065483:2050] [node 3] ICS01 socket: 468 reason# EndOfStream
2025-12-24T02:11:09.056005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268356098091:2050] [node 3] ICP30 unregister session Session# [5:7587241272651065483:2050] VirtualId# [5:7587241268356098177:0]
2025-12-24T02:11:09.056014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241272651065483:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.055989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:09.055998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:09.056065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241268851867424:2050] [node 3] ICS01 socket: 429 reason# EndOfStream
2025-12-24T02:11:09.056066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.056076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241268851867339:2050] [node 3] ICP30 unregister session Session# [6:7587241268851867424:2050] VirtualId# [6:7587241268851867416:0]
2025-12-24T02:11:09.056086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241268851867424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.056094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241273814419864:2050] [node 3] ICS01 socket: 471 reason# EndOfStream
2025-12-24T02:11:09.056098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.056121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241269519452476:2050] [node 3] ICP30 unregister session Session# [4:7587241273814419864:2050] VirtualId# [4:7587241273814419862:0]
2025-12-24T02:11:09.056132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241273814419864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.056137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.056165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.056194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.056232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.064530Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:09.064640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241272651065487:2109] [node 4] ICS01 socket: 437 reason# EndOfStream
2025-12-24T02:11:09.064656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241268356098153:2109] [node 4] ICP30 unregister session Session# [5:7587241272651065487:2109] VirtualId# [5:7587241272651065485:0]
2025-12-24T02:11:09.064664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241272651065487:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.064708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:09.064720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:09.064754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:09.064801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241273146834736:2114] [node 4] ICS07 socket disconnect 465 reason# EndOfStream
2025-12-24T02:11:09.064816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241273146834736:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.064875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:09.064923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241273146834736:2114] [node 4] ICS15 start handshake
2025-12-24T02:11:09.065320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241268851867413:2114] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14114) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.065334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241273146834736:2114] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.065340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241268851867413:2114] [node 4] ICP30 unregister session Session# [6:7587241273146834736:2114] VirtualId# [6:7587241268851867426:0]
2025-12-24T02:11:09.065363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:09.065375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241268851867413:2114] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.065382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14114) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.076724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241268851867413:2114] [node 4] ICP33 wake up from error state
2025-12-24T02:11:09.076781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:09.076873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241273146834728:2104] [node 5] ICS01 socket: 443 reason# EndOfStream
2025-12-24T02:11:09.076889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241268851867395:2104] [node 5] ICP30 unregister session Session# [6:7587241273146834728:2104] VirtualId# [6:7587241268851867423:0]
2025-12-24T02:11:09.076900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241273146834728:2104] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.076955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:09.076993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:09.091959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276096265150:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.091974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276096265151:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.093609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275315478214:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.093622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275315478215:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.095578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274821326050:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.095586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274821326051:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.095590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274821326052:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.097366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241276317441948:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.097372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241276317441949:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.097375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241276317441950:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.094904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241275298544643:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:09.094914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241275298544644:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.099548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274343187717:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.099566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274343187718:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.099570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274343187719:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.110306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275315478275:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:09.110873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241275298544708:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:09.110979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241275298544710:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:09.109220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274343187778:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.114964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187782:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.119593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276096265218:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:09.119708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276096265219:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:09.120356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275315478283:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:09.120862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275315478286:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:09.121299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274821326120:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:09.122820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241276317442019:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.126815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241276317442023:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.127150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241275298544718:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47738
2025-12-24T02:11:09.127804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241275298544719:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47738
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.130848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241275298544644:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20996363ns) {Configure:104(20995972ns) {ProcessConfigured:135(19939658ns) {StartInitialHandshake:157(19938890ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(286ns) PrepareNewSessionHandshake:477(1310ns) {SwitchToState:179(624ns)}}}}}
2025-12-24T02:11:09.131591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241275298544721:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47738
2025-12-24T02:11:09.131671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274821326118:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:09.134739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276096265215:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:09.134726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241275298544714:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:09.135215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241275298544720:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47738
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.135567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274343187792:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:09.136197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187789:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.136210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187790:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.136722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442027:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.136744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442028:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.136756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442025:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.136771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442026:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.136475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326126:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.136499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326127:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.139135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478295:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
2025-12-24T02:11:09.139171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478293:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
2025-12-24T02:11:09.139191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478294:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.146613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478293:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.145002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265228:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.145045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265227:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.146330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241276096265150:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30978307ns) {Configure:104(30976945ns) {ProcessConfigured:135(30548145ns) {StartInitialHandshake:157(30547569ns) {DropHandshakes:470(692ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1263ns) {SwitchToState:179(300ns)}}}}}
2025-12-24T02:11:09.149330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478294:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.149392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241275315478215:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.149791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265227:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.149184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544718:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.149833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241276096265150:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.149922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326126:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.149938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544705:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.149958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241275298544705:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.149979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274821326050:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.150375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442025:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442026:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241276317441949:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.150452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241276317441950:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.150476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544720:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265221:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276096265221:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.150628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442027:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241276317441948:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.150552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478285:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.150563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275315478285:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.150873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241275298544705:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.151199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276096265221:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.151271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241275298544643:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.151303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.151311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241275298544643:2048] [node 2] ICP22 created new session: [1:7587241275298544729:2048]
2025-12-24T02:11:09.151319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241275298544729:2048] [node 2] ICS09 handshake done sender: [1:7587241275298544705:2108] self: [1:7587241275298544704:0] peer: [2:7587241276096265230:0] socket: 414 qp: -1
2025-12-24T02:11:09.151326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241275298544729:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:09.151336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.151731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
2025-12-24T02:11:09.151602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276096265218:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.151617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.151623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276096265218:2109] [node 5] ICP22 created new session: [2:7587241276096265232:2109]
2025-12-24T02:11:09.151629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276096265232:2109] [node 5] ICS09 handshake done sender: [2:7587241276096265221:2111] self: [2:7587241276096265220:0] peer: [5:7587241276317442030:0] socket: 424 qp: -1
2025-12-24T02:11:09.151635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276096265232:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:09.151642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.151247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442033:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.151474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241276317442025:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.151903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241276317441949:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.151915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.151921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241276317441949:2049] [node 2] ICP22 created new session: [5:7587241276317442034:2049]
2025-12-24T02:11:09.151927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241276317442034:2049] [node 2] ICS09 handshake done sender: [5:7587241276317442025:2114] self: [5:7587241276317442030:0] peer: [2:7587241276096265220:0] socket: 426 qp: -1
2025-12-24T02:11:09.151947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241276317442034:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.151955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.154260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265212:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154263Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241276096265215:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19432091ns) {Configure:104(19431156ns) {ProcessConfigured:135(19058106ns) {StartInitialHandshake:157(19057376ns) {DropHandshakes:470(1107ns) {DropIncomingHandshake:442(158ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1328ns) {SwitchToState:179(202ns)}}}}}
2025-12-24T02:11:09.154284Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241274343187719:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41470850ns) {Configure:104(41470176ns) {ProcessConfigured:135(39530190ns) {StartInitialHandshake:157(39528968ns) {DropHandshakes:470(480ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(929ns) {SwitchToState:179(436ns)}}}}}
2025-12-24T02:11:09.154277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276096265212:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.154264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544715:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544717:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241275298544715:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.154292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241275298544717:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.154403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274343187790:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274343187782:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274343187789:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274343187719:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.154520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274343187718:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.154853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265223:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276096265223:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.154903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326122:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241274821326122:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.154865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478277:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.154881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275315478277:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.155145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275315478285:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.155355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275315478283:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.155370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.155380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275315478283:2110] [node 5] ICP22 created new session: [3:7587241275315478300:2110]
2025-12-24T02:11:09.155388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275315478300:2110] [node 5] ICS09 handshake done sender: [3:7587241275315478285:2112] self: [3:7587241275315478284:0] peer: [5:7587241276317442031:0] socket: 429 qp: -1
2025-12-24T02:11:09.155396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275315478300:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:09.155407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.155765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275315478277:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.155908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275315478275:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.155922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.155929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275315478275:2107] [node 6] ICP22 created new session: [3:7587241275315478304:2107]
2025-12-24T02:11:09.155946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275315478304:2107] [node 6] ICS09 handshake done sender: [3:7587241275315478277:2108] self: [3:7587241275315478276:0] peer: [6:7587241274343187798:0] socket: 415 qp: -1
2025-12-24T02:11:09.155957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275315478304:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:09.155979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.156926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544721:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.157115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241275298544731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47738
2025-12-24T02:11:09.157481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241275298544715:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.157755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544731:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.157777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241275298544708:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.157794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.157801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241275298544708:2107] [node 4] ICP22 created new session: [1:7587241275298544733:2107]
2025-12-24T02:11:09.157808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241275298544733:2107] [node 4] ICS09 handshake done sender: [1:7587241275298544715:2111] self: [1:7587241275298544712:0] peer: [4:7587241274821326130:0] socket: 439 qp: -1
2025-12-24T02:11:09.157813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241275298544733:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:09.157822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.158410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241275298544643:2048] [node 2] ICP18 handshake (actor [1:7587241275298544731:2118]) is too old
2025-12-24T02:11:09.159347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241274821326050:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241274821326126:2113] held: no
2025-12-24T02:11:09.159173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274343187792:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.159266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.159400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274343187782:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.159468Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241275298544714:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24615103ns) {Configure:104(24613868ns) {ProcessConfigured:135(24238974ns) {StartInitialHandshake:157(24237923ns) {DropHandshakes:470(2051ns) {DropIncomingHandshake:442(244ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:477(1683ns) {SwitchToState:179(855ns)}}}}}
2025-12-24T02:11:09.159615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241275298544717:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.161700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241276096265150:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241276096265227:2113] held: no
2025-12-24T02:11:09.161834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276096265223:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.161025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241275298544710:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.161049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.161055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241275298544710:2109] [node 5] ICP22 created new session: [1:7587241275298544739:2109]
2025-12-24T02:11:09.161062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241275298544739:2109] [node 5] ICS09 handshake done sender: [1:7587241275298544717:2112] self: [1:7587241275298544713:0] peer: [5:7587241276317442032:0] socket: 444 qp: -1
2025-12-24T02:11:09.161068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241275298544739:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:09.161082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.162125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274821326122:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.162324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241274821326120:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.162346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.162353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274821326120:2111] [node 6] ICP22 created new session: [4:7587241274821326133:2111]
2025-12-24T02:11:09.162360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274821326133:2111] [node 6] ICS09 handshake done sender: [4:7587241274821326122:2112] self: [4:7587241274821326121:0] peer: [6:7587241274343187799:0] socket: 431 qp: -1
2025-12-24T02:11:09.162368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274821326133:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:09.162389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.162153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276096265219:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.162173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.162184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276096265219:2110] [node 6] ICP22 created new session: [2:7587241276096265238:2110]
2025-12-24T02:11:09.162191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276096265238:2110] [node 6] ICS09 handshake done sender: [2:7587241276096265223:2112] self: [2:7587241276096265222:0] peer: [6:7587241274343187800:0] socket: 425 qp: -1
2025-12-24T02:11:09.162198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276096265238:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:09.162207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.164210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478307:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.164385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.164599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326138:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.164696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274821326126:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.164775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544719:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.165224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274343187719:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.165278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.165288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274343187719:2050] [node 3] ICP22 created new session: [6:7587241274343187802:2050]
2025-12-24T02:11:09.165303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274343187802:2050] [node 3] ICS09 handshake done sender: [6:7587241274343187782:2109] self: [6:7587241274343187798:0] peer: [3:7587241275315478276:0] socket: 416 qp: -1
2025-12-24T02:11:09.165318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274343187802:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.165343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.165927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.165932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.166090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544723:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.166100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241275298544723:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.166561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241275298544723:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.166743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241275298544644:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.166765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.166776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241275298544644:2049] [node 3] ICP22 created new session: [1:7587241275298544743:2049]
2025-12-24T02:11:09.166782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241275298544743:2049] [node 3] ICS09 handshake done sender: [1:7587241275298544723:2117] self: [1:7587241275298544709:0] peer: [3:7587241275315478308:0] socket: 460 qp: -1
2025-12-24T02:11:09.166794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241275298544743:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.166802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.166852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276096265212:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.167971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326138:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.168022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274821326051:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.165965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478298:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.166014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241275315478214:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.166662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275315478309:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46439
2025-12-24T02:11:09.166872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275315478298:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.167036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241275315478214:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.167049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.167056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275315478214:2048] [node 1] ICP22 created new session: [3:7587241275315478310:2048]
2025-12-24T02:11:09.167062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275315478310:2048] [node 1] ICS09 handshake done sender: [3:7587241275315478298:2118] self: [3:7587241275315478308:0] peer: [1:7587241275298544709:0] socket: 461 qp: -1
2025-12-24T02:11:09.167069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275315478310:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.167078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.168164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275315478294:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.168594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241275315478215:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.168612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.168620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275315478215:2049] [node 2] ICP22 created new session: [3:7587241275315478312:2049]
2025-12-24T02:11:09.168626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275315478312:2049] [node 2] ICS09 handshake done sender: [3:7587241275315478294:2116] self: [3:7587241275315478297:0] peer: [2:7587241276096265211:0] socket: 453 qp: -1
2025-12-24T02:11:09.168631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275315478312:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.168637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.169840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274821326050:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.169862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.169873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274821326050:2048] [node 1] ICP22 created new session: [4:7587241274821326140:2048]
2025-12-24T02:11:09.169881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274821326140:2048] [node 1] ICS09 handshake done sender: [4:7587241274821326126:2113] self: [4:7587241274821326130:0] peer: [1:7587241275298544712:0] socket: 445 qp: -1
2025-12-24T02:11:09.169888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274821326140:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.169908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.170312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442037:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.170338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442036:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.170330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326127:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.170337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326136:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.170383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241274821326120:2111] [node 6] ICP18 handshake (actor [4:7587241274821326127:2114]) is too old
2025-12-24T02:11:09.170329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.170401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276096265151:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.170412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.170419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276096265151:2049] [node 3] ICP22 created new session: [2:7587241276096265244:2049]
2025-12-24T02:11:09.170429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276096265244:2049] [node 3] ICS09 handshake done sender: [2:7587241276096265212:2107] self: [2:7587241276096265211:0] peer: [3:7587241275315478297:0] socket: 420 qp: -1
2025-12-24T02:11:09.170435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276096265244:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.170453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.170754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326136:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.170786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274821326052:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.174956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265235:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.174989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276096265235:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.175403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276096265227:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.176114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265243:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.176150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241276096265150:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.176167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.176176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276096265150:2048] [node 1] ICP22 created new session: [2:7587241276096265247:2048]
2025-12-24T02:11:09.176184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276096265247:2048] [node 1] ICS09 handshake done sender: [2:7587241276096265227:2113] self: [2:7587241276096265230:0] peer: [1:7587241275298544704:0] socket: 418 qp: -1
2025-12-24T02:11:09.176194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276096265247:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.176204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.176709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276096265235:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.179106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326143:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.178933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276096265215:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.178963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.178977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276096265215:2108] [node 4] ICP22 created new session: [2:7587241276096265250:2108]
2025-12-24T02:11:09.178986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276096265250:2108] [node 4] ICS09 handshake done sender: [2:7587241276096265235:2116] self: [2:7587241276096265226:0] peer: [4:7587241274821326139:0] socket: 475 qp: -1
2025-12-24T02:11:09.178997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276096265250:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:09.179008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.179522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274821326138:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.179617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274821326051:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.179635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.179644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274821326051:2049] [node 2] ICP22 created new session: [4:7587241274821326144:2049]
2025-12-24T02:11:09.179650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274821326144:2049] [node 2] ICS09 handshake done sender: [4:7587241274821326138:2119] self: [4:7587241274821326139:0] peer: [2:7587241276096265226:0] socket: 480 qp: -1
2025-12-24T02:11:09.179658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274821326144:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.179665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.181226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241276317442026:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.181724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241276317441950:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.181738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.181745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241276317441950:2050] [node 3] ICP22 created new session: [5:7587241276317442039:2050]
2025-12-24T02:11:09.181754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241276317442039:2050] [node 3] ICS09 handshake done sender: [5:7587241276317442026:2115] self: [5:7587241276317442031:0] peer: [3:7587241275315478284:0] socket: 450 qp: -1
2025-12-24T02:11:09.181761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241276317442039:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.181768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.181940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442038:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.182892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478295:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.183067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241275315478283:2110] [node 5] ICP18 handshake (actor [3:7587241275315478295:2117]) is too old
2025-12-24T02:11:09.183165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241275298544710:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(22101259ns) {HandleHandshakeStatus:329(22100533ns) {DropIncomingHandshake:442(3645ns) DropOutgoingHandshake:457(137ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(22005591ns) {ForwardSessionEventToSession:612(21989402ns) ForwardSessionEventToSession:612(2469ns) ForwardSessionEventToSession:612(711ns) ForwardSessionEventToSession:612(1222ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(1395ns) ForwardSessionEventToSession:612(894ns) ForwardSessionEventToSession:612(1027ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(1300ns)}}}
2025-12-24T02:11:09.183340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275315478292:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.183351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275315478292:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.183007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442038:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.183083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241276317441948:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241276317442027:2116] held: no
2025-12-24T02:11:09.183131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241276317442027:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.183977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274343187792:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241274343187790:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.186935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241276096265218:2109] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.184610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187806:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.190826Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241274343187719:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25562880ns) {HandleHandshakeStatus:329(25561904ns) {DropIncomingHandshake:442(521ns) DropOutgoingHandshake:457(2434ns) SwitchToState:179(465ns) ProcessPendingSessionEvents:495(20300191ns) {ForwardSessionEventToSession:612(20282702ns) ForwardSessionEventToSession:612(2697ns) ForwardSessionEventToSession:612(1093ns) ForwardSessionEventToSession:612(993ns) ForwardSessionEventToSession:612(1259ns) ForwardSessionEventToSession:612(1115ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(894ns) ForwardSessionEventToSession:612(875ns) ForwardSessionEventToSession:612(1072ns)}}}
2025-12-24T02:11:09.191022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187807:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.191082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.191470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274343187789:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.191692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274343187805:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.191732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274343187717:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.191847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241275298544736:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.191858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241275298544736:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.192582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274343187718:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.192601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.192608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274343187718:2049] [node 2] ICP22 created new session: [6:7587241274343187809:2049]
2025-12-24T02:11:09.192614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274343187809:2049] [node 2] ICS09 handshake done sender: [6:7587241274343187789:2113] self: [6:7587241274343187800:0] peer: [2:7587241276096265222:0] socket: 427 qp: -1
2025-12-24T02:11:09.192622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274343187809:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.192630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.192745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241275298544736:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.192821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241275298544714:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.192834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.192839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241275298544714:2110] [node 6] ICP22 created new session: [1:7587241275298544746:2110]
2025-12-24T02:11:09.192846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241275298544746:2110] [node 6] ICS09 handshake done sender: [1:7587241275298544736:2119] self: [1:7587241275298544724:0] peer: [6:7587241274343187808:0] socket: 470 qp: -1
2025-12-24T02:11:09.192850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241275298544746:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:09.192858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.192436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276096265252:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18541
2025-12-24T02:11:09.193674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442028:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.193816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241276317442019:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.194262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275315478292:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.194367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275315478286:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.194394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.194403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275315478286:2109] [node 4] ICP22 created new session: [3:7587241275315478315:2109]
2025-12-24T02:11:09.194416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275315478315:2109] [node 4] ICS09 handshake done sender: [3:7587241275315478292:2114] self: [3:7587241275315478291:0] peer: [4:7587241274821326142:0] socket: 468 qp: -1
2025-12-24T02:11:09.194421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275315478315:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:09.194431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.194604Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326129:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.194624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241274821326129:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.194677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241276317441948:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.194721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.194732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241276317441948:2048] [node 1] ICP22 created new session: [5:7587241276317442045:2048]
2025-12-24T02:11:09.194873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241274821326052:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26549) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241274821326136:2117] held: no
2025-12-24T02:11:09.194739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241276317442045:2048] [node 1] ICS09 handshake done sender: [5:7587241276317442027:2116] self: [5:7587241276317442032:0] peer: [1:7587241275298544713:0] socket: 451 qp: -1
2025-12-24T02:11:09.194754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241276317442045:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.194789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.195820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276096265252:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.195875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241276096265219:2110] [node 6] ICP18 handshake (actor [2:7587241276096265252:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.196285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326146:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.196416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241276096265151:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:09.197438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274343187790:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.197455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274343187717:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31418) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241274343187805:2119] held: no
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:09.197549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187812:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.197570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274343187792:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.197587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.197598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274343187792:2112] [node 4] ICP22 created new session: [6:7587241274343187813:2112]
2025-12-24T02:11:09.197604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274343187813:2112] [node 4] ICS09 handshake done sender: [6:7587241274343187790:2114] self: [6:7587241274343187799:0] peer: [4:7587241274821326121:0] socket: 446 qp: -1
2025-12-24T02:11:09.197612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274343187813:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:09.197627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.198545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274821326129:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.198974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241274821326118:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.198991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.198998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274821326118:2110] [node 5] ICP22 created new session: [4:7587241274821326148:2110]
2025-12-24T02:11:09.199004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274821326148:2110] [node 5] ICS09 handshake done sender: [4:7587241274821326129:2116] self: [4:7587241274821326125:0] peer: [5:7587241276317442043:0] socket: 477 qp: -1
2025-12-24T02:11:09.199011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274821326148:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:09.199018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.199543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187811:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.199743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274821326146:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.199777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241274821326118:2110] [node 5] ICP18 handshake (actor [4:7587241274821326146:2121]) is too old
2025-12-24T02:11:09.199843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274343187805:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.199979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274343187717:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.200005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.200015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274343187717:2048] [node 1] ICP22 created new session: [6:7587241274343187815:2048]
2025-12-24T02:11:09.200023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274343187815:2048] [node 1] ICS09 handshake done sender: [6:7587241274343187805:2119] self: [6:7587241274343187808:0] peer: [1:7587241275298544724:0] socket: 484 qp: -1
2025-12-24T02:11:09.200037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274343187815:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.200055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.200082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274821326150:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33643
2025-12-24T02:11:09.200161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274821326136:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.200228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274821326052:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.200250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.200261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274821326052:2050] [node 3] ICP22 created new session: [4:7587241274821326151:2050]
2025-12-24T02:11:09.200267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274821326151:2050] [node 3] ICS09 handshake done sender: [4:7587241274821326136:2117] self: [4:7587241274821326142:0] peer: [3:7587241275315478291:0] socket: 469 qp: -1
2025-12-24T02:11:09.200277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274821326151:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.200288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.202989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241276317442047:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16159
2025-12-24T02:11:09.203119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241276317442038:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.203209Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241276317442019:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241276317442038:2122] held: no
2025-12-24T02:11:09.203323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274343187812:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.203395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274343187778:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.203674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241276317442019:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.203697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.203698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241276317442029:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.203703Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241276317442029:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.203756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241276317442019:2111] [node 4] ICP22 created new session: [5:7587241276317442048:2111]
2025-12-24T02:11:09.203820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241276317442048:2111] [node 4] ICS09 handshake done sender: [5:7587241276317442038:2122] self: [5:7587241276317442043:0] peer: [4:7587241274821326125:0] socket: 442 qp: -1
2025-12-24T02:11:09.203834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241276317442048:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:09.203856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.204488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241276317442029:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.204510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274343187818:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32829
2025-12-24T02:11:09.204621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274343187812:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.204842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274343187778:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.204862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.204869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274343187778:2107] [node 5] ICP22 created new session: [6:7587241274343187819:2107]
2025-12-24T02:11:09.204876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274343187819:2107] [node 5] ICS09 handshake done sender: [6:7587241274343187812:2123] self: [6:7587241274343187817:0] peer: [5:7587241276317442024:0] socket: 437 qp: -1
2025-12-24T02:11:09.204884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274343187819:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:09.204896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.204641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241276317442023:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.204669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.204674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241276317442023:2108] [node 6] ICP22 created new session: [5:7587241276317442051:2108]
2025-12-24T02:11:09.204679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241276317442051:2108] [node 6] ICS09 handshake done sender: [5:7587241276317442029:2118] self: [5:7587241276317442024:0] peer: [6:7587241274343187817:0] socket: 436 qp: -1
2025-12-24T02:11:09.204684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241276317442051:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:09.204691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.224209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:09.224326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241276096265247:2048] [node 1] ICS01 socket: 418 reason# EndOfStream
2025-12-24T02:11:09.224339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241276096265150:2048] [node 1] ICP30 unregister session Session# [2:7587241276096265247:2048] VirtualId# [2:7587241276096265230:0]
2025-12-24T02:11:09.224347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241276096265247:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.224395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.224419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.225198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:09.225294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241275315478310:2048] [node 1] ICS01 socket: 461 reason# EndOfStream
2025-12-24T02:11:09.225312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241275315478214:2048] [node 1] ICP30 unregister session Session# [3:7587241275315478310:2048] VirtualId# [3:7587241275315478308:0]
2025-12-24T02:11:09.225320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241275315478310:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.225383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.225419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.225851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:09.225928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241274821326140:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:09.225941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241274821326050:2048] [node 1] ICP30 unregister session Session# [4:7587241274821326140:2048] VirtualId# [4:7587241274821326130:0]
2025-12-24T02:11:09.225946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241274821326140:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.226001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.226036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.229042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:09.229148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274343187815:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-12-24T02:11:09.229156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274343187717:2048] [node 1] ICP30 unregister session Session# [6:7587241274343187815:2048] VirtualId# [6:7587241274343187808:0]
2025-12-24T02:11:09.229163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274343187815:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.229219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.229248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.230768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:09.230841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241276317442045:2048] [node 1] ICS01 socket: 451 reason# EndOfStream
2025-12-24T02:11:09.230848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241276317441948:2048] [node 1] ICP30 unregister session Session# [5:7587241276317442045:2048] VirtualId# [5:7587241276317442032:0]
2025-12-24T02:11:09.230854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241276317442045:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.230902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.230934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.238782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:09.238904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241275315478312:2049] [node 2] ICS01 socket: 453 reason# EndOfStream
2025-12-24T02:11:09.238915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241275315478215:2049] [node 2] ICP30 unregister session Session# [3:7587241275315478312:2049] VirtualId# [3:7587241275315478297:0]
2025-12-24T02:11:09.238924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241275315478312:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.239003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.239038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.239512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:09.239605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241274343187809:2049] [node 2] ICS07 socket disconnect 427 reason# EndOfStream
2025-12-24T02:11:09.239616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274343187809:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.239602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:09.239661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.239676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241274821326144:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-12-24T02:11:09.239688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241274821326051:2049] [node 2] ICP30 unregister session Session# [4:7587241274821326144:2049] VirtualId# [4:7587241274821326139:0]
2025-12-24T02:11:09.239689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241274343187809:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:09.239694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241274821326144:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.239743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.239777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.240110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274343187718:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27976) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.240130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274343187809:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.240136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274343187718:2049] [node 2] ICP30 unregister session Session# [6:7587241274343187809:2049] VirtualId# [6:7587241274343187800:0]
2025-12-24T02:11:09.240163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.240177Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241274343187718:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.240185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27976) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.242794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:09.242898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241276317442034:2049] [node 2] ICS01 socket: 426 reason# EndOfStream
2025-12-24T02:11:09.242904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241276317441949:2049] [node 2] ICP30 unregister session Session# [5:7587241276317442034:2049] VirtualId# [5:7587241276317442030:0]
2025-12-24T02:11:09.242911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241276317442034:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.242965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.242995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.250195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241274343187718:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:09.256196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:09.256308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241276317442039:2050] [node 3] ICS01 socket: 450 reason# EndOfStream
2025-12-24T02:11:09.256321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241276317441950:2050] [node 3] ICP30 unregister session Session# [5:7587241276317442039:2050] VirtualId# [5:7587241276317442031:0]
2025-12-24T02:11:09.256329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241276317442039:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.256398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.256428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.259525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:09.259632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241274343187802:2050] [node 3] ICS07 socket disconnect 416 reason# EndOfStream
2025-12-24T02:11:09.259646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274343187802:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.259712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.259741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241274343187802:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:09.260125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274343187719:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26549) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.260144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274343187802:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.260151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274343187719:2050] [node 3] ICP30 unregister session Session# [6:7587241274343187802:2050] VirtualId# [6:7587241274343187798:0]
2025-12-24T02:11:09.260182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.260190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241274343187719:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.260193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26549) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.265496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:09.265624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241276317442048:2111] [node 4] ICS01 socket: 442 reason# EndOfStream
2025-12-24T02:11:09.265644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241276317442019:2111] [node 4] ICP30 unregister session Session# [5:7587241276317442048:2111] VirtualId# [5:7587241276317442043:0]
2025-12-24T02:11:09.265659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241276317442048:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.265715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:09.265749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:09.267497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:09.267595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274343187813:2112] [node 4] ICS01 socket: 446 reason# EndOfStream
2025-12-24T02:11:09.267608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274343187792:2112] [node 4] ICP30 unregister session Session# [6:7587241274343187813:2112] VirtualId# [6:7587241274343187799:0]
2025-12-24T02:11:09.267615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274343187813:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.267698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:09.267735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:09.269637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241274343187719:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:09.279371Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:09.279480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274343187819:2107] [node 5] ICS01 socket: 437 reason# EndOfStream
2025-12-24T02:11:09.279498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274343187778:2107] [node 5] ICP30 unregister session Session# [6:7587241274343187819:2107] VirtualId# [6:7587241274343187817:0]
2025-12-24T02:11:09.279507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274343187819:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.279565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:09.279598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:09.300416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276094531860:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.300425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276094531861:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.299945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272580864678:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:09.299956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272580864679:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.300225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274909348026:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.300231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274909348027:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.300236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274909348028:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.300060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241274256683034:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.300068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241274256683035:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.307254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241273874714379:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.307264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241273874714380:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.307269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241273874714381:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.311157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274086812782:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.311167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274086812783:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.311171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274086812784:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.333598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272580864745:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:09.333694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272580864746:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.334928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241272580864750:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.336174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683061:2074] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.338037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276094531924:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:09.338123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276094531925:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:09.338224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241276094531926:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:09.337916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272580864756:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18714
2025-12-24T02:11:09.338788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531920:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.343721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531933:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
2025-12-24T02:11:09.346858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241274256683088:2082] [node 6] ICP01 ready to work
2025-12-24T02:11:09.346929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241273874714409:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:09.348339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272580864758:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18714
2025-12-24T02:11:09.350922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864758:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.350896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241273874714420:2073] [node 6] ICP01 ready to work
2025-12-24T02:11:09.351044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274909348068:2064] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.355243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683102:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.358866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241276094531925:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20014245ns) {Configure:104(16066573ns) {ProcessConfigured:135(15754436ns) {StartInitialHandshake:157(15753768ns) {DropHandshakes:470(519ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(1609ns) {SwitchToState:179(911ns)}}}}}
2025-12-24T02:11:09.359017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683061:2074] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.359120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241274256683034:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.359304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683103:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
2025-12-24T02:11:09.359842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531920:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.359877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241276094531860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.360769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683105:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
2025-12-24T02:11:09.359505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864743:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.359522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272580864743:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.359920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864756:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.360056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864739:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.360060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272580864739:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.362816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241274909348026:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19873509ns) {Configure:104(19872741ns) {ProcessConfigured:135(19599258ns) {StartInitialHandshake:157(19598532ns) {DropHandshakes:470(432ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(687ns) PrepareNewSessionHandshake:477(793ns) {SwitchToState:179(358ns)}}}}}
2025-12-24T02:11:09.363184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241274256683088:2082] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16265069ns) {Configure:104(16264582ns) {ProcessConfigured:135(15967037ns) {StartInitialHandshake:157(15966704ns) {DropHandshakes:470(364ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(704ns) {SwitchToState:179(383ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.364774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683110:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
2025-12-24T02:11:09.363046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241276094531926:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24199378ns) {Configure:104(24198160ns) {ProcessConfigured:135(23953489ns) {StartInitialHandshake:157(23953166ns) {DropHandshakes:470(238ns) {DropIncomingHandshake:442(36ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(659ns) {SwitchToState:179(468ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.364922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531941:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
2025-12-24T02:11:09.363364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241273874714381:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16584833ns) {Configure:104(16583680ns) {ProcessConfigured:135(15884122ns) {StartInitialHandshake:157(15883972ns) {DropHandshakes:470(404ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1049ns) {SwitchToState:179(29ns)}}}}}
2025-12-24T02:11:09.366786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241273874714409:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19784952ns) {Configure:104(19784287ns) {ProcessConfigured:135(19160722ns) {StartInitialHandshake:157(19160126ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(55ns) PrepareNewSessionHandshake:477(986ns) {SwitchToState:179(417ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.366815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272580864743:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.366834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272580864739:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.366854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272580864763:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18714
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.366903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812848:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.366970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812849:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.367070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276094531920:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.367686Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241274256683061:2074] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.367828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348093:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.368356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274086812848:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.368029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241274256683034:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.368048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.368054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241274256683034:2048] [node 1] ICP22 created new session: [3:7587241274256683111:2048]
2025-12-24T02:11:09.368062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274256683111:2048] [node 1] ICS09 handshake done sender: [3:7587241274256683061:2074] self: [3:7587241274256683104:0] peer: [1:7587241272580864742:0] socket: 425 qp: -1
2025-12-24T02:11:09.368068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274256683111:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.368074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.369091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683110:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.367247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272580864678:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.367337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.367345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272580864678:2048] [node 2] ICP22 created new session: [1:7587241272580864764:2048]
2025-12-24T02:11:09.367352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272580864764:2048] [node 2] ICS09 handshake done sender: [1:7587241272580864739:2107] self: [1:7587241272580864738:0] peer: [2:7587241276094531939:0] socket: 420 qp: -1
2025-12-24T02:11:09.367360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272580864764:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:09.367383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.374751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241276094531860:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.374778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.374789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276094531860:2048] [node 1] ICP22 created new session: [2:7587241276094531942:2048]
2025-12-24T02:11:09.374795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276094531942:2048] [node 1] ICS09 handshake done sender: [2:7587241276094531920:2107] self: [2:7587241276094531939:0] peer: [1:7587241272580864738:0] socket: 421 qp: -1
2025-12-24T02:11:09.374803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276094531942:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.374829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.381144Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241274909348027:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(38350641ns) {Configure:104(30355379ns) {ProcessConfigured:135(29919587ns) {StartInitialHandshake:157(29918410ns) {DropHandshakes:470(384ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(1352ns) {SwitchToState:179(817ns)}}}}}
2025-12-24T02:11:09.381217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348093:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.381494Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241274909348068:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30376124ns) {Configure:104(30375524ns) {ProcessConfigured:135(30086749ns) {StartInitialHandshake:157(30086346ns) {DropHandshakes:470(458ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(566ns)}}}}}
2025-12-24T02:11:09.379128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241276094531924:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40836575ns) {Configure:104(40834815ns) {ProcessConfigured:135(40247843ns) {StartInitialHandshake:157(40246841ns) {DropHandshakes:470(453ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(128ns) PrepareNewSessionHandshake:477(2309ns) {SwitchToState:179(1167ns)}}}}}
2025-12-24T02:11:09.380960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714449:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.381043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714450:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.381673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241273874714379:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30670481ns) {Configure:104(30669588ns) {ProcessConfigured:135(30208967ns) {StartInitialHandshake:157(30208886ns) {DropHandshakes:470(347ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(754ns) {SwitchToState:179(33ns)}}}}}
2025-12-24T02:11:09.386851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241273874714449:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.386921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241273874714379:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.387927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714453:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.388273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241273874714449:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.388378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241273874714379:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.388402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.388408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241273874714379:2048] [node 1] ICP22 created new session: [5:7587241273874714454:2048]
2025-12-24T02:11:09.388415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241273874714454:2048] [node 1] ICS09 handshake done sender: [5:7587241273874714449:2107] self: [5:7587241273874714452:0] peer: [1:7587241272580864751:0] socket: 430 qp: -1
2025-12-24T02:11:09.388423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241273874714454:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.388443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.387140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864752:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.387154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272580864752:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.388077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272580864752:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.388815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272580864746:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.388825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.388831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272580864746:2110] [node 5] ICP22 created new session: [1:7587241272580864769:2110]
2025-12-24T02:11:09.388836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272580864769:2110] [node 5] ICS09 handshake done sender: [1:7587241272580864752:2112] self: [1:7587241272580864751:0] peer: [5:7587241273874714452:0] socket: 423 qp: -1
2025-12-24T02:11:09.388841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272580864769:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:09.388846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.391125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241273874714450:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.394903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531933:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.399152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241273874714380:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(47377521ns) {Configure:104(47376541ns) {ProcessConfigured:135(47169535ns) {StartInitialHandshake:157(47168645ns) {DropHandshakes:470(297ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(1479ns) {SwitchToState:179(575ns)}}}}}
2025-12-24T02:11:09.399224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241273874714380:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.403085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683102:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.403165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241274256683035:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.406824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241274086812784:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35852317ns) {Configure:104(35850335ns) {ProcessConfigured:135(35461652ns) {StartInitialHandshake:157(35460602ns) {DropHandshakes:470(480ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(1757ns) {SwitchToState:179(892ns)}}}}}
2025-12-24T02:11:09.406891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274086812849:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.407131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274086812784:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.407310Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241274086812782:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56370198ns) {Configure:104(56369353ns) {ProcessConfigured:135(55677877ns) {StartInitialHandshake:157(55677673ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(141ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(710ns) {SwitchToState:179(17ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.407572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348103:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.410759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274086812814:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:09.408082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683108:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.408097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241274256683108:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.410851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531937:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.410867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276094531937:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.410916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531935:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.410923Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276094531935:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.410939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348103:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.411029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274909348026:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.410839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812857:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.411236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864763:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.411692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274086812857:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.411724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274086812782:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.411222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241274256683082:2080] [node 5] ICP01 ready to work
2025-12-24T02:11:09.414758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864755:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.414777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272580864755:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.414846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683103:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.415949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272580864755:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.416033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272580864750:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.416047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.416055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272580864750:2111] [node 6] ICP22 created new session: [1:7587241272580864773:2111]
2025-12-24T02:11:09.416063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272580864773:2111] [node 6] ICS09 handshake done sender: [1:7587241272580864755:2113] self: [1:7587241272580864754:0] peer: [6:7587241274086812859:0] socket: 455 qp: -1
2025-12-24T02:11:09.416070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272580864773:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:09.416110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.415537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
2025-12-24T02:11:09.415591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531948:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
2025-12-24T02:11:09.415624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241276094531949:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54883
2025-12-24T02:11:09.415829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531948:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.418836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241276094531860:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44053845ns) {HandleHandshakeStatus:329(44052870ns) {DropIncomingHandshake:442(282ns) DropOutgoingHandshake:457(1490ns) SwitchToState:179(207ns) ProcessPendingSessionEvents:495(32032363ns) {ForwardSessionEventToSession:612(32020047ns) ForwardSessionEventToSession:612(1474ns) ForwardSessionEventToSession:612(1113ns) ForwardSessionEventToSession:612(880ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(1224ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(787ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(1238ns)}}}
2025-12-24T02:11:09.418864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241276094531860:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:09.418918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531947:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.419966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241274086812830:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:09.419525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241274909348069:2071] [node 6] ICP01 ready to work
2025-12-24T02:11:09.422731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683118:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
2025-12-24T02:11:09.422762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241274256683101:2086] [node 4] ICP01 ready to work
2025-12-24T02:11:09.422963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241274256683108:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.422615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272580864679:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.422676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.422687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272580864679:2049] [node 3] ICP22 created new session: [1:7587241272580864775:2049]
2025-12-24T02:11:09.422710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272580864775:2049] [node 3] ICS09 handshake done sender: [1:7587241272580864743:2109] self: [1:7587241272580864742:0] peer: [3:7587241274256683104:0] socket: 422 qp: -1
2025-12-24T02:11:09.422719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272580864775:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.422743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.422763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864757:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.422777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241272580864757:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.423359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241272580864777:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18714
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.423522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.423583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812862:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.425555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812863:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.426852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276094531937:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.426897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276094531935:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.427073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276094531925:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.427090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.427097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276094531925:2109] [node 5] ICP22 created new session: [2:7587241276094531954:2109]
2025-12-24T02:11:09.427103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276094531954:2109] [node 5] ICS09 handshake done sender: [2:7587241276094531937:2114] self: [2:7587241276094531931:0] peer: [5:7587241273874714457:0] socket: 438 qp: -1
2025-12-24T02:11:09.427110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276094531954:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:09.427119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.434944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241272580864679:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(64754701ns) {HandleHandshakeStatus:329(64753274ns) {DropIncomingHandshake:442(24232ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(211ns) ProcessPendingSessionEvents:495(12112199ns) {ForwardSessionEventToSession:612(12095840ns) ForwardSessionEventToSession:612(2352ns) ForwardSessionEventToSession:612(1233ns) ForwardSessionEventToSession:612(756ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(1611ns) ForwardSessionEventToSession:612(963ns) ForwardSessionEventToSession:612(697ns) ForwardSessionEventToSession:612(933ns) ForwardSessionEventToSession:612(1562ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.438947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348108:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.442835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241272580864757:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.442975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241272580864745:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.443001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.443010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241272580864745:2108] [node 4] ICP22 created new session: [1:7587241272580864779:2108]
2025-12-24T02:11:09.443017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241272580864779:2108] [node 4] ICS09 handshake done sender: [1:7587241272580864757:2115] self: [1:7587241272580864748:0] peer: [4:7587241274909348105:0] socket: 426 qp: -1
2025-12-24T02:11:09.443024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241272580864779:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:09.443037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.442913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274086812861:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.443359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274086812814:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.443399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241274086812814:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32542690ns) {Configure:104(32542263ns) {ProcessConfigured:135(32164332ns) {StartInitialHandshake:157(32139804ns) {DropHandshakes:470(817ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(444ns) PrepareNewSessionHandshake:477(955ns) {SwitchToState:179(576ns)}} IncomingHandshake:261(17143ns) {DropIncomingHandshake:442(54ns) PrepareNewSessionHandshake:477(196ns) {SwitchToState:179(169ns)} GenerateSessionVirtualId:988(203ns) IssueIncomingHandshakeReply:193(11266ns) {SwitchToState:179(37ns)}}}}}
2025-12-24T02:11:09.443488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274086812783:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.443097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241274256683101:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20224051ns) {Configure:104(20223667ns) {ProcessConfigured:135(19879623ns) {StartInitialHandshake:157(19859048ns) {DropHandshakes:470(401ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(769ns) {SwitchToState:179(125ns)}} IncomingHandshake:261(8700ns) {DropIncomingHandshake:442(128ns) PrepareNewSessionHandshake:477(195ns) {SwitchToState:179(143ns)} GenerateSessionVirtualId:988(196ns) IssueIncomingHandshakeReply:193(579ns)}}}}
2025-12-24T02:11:09.446761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274909348103:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.446807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241274909348026:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241274909348103:2114] held: no
2025-12-24T02:11:09.446727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241274256683088:2082] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.446758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.446772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241274256683088:2082] [node 6] ICP22 created new session: [3:7587241274256683125:2082]
2025-12-24T02:11:09.446780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274256683125:2082] [node 6] ICS09 handshake done sender: [3:7587241274256683108:2112] self: [3:7587241274256683090:0] peer: [6:7587241274086812854:0] socket: 445 qp: -1
2025-12-24T02:11:09.446789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274256683125:2082] [node 6] ICS10 traffic start
2025-12-24T02:11:09.446817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.448753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274086812857:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.448922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241273874714427:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.448936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241273874714427:2089] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.449958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241273874714380:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241273874714450:2108] held: no
2025-12-24T02:11:09.449995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714465:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.450379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241273874714427:2089] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.450411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241273874714450:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.450475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241273874714380:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.450489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.450495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241273874714380:2049] [node 2] ICP22 created new session: [5:7587241273874714467:2049]
2025-12-24T02:11:09.450502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241273874714467:2049] [node 2] ICS09 handshake done sender: [5:7587241273874714450:2108] self: [5:7587241273874714457:0] peer: [2:7587241276094531931:0] socket: 458 qp: -1
2025-12-24T02:11:09.450508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241273874714467:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.450527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.447252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531940:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.447268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276094531940:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.448053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276094531940:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.448151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276094531926:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.448164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.448173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276094531926:2110] [node 6] ICP22 created new session: [2:7587241276094531958:2110]
2025-12-24T02:11:09.448179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276094531958:2110] [node 6] ICS09 handshake done sender: [2:7587241276094531940:2115] self: [2:7587241276094531930:0] peer: [6:7587241274086812867:0] socket: 462 qp: -1
2025-12-24T02:11:09.448185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276094531958:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:09.448194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.450999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241274256683127:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13143
2025-12-24T02:11:09.452399Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241274256683102:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.450784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241273874714420:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.450804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.450812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241273874714420:2073] [node 6] ICP22 created new session: [5:7587241273874714469:2073]
2025-12-24T02:11:09.452474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241274256683035:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.450818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241273874714469:2073] [node 6] ICS09 handshake done sender: [5:7587241273874714427:2089] self: [5:7587241273874714426:0] peer: [6:7587241274086812866:0] socket: 446 qp: -1
2025-12-24T02:11:09.452487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.450824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241273874714469:2073] [node 6] ICS10 traffic start
2025-12-24T02:11:09.452493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241274256683035:2049] [node 2] ICP22 created new session: [3:7587241274256683128:2049]
2025-12-24T02:11:09.450836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.452498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274256683128:2049] [node 2] ICS09 handshake done sender: [3:7587241274256683102:2107] self: [3:7587241274256683117:0] peer: [2:7587241276094531929:0] socket: 436 qp: -1
2025-12-24T02:11:09.452502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274256683128:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.452531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.452298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276094531861:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.452327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.452334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276094531861:2049] [node 3] ICP22 created new session: [2:7587241276094531960:2049]
2025-12-24T02:11:09.452341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276094531960:2049] [node 3] ICS09 handshake done sender: [2:7587241276094531935:2113] self: [2:7587241276094531929:0] peer: [3:7587241274256683117:0] socket: 435 qp: -1
2025-12-24T02:11:09.452346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276094531960:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.452367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.454750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274909348026:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.454768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.454776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274909348026:2048] [node 1] ICP22 created new session: [4:7587241274909348109:2048]
2025-12-24T02:11:09.454784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274909348109:2048] [node 1] ICS09 handshake done sender: [4:7587241274909348103:2114] self: [4:7587241274909348105:0] peer: [1:7587241272580864748:0] socket: 433 qp: -1
2025-12-24T02:11:09.454791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274909348109:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.454852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.457434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274086812782:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.457457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.457464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274086812782:2048] [node 1] ICP22 created new session: [6:7587241274086812871:2048]
2025-12-24T02:11:09.457472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274086812871:2048] [node 1] ICS09 handshake done sender: [6:7587241274086812857:2113] self: [6:7587241274086812859:0] peer: [1:7587241272580864754:0] socket: 457 qp: -1
2025-12-24T02:11:09.457479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274086812871:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.457487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.457534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348101:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.457546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241274909348101:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.455708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714471:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.457333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241273874714471:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.457377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241273874714409:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.462794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241272580864777:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.462888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241272580864750:2111] [node 6] ICP18 handshake (actor [1:7587241272580864777:2118]) is too old
2025-12-24T02:11:09.462882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241274256683082:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51551556ns) {Configure:104(51550996ns) {ProcessConfigured:135(51216014ns) {StartInitialHandshake:157(51191425ns) {DropHandshakes:470(355ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(338ns) PrepareNewSessionHandshake:477(2366ns) {SwitchToState:179(1514ns)}} IncomingHandshake:261(10289ns) {DropIncomingHandshake:442(213ns) PrepareNewSessionHandshake:477(195ns) {SwitchToState:179(123ns)} GenerateSessionVirtualId:988(320ns) IssueIncomingHandshakeReply:193(478ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.463413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274086812848:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.463522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241274086812830:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(43475198ns) {Configure:104(43475038ns) {ProcessConfigured:135(43186021ns) {StartInitialHandshake:157(43185495ns) {DropHandshakes:470(310ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(580ns) {SwitchToState:179(177ns)}}}}}
2025-12-24T02:11:09.463598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812869:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.466868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812870:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.466869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274086812784:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.466888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.466897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274086812784:2050] [node 3] ICP22 created new session: [6:7587241274086812874:2050]
2025-12-24T02:11:09.466904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274086812874:2050] [node 3] ICS09 handshake done sender: [6:7587241274086812848:2109] self: [6:7587241274086812854:0] peer: [3:7587241274256683090:0] socket: 452 qp: -1
2025-12-24T02:11:09.466914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274086812874:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.466935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.466883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531949:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.466940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241276094531926:2110] [node 6] ICP18 handshake (actor [2:7587241276094531949:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.467183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274086812861:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.470834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241276094531925:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(43722130ns) {HandleHandshakeStatus:329(43721786ns) {DropIncomingHandshake:442(3730ns) DropOutgoingHandshake:457(102ns) SwitchToState:179(193ns) ProcessPendingSessionEvents:495(43638195ns) {ForwardSessionEventToSession:612(43621297ns) ForwardSessionEventToSession:612(2484ns) ForwardSessionEventToSession:612(812ns) ForwardSessionEventToSession:612(2053ns) ForwardSessionEventToSession:612(578ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(687ns) ForwardSessionEventToSession:612(1180ns) ForwardSessionEventToSession:612(1169ns)}}}
2025-12-24T02:11:09.471355Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241274256683088:2082] [node 6] ICP35 event processing took too much time PendingConnection:279(46635441ns) {HandleHandshakeStatus:329(46634331ns) {DropIncomingHandshake:442(524ns) DropOutgoingHandshake:457(1169ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(19052110ns) {ForwardSessionEventToSession:612(19038357ns) ForwardSessionEventToSession:612(1442ns) ForwardSessionEventToSession:612(901ns) ForwardSessionEventToSession:612(962ns) ForwardSessionEventToSession:612(639ns) ForwardSessionEventToSession:612(959ns) ForwardSessionEventToSession:612(1235ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(688ns) ForwardSessionEventToSession:612(1024ns)}}}
2025-12-24T02:11:09.474898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274086812783:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.474920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.474927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274086812783:2049] [node 2] ICP22 created new session: [6:7587241274086812876:2049]
2025-12-24T02:11:09.474933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274086812876:2049] [node 2] ICS09 handshake done sender: [6:7587241274086812861:2114] self: [6:7587241274086812867:0] peer: [2:7587241276094531930:0] socket: 467 qp: -1
2025-12-24T02:11:09.474940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274086812876:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.474947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.483392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274909348101:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.483489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241274909348068:2064] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.483511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.483522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274909348068:2064] [node 5] ICP22 created new session: [4:7587241274909348114:2064]
2025-12-24T02:11:09.483531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274909348114:2064] [node 5] ICS09 handshake done sender: [4:7587241274909348101:2113] self: [4:7587241274909348083:0] peer: [5:7587241273874714472:0] socket: 444 qp: -1
2025-12-24T02:11:09.483540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274909348114:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:09.483550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.489305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348112:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.489685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348112:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.489731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274909348028:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.490816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241273874714420:2073] [node 6] ICP35 event processing took too much time PendingConnection:279(39986896ns) {HandleHandshakeStatus:329(39986186ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(154ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(38171183ns) {ForwardSessionEventToSession:612(38156303ns) ForwardSessionEventToSession:612(2637ns) ForwardSessionEventToSession:612(1135ns) ForwardSessionEventToSession:612(724ns) ForwardSessionEventToSession:612(801ns) ForwardSessionEventToSession:612(1487ns) ForwardSessionEventToSession:612(1080ns) ForwardSessionEventToSession:612(586ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(1203ns)}}}
2025-12-24T02:11:09.491078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241274256683035:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(38572988ns) {HandleHandshakeStatus:329(38572692ns) {DropIncomingHandshake:442(152ns) DropOutgoingHandshake:457(1087ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(20212110ns) {ForwardSessionEventToSession:612(20199475ns) ForwardSessionEventToSession:612(1268ns) ForwardSessionEventToSession:612(960ns) ForwardSessionEventToSession:612(668ns) ForwardSessionEventToSession:612(604ns) ForwardSessionEventToSession:612(921ns) ForwardSessionEventToSession:612(1226ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(806ns) ForwardSessionEventToSession:612(1028ns)}}}
2025-12-24T02:11:09.492302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241276094531861:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(65227926ns) {HandleHandshakeStatus:329(65226760ns) {DropIncomingHandshake:442(9573ns) DropOutgoingHandshake:457(431ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(21235506ns) {ForwardSessionEventToSession:612(21221265ns) ForwardSessionEventToSession:612(2190ns) ForwardSessionEventToSession:612(1349ns) ForwardSessionEventToSession:612(850ns) ForwardSessionEventToSession:612(950ns) ForwardSessionEventToSession:612(1455ns) ForwardSessionEventToSession:612(750ns) ForwardSessionEventToSession:612(715ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(1423ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.494884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714473:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.494969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274086812849:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.498727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.499095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241273874714473:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.499137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241273874714471:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.499177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241273874714381:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.499373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241273874714409:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.499394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.499401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241273874714409:2069] [node 4] ICP22 created new session: [5:7587241273874714476:2069]
2025-12-24T02:11:09.499407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241273874714476:2069] [node 4] ICS09 handshake done sender: [5:7587241273874714471:2119] self: [5:7587241273874714472:0] peer: [4:7587241274909348083:0] socket: 463 qp: -1
2025-12-24T02:11:09.499414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241273874714476:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:09.499439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.498720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683122:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.498742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241274256683122:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.499455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241274086812783:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(24522469ns) {HandleHandshakeStatus:329(24521376ns) {DropIncomingHandshake:442(513ns) DropOutgoingHandshake:457(1209ns) SwitchToState:179(251ns) ProcessPendingSessionEvents:495(12345072ns) {ForwardSessionEventToSession:612(12331606ns) ForwardSessionEventToSession:612(2600ns) ForwardSessionEventToSession:612(1389ns) ForwardSessionEventToSession:612(669ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(874ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(510ns) ForwardSessionEventToSession:612(959ns)}}}
2025-12-24T02:11:09.500454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274086812814:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.500464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.500468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274086812814:2066] [node 5] ICP22 created new session: [6:7587241274086812881:2066]
2025-12-24T02:11:09.500475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274086812881:2066] [node 5] ICS09 handshake done sender: [6:7587241274086812849:2110] self: [6:7587241274086812866:0] peer: [5:7587241273874714426:0] socket: 453 qp: -1
2025-12-24T02:11:09.500480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274086812881:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:09.500484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.503220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348118:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.503270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348119:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.505247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241274256683088:2082] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:09.503351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812878:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.507478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348118:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.507686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241274909348027:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.507710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348120:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.510844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241274256683130:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.510860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241274256683130:2121] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.511229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348119:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.511632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714478:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.511967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241276094531944:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.511981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241276094531944:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.510955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241274086812878:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.511004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241274086812830:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.511470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241274256683122:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.514737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241274256683101:2086] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.514768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.514778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241274256683101:2086] [node 4] ICP22 created new session: [3:7587241274256683134:2086]
2025-12-24T02:11:09.514785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274256683134:2086] [node 4] ICS09 handshake done sender: [3:7587241274256683122:2119] self: [3:7587241274256683121:0] peer: [4:7587241274909348117:0] socket: 482 qp: -1
2025-12-24T02:11:09.514794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274256683134:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:09.514807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.514934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241274909348107:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.514948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241274909348107:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.515848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274909348107:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.515927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241274909348069:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.515946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.515952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274909348069:2071] [node 6] ICP22 created new session: [4:7587241274909348124:2071]
2025-12-24T02:11:09.515957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274909348124:2071] [node 6] ICS09 handshake done sender: [4:7587241274909348107:2116] self: [4:7587241274909348106:0] peer: [6:7587241274086812883:0] socket: 469 qp: -1
2025-12-24T02:11:09.515962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274909348124:2071] [node 6] ICS10 traffic start
2025-12-24T02:11:09.515968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.516481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274909348112:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.516584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274909348028:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.516599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.516604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274909348028:2050] [node 3] ICP22 created new session: [4:7587241274909348126:2050]
2025-12-24T02:11:09.516609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274909348126:2050] [node 3] ICS09 handshake done sender: [4:7587241274909348112:2118] self: [4:7587241274909348117:0] peer: [3:7587241274256683121:0] socket: 451 qp: -1
2025-12-24T02:11:09.516614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274909348126:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.516622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.518175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274086812830:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64832) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241274086812878:2123] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.519417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241273874714479:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56901
2025-12-24T02:11:09.519900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241273874714473:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.520021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241273874714381:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.520035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.520041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241273874714381:2050] [node 3] ICP22 created new session: [5:7587241273874714480:2050]
2025-12-24T02:11:09.520047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241273874714480:2050] [node 3] ICS09 handshake done sender: [5:7587241273874714473:2120] self: [5:7587241273874714475:0] peer: [3:7587241274256683119:0] socket: 489 qp: -1
2025-12-24T02:11:09.520052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241273874714480:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.520060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.518246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241274086812884:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63505
2025-12-24T02:11:09.520201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241274086812878:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.520257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241274086812830:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.520264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.520269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241274086812830:2070] [node 4] ICP22 created new session: [6:7587241274086812885:2070]
2025-12-24T02:11:09.520275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241274086812885:2070] [node 4] ICS09 handshake done sender: [6:7587241274086812878:2123] self: [6:7587241274086812883:0] peer: [4:7587241274909348106:0] socket: 491 qp: -1
2025-12-24T02:11:09.520278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241274086812885:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:09.520284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.519820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241274256683130:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.520423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241274256683082:2080] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.520434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.520439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241274256683082:2080] [node 5] ICP22 created new session: [3:7587241274256683137:2080]
2025-12-24T02:11:09.520443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241274256683137:2080] [node 5] ICS09 handshake done sender: [3:7587241274256683130:2121] self: [3:7587241274256683119:0] peer: [5:7587241273874714475:0] socket: 488 qp: -1
2025-12-24T02:11:09.520446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241274256683137:2080] [node 5] ICS10 traffic start
2025-12-24T02:11:09.520451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.522900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241276094531944:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.523058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241276094531924:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.523076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.523082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241276094531924:2108] [node 4] ICP22 created new session: [2:7587241276094531963:2108]
2025-12-24T02:11:09.523088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241276094531963:2108] [node 4] ICS09 handshake done sender: [2:7587241276094531944:2117] self: [2:7587241276094531928:0] peer: [4:7587241274909348121:0] socket: 472 qp: -1
2025-12-24T02:11:09.523093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241276094531963:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:09.523100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.534831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241274256683101:2086] [node 4] ICP35 event processing took too much time PendingConnection:279(20040409ns) {HandleHandshakeStatus:329(20038444ns) {DropIncomingHandshake:442(6901ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(225ns) ProcessPendingSessionEvents:495(19911641ns) {ForwardSessionEventToSession:612(19893407ns) ForwardSessionEventToSession:612(3070ns) ForwardSessionEventToSession:612(759ns) ForwardSessionEventToSession:612(3299ns) ForwardSessionEventToSession:612(748ns) ForwardSessionEventToSession:612(1364ns) ForwardSessionEventToSession:612(850ns) ForwardSessionEventToSession:612(752ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(1557ns)}}}
2025-12-24T02:11:09.535199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241274909348128:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16637
2025-12-24T02:11:09.543231Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241274909348118:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.547009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241274909348027:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.547037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.547047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241274909348027:2049] [node 2] ICP22 created new session: [4:7587241274909348129:2049]
2025-12-24T02:11:09.547054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241274909348129:2049] [node 2] ICS09 handshake done sender: [4:7587241274909348118:2119] self: [4:7587241274909348121:0] peer: [2:7587241276094531928:0] socket: 487 qp: -1
2025-12-24T02:11:09.547067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241274909348129:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.547084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.566776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:09.566928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241274256683111:2048] [node 1] ICS01 socket: 425 reason# EndOfStream
2025-12-24T02:11:09.566939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241274256683034:2048] [node 1] ICP30 unregister session Session# [3:7587241274256683111:2048] VirtualId# [3:7587241274256683104:0]
2025-12-24T02:11:09.566946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241274256683111:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.566984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.567010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.566776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:09.566888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241276094531942:2048] [node 1] ICS07 socket disconnect 421 reason# EndOfStream
2025-12-24T02:11:09.566897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241276094531942:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.566958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.566984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241276094531942:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:09.567356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241276094531860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.567369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241276094531942:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.567373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241276094531860:2048] [node 1] ICP30 unregister session Session# [2:7587241276094531942:2048] VirtualId# [2:7587241276094531939:0]
2025-12-24T02:11:09.567389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.567395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241276094531860:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.567398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.574846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:09.574937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274086812871:2048] [node 1] ICS01 socket: 457 reason# EndOfStream
2025-12-24T02:11:09.574944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274086812782:2048] [node 1] ICP30 unregister session Session# [6:7587241274086812871:2048] VirtualId# [6:7587241274086812859:0]
2025-12-24T02:11:09.574950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274086812871:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.575008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.575031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.574846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:09.574881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:09.574948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241273874714454:2048] [node 1] ICS07 socket disconnect 430 reason# EndOfStream
2025-12-24T02:11:09.574954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241273874714454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.575008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.575030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241273874714454:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:09.575810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241273874714379:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.575820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241273874714454:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.575824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241273874714379:2048] [node 1] ICP30 unregister session Session# [5:7587241273874714454:2048] VirtualId# [5:7587241273874714452:0]
2025-12-24T02:11:09.575834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.575838Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241273874714379:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.575840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6729) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.575548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241274909348109:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:09.575559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241274909348026:2048] [node 1] ICP30 unregister session Session# [4:7587241274909348109:2048] VirtualId# [4:7587241274909348105:0]
2025-12-24T02:11:09.575564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241274909348109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.575601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.575622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.586800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:09.586893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241274256683128:2049] [node 2] ICS01 socket: 436 reason# EndOfStream
2025-12-24T02:11:09.586901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241274256683035:2049] [node 2] ICP30 unregister session Session# [3:7587241274256683128:2049] VirtualId# [3:7587241274256683117:0]
2025-12-24T02:11:09.586908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241274256683128:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.586958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.586982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.590801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241273874714379:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:09.614072Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:09.614178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241274086812876:2049] [node 2] ICS07 socket disconnect 467 reason# EndOfStream
2025-12-24T02:11:09.614186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274086812876:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.614240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.614261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241274086812876:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:09.614856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241274086812783:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:09.614872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274086812876:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:09.614877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274086812783:2049] [node 2] ICP30 unregister session Session# [6:7587241274086812876:2049] VirtualId# [6:7587241274086812867:0]
2025-12-24T02:11:09.614894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.614901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241274086812783:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.614904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:09.617880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:09.617957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241274909348129:2049] [node 2] ICS01 socket: 487 reason# EndOfStream
2025-12-24T02:11:09.617963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241274909348027:2049] [node 2] ICP30 unregister session Session# [4:7587241274909348129:2049] VirtualId# [4:7587241274909348121:0]
2025-12-24T02:11:09.617969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241274909348129:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.618023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.618050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.618089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:09.618153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241273874714467:2049] [node 2] ICS01 socket: 458 reason# EndOfStream
2025-12-24T02:11:09.618159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241273874714380:2049] [node 2] ICP30 unregister session Session# [5:7587241273874714467:2049] VirtualId# [5:7587241273874714457:0]
2025-12-24T02:11:09.618165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241273874714467:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.618197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.618217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.626932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241274086812783:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:09.630789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:09.630896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274086812874:2050] [node 3] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:09.630909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274086812784:2050] [node 3] ICP30 unregister session Session# [6:7587241274086812874:2050] VirtualId# [6:7587241274086812854:0]
2025-12-24T02:11:09.630917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274086812874:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.630987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.631013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.634346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:09.634432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241273874714480:2050] [node 3] ICS01 socket: 489 reason# EndOfStream
2025-12-24T02:11:09.634439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241273874714381:2050] [node 3] ICP30 unregister session Session# [5:7587241273874714480:2050] VirtualId# [5:7587241273874714475:0]
2025-12-24T02:11:09.634445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241273874714480:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.634506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:09.634532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:09.645808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:09.645914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241274086812885:2070] [node 4] ICS01 socket: 491 reason# EndOfStream
2025-12-24T02:11:09.645924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241274086812830:2070] [node 4] ICP30 unregister session Session# [6:7587241274086812885:2070] VirtualId# [6:7587241274086812883:0]
2025-12-24T02:11:09.645931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241274086812885:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.645979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:09.646003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:09.695769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241273196585211:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.695785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241273196585212:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.695790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241273196585213:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.690833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241272482365927:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.690848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241272482365928:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.698787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275945514998:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.698803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275945514999:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.694911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241274851964601:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:09.694923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241274851964602:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:09.694832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241274515778734:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.694842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241274515778735:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.694847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241274515778736:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.706921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241275329903114:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:09.706935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241275329903115:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:09.706939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241275329903116:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:09.718561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241272482365979:2098] [node 4] ICP01 ready to work
2025-12-24T02:11:09.722908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241274851964662:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:09.723274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241274515778756:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:09.727126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241275329903149:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:09.727215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241275329903150:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:09.724046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241274851964671:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:09.724143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241274851964672:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:09.719050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275945515055:2100] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.731122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241272482365990:2101] [node 6] ICP01 ready to work
2025-12-24T02:11:09.731343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241272482365993:2103] [node 5] ICP01 ready to work
2025-12-24T02:11:09.732431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241272482365991:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49984
2025-12-24T02:11:09.732449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241272482365992:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49984
2025-12-24T02:11:09.731005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241274515778790:2076] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.734976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241273196585266:2094] [node 6] ICP01 ready to work
2025-12-24T02:11:09.735084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241273196585267:2098] [node 5] ICP01 ready to work
2025-12-24T02:11:09.736181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585268:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.748167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903180:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.748193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903181:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.748205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903182:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.748217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903183:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.758879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241274515778734:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27307714ns) {Configure:104(27306790ns) {ProcessConfigured:135(27035170ns) {StartInitialHandshake:157(27033933ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(110ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(2853ns) {SwitchToState:179(1131ns)}}}}}
2025-12-24T02:11:09.772904Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241275945515055:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(53764389ns) {Configure:104(53763619ns) {ProcessConfigured:135(53372958ns) {StartInitialHandshake:157(53371641ns) {DropHandshakes:470(308ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:477(2124ns) {SwitchToState:179(1329ns)}}}}}
2025-12-24T02:11:09.773172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275945515064:2104] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.783096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241275945515069:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:09.784298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515065:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
2025-12-24T02:11:09.784316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515066:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
2025-12-24T02:11:09.784328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515067:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
2025-12-24T02:11:09.784341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515068:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.793723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778811:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.793751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778812:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.800982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241272482366008:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49984
2025-12-24T02:11:09.805077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241275329903182:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.805136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241275329903115:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.805995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903197:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.806293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241275329903182:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.805283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482365995:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.805296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241272482365995:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.806098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241272482365995:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.806348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241275329903115:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.806372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.806378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241275329903115:2049] [node 2] ICP22 created new session: [6:7587241275329903198:2049]
2025-12-24T02:11:09.806385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241275329903198:2049] [node 2] ICS09 handshake done sender: [6:7587241275329903182:2112] self: [6:7587241275329903196:0] peer: [2:7587241272482365994:0] socket: 445 qp: -1
2025-12-24T02:11:09.806396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241275329903198:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.806417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.808436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515066:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.807048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241272482365990:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.807066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.807072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241272482365990:2101] [node 6] ICP22 created new session: [2:7587241272482366010:2101]
2025-12-24T02:11:09.807077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241272482366010:2101] [node 6] ICS09 handshake done sender: [2:7587241272482365995:2110] self: [2:7587241272482365994:0] peer: [6:7587241275329903196:0] socket: 435 qp: -1
2025-12-24T02:11:09.807083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241272482366010:2101] [node 6] ICS10 traffic start
2025-12-24T02:11:09.807092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.818814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241275329903180:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.818924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241275329903149:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.819322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585290:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.819351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.819493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241275329903183:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.819542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241275329903150:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.819625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585291:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.819651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585272:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.819657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241273196585272:2103] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.820281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903203:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.820370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241273196585272:2103] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.820561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241275329903183:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.820623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241275329903150:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.820641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.820647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241275329903150:2070] [node 4] ICP22 created new session: [6:7587241275329903204:2070]
2025-12-24T02:11:09.820654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241275329903204:2070] [node 4] ICS09 handshake done sender: [6:7587241275329903183:2113] self: [6:7587241275329903202:0] peer: [4:7587241273196585271:0] socket: 446 qp: -1
2025-12-24T02:11:09.820661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241275329903204:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:09.820670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.821487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241273196585266:2094] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.821503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.821508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241273196585266:2094] [node 6] ICP22 created new session: [4:7587241273196585294:2094]
2025-12-24T02:11:09.821514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273196585294:2094] [node 6] ICS09 handshake done sender: [4:7587241273196585272:2103] self: [4:7587241273196585271:0] peer: [6:7587241275329903202:0] socket: 441 qp: -1
2025-12-24T02:11:09.821518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273196585294:2094] [node 6] ICS10 traffic start
2025-12-24T02:11:09.821524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.821951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241275329903181:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.821972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241275329903114:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.821900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241274851964680:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45862
2025-12-24T02:11:09.821939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241274851964681:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45862
2025-12-24T02:11:09.821963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241274851964682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45862
2025-12-24T02:11:09.821973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241274851964683:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45862
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.822805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482365991:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.823008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241272482366013:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49984
2025-12-24T02:11:09.826832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585268:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.826902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241273196585212:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.827010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482366013:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.830731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482365989:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.830743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241272482365989:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.835153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241272482365989:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.834869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964666:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.834891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241274851964666:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.835463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964681:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.835483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964680:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.835556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964683:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.835769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241274851964666:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.835837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241274851964662:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.835854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.835862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241274851964662:2107] [node 6] ICP22 created new session: [1:7587241274851964688:2107]
2025-12-24T02:11:09.835868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241274851964688:2107] [node 6] ICS09 handshake done sender: [1:7587241274851964666:2110] self: [1:7587241274851964665:0] peer: [6:7587241275329903206:0] socket: 428 qp: -1
2025-12-24T02:11:09.835876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241274851964688:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:09.835886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.835057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778811:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.840397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241272482365979:2098] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.840418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.840426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241272482365979:2098] [node 4] ICP22 created new session: [2:7587241272482366017:2098]
2025-12-24T02:11:09.840433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241272482366017:2098] [node 4] ICS09 handshake done sender: [2:7587241272482365989:2107] self: [2:7587241272482365984:0] peer: [4:7587241273196585297:0] socket: 434 qp: -1
2025-12-24T02:11:09.840439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241272482366017:2098] [node 4] ICS10 traffic start
2025-12-24T02:11:09.840450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.840606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482365992:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.840890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.841084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241272482365927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.841246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241275329903114:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9907) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241275329903181:2111] held: no
2025-12-24T02:11:09.841407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903207:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.839342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778760:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.842742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903208:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.839354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241274515778760:2070] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.841539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241274515778760:2070] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.842893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964668:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.842905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241274851964668:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.843610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241274851964668:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.846879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241275329903181:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.846968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241272482366021:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49984
2025-12-24T02:11:09.847124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241272482365992:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.847185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241275329903180:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.847241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241275329903114:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.847257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.847262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241275329903114:2048] [node 1] ICP22 created new session: [6:7587241275329903209:2048]
2025-12-24T02:11:09.847268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241275329903209:2048] [node 1] ICS09 handshake done sender: [6:7587241275329903181:2111] self: [6:7587241275329903206:0] peer: [1:7587241274851964665:0] socket: 444 qp: -1
2025-12-24T02:11:09.847275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241275329903209:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.847294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.847403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241275329903149:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.847412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.847415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241275329903149:2064] [node 5] ICP22 created new session: [6:7587241275329903211:2064]
2025-12-24T02:11:09.847419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241275329903211:2064] [node 5] ICS09 handshake done sender: [6:7587241275329903180:2110] self: [6:7587241275329903201:0] peer: [5:7587241274515778759:0] socket: 431 qp: -1
2025-12-24T02:11:09.847424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241275329903211:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:09.847429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.848030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241272482365927:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.848044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.848049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241272482365927:2048] [node 1] ICP22 created new session: [2:7587241272482366022:2048]
2025-12-24T02:11:09.848054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241272482366022:2048] [node 1] ICS09 handshake done sender: [2:7587241272482365992:2109] self: [2:7587241272482366020:0] peer: [1:7587241274851964667:0] socket: 433 qp: -1
2025-12-24T02:11:09.848059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241272482366022:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.848065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.851991Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241273196585212:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241273196585268:2101] held: no
2025-12-24T02:11:09.852050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585290:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.852118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241273196585268:2101] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.852272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241273196585212:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.852285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.852292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241273196585212:2049] [node 2] ICP22 created new session: [4:7587241273196585302:2049]
2025-12-24T02:11:09.852297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273196585302:2049] [node 2] ICS09 handshake done sender: [4:7587241273196585268:2101] self: [4:7587241273196585297:0] peer: [2:7587241272482365984:0] socket: 438 qp: -1
2025-12-24T02:11:09.852301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273196585302:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.852306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.852420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.851301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241274851964601:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.851352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.851035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515068:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.851072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515067:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.851371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241274515778756:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.851218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241275945514999:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.851389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.851395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241274515778756:2064] [node 6] ICP22 created new session: [5:7587241274515778817:2064]
2025-12-24T02:11:09.851401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241274515778817:2064] [node 6] ICS09 handshake done sender: [5:7587241274515778760:2070] self: [5:7587241274515778759:0] peer: [6:7587241275329903201:0] socket: 427 qp: -1
2025-12-24T02:11:09.851406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241274515778817:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:09.851412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.854813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482365997:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.854822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241272482365997:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.851359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241274851964601:2048] [node 2] ICP22 created new session: [1:7587241274851964692:2048]
2025-12-24T02:11:09.851366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241274851964692:2048] [node 2] ICS09 handshake done sender: [1:7587241274851964668:2111] self: [1:7587241274851964667:0] peer: [2:7587241272482366020:0] socket: 429 qp: -1
2025-12-24T02:11:09.856541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515065:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.851372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241274851964692:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:09.856574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241275945514998:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.851380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.856629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515084:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
2025-12-24T02:11:09.856042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778812:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.855625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964682:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.856233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241274515778736:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.856309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778819:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.857028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964670:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.856346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778820:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.856359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778816:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.857032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241274851964670:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.857420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241274851964696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45862
2025-12-24T02:11:09.857497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241274851964670:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.857552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241274851964602:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.857563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.857568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241274851964602:2049] [node 3] ICP22 created new session: [1:7587241274851964698:2049]
2025-12-24T02:11:09.857573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241274851964698:2049] [node 3] ICS09 handshake done sender: [1:7587241274851964670:2112] self: [1:7587241274851964669:0] peer: [3:7587241275945515085:0] socket: 430 qp: -1
2025-12-24T02:11:09.857578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241274851964698:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.857584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.858802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585299:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.858886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241273196585211:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.858751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515063:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.858760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275945515063:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.862801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241272482365997:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.862909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964674:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.862919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241274851964674:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.863547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241274851964674:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.863617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241274851964671:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.863631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.863637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241274851964671:2108] [node 4] ICP22 created new session: [1:7587241274851964701:2108]
2025-12-24T02:11:09.863641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241274851964701:2108] [node 4] ICS09 handshake done sender: [1:7587241274851964674:2113] self: [1:7587241274851964673:0] peer: [4:7587241273196585304:0] socket: 464 qp: -1
2025-12-24T02:11:09.863647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241274851964701:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:09.863653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.866755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778820:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.867456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241273196585211:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9907) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241273196585299:2119] held: no
2025-12-24T02:11:09.867511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585276:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.867514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241273196585276:2105] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.867245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241275945514998:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9907) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241275945515065:2111] held: no
2025-12-24T02:11:09.866826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241274515778734:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.866876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778819:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.866893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241274515778790:2076] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.871478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482366008:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.871492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241272482365928:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.871504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.871512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241272482365928:2049] [node 3] ICP22 created new session: [2:7587241272482366025:2049]
2025-12-24T02:11:09.871518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241272482366025:2049] [node 3] ICS09 handshake done sender: [2:7587241272482365997:2111] self: [2:7587241272482365996:0] peer: [3:7587241275945515081:0] socket: 436 qp: -1
2025-12-24T02:11:09.871526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241272482366025:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:09.871533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.871664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241272482365928:2049] [node 3] ICP18 handshake (actor [2:7587241272482366008:2117]) is too old
2025-12-24T02:11:09.870759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964696:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.870793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241274851964676:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.870797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241274851964676:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.878275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241274851964676:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.874758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241274515778816:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.874788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241274515778735:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.874947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778826:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.874958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778827:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.878338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241272482366001:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.878348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241272482366001:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.879134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.879410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241273196585276:2105] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.879497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241273196585267:2098] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.879516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.879524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241273196585267:2098] [node 5] ICP22 created new session: [4:7587241273196585307:2098]
2025-12-24T02:11:09.879530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273196585307:2098] [node 5] ICS09 handshake done sender: [4:7587241273196585276:2105] self: [4:7587241273196585275:0] peer: [5:7587241274515778824:0] socket: 448 qp: -1
2025-12-24T02:11:09.879536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273196585307:2098] [node 5] ICS10 traffic start
2025-12-24T02:11:09.879547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.879677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241273196585299:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.879900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241273196585211:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.879911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.879915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241273196585211:2048] [node 1] ICP22 created new session: [4:7587241273196585309:2048]
2025-12-24T02:11:09.879920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273196585309:2048] [node 1] ICS09 handshake done sender: [4:7587241273196585299:2119] self: [4:7587241273196585304:0] peer: [1:7587241274851964673:0] socket: 479 qp: -1
2025-12-24T02:11:09.879923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273196585309:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.879928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.883461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241274851964672:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.883483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.883490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241274851964672:2109] [node 5] ICP22 created new session: [1:7587241274851964705:2109]
2025-12-24T02:11:09.883496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241274851964705:2109] [node 5] ICS09 handshake done sender: [1:7587241274851964676:2114] self: [1:7587241274851964675:0] peer: [5:7587241274515778823:0] socket: 468 qp: -1
2025-12-24T02:11:09.883501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241274851964705:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:09.883509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.887118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275945515063:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.887237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241275945514999:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241275945515066:2112] held: no
2025-12-24T02:11:09.887276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275945515055:2100] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.887296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.887302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275945515055:2100] [node 5] ICP22 created new session: [3:7587241275945515089:2100]
2025-12-24T02:11:09.887308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275945515089:2100] [node 5] ICS09 handshake done sender: [3:7587241275945515063:2110] self: [3:7587241275945515056:0] peer: [5:7587241274515778822:0] socket: 456 qp: -1
2025-12-24T02:11:09.887314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275945515089:2100] [node 5] ICS10 traffic start
2025-12-24T02:11:09.887322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.887536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515086:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.887678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585311:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
2025-12-24T02:11:09.887823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275945515066:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.887901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241275945514999:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.887912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.887916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275945514999:2049] [node 2] ICP22 created new session: [3:7587241275945515091:2049]
2025-12-24T02:11:09.887921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275945515091:2049] [node 2] ICS09 handshake done sender: [3:7587241275945515066:2112] self: [3:7587241275945515081:0] peer: [2:7587241272482365996:0] socket: 453 qp: -1
2025-12-24T02:11:09.887926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275945515091:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.887931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.888035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275945515065:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.888213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241275945514998:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.888223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.888228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275945514998:2048] [node 1] ICP22 created new session: [3:7587241275945515094:2048]
2025-12-24T02:11:09.888232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275945515094:2048] [node 1] ICS09 handshake done sender: [3:7587241275945515065:2111] self: [3:7587241275945515085:0] peer: [1:7587241274851964669:0] socket: 452 qp: -1
2025-12-24T02:11:09.888236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275945515094:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.888242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.886840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241272482366001:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:09.886927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241272482365993:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.886942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:09.886948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241272482365993:2103] [node 5] ICP22 created new session: [2:7587241272482366028:2103]
2025-12-24T02:11:09.886954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241272482366028:2103] [node 5] ICS09 handshake done sender: [2:7587241272482366001:2113] self: [2:7587241272482366000:0] peer: [5:7587241274515778825:0] socket: 475 qp: -1
2025-12-24T02:11:09.886960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241272482366028:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:09.886969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:09.890810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241273196585311:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.890852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241273196585213:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.898853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903213:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.898840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515075:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.898856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275945515075:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.899548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241273196585313:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7252
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.899491Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241274515778735:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16579) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241274515778816:2117] held: no
2025-12-24T02:11:09.899567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241274515778790:2076] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21532) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241274515778819:2118] held: no
2025-12-24T02:11:09.900381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778828:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.900510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241274515778829:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62273
2025-12-24T02:11:09.900648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241274515778820:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:09.900759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241274515778816:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:09.900808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241274515778735:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.900821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:09.900828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241274515778735:2049] [node 2] ICP22 created new session: [5:7587241274515778831:2049]
2025-12-24T02:11:09.900835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241274515778831:2049] [node 2] ICS09 handshake done sender: [5:7587241274515778816:2117] self: [5:7587241274515778825:0] peer: [2:7587241272482366000:0] socket: 484 qp: -1
2025-12-24T02:11:09.900843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241274515778831:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:09.900851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:09.900981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241274515778734:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9907) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241274515778820:2119] held: no
2025-12-24T02:11:09.900994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241274515778734:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.900999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:09.901003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241274515778734:2048] [node 1] ICP22 created new session: [5:7587241274515778833:2048]
2025-12-24T02:11:09.901008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241274515778833:2048] [node 1] ICS09 handshake done sender: [5:7587241274515778820:2119] self: [5:7587241274515778823:0] peer: [1:7587241274851964675:0] socket: 472 qp: -1
2025-12-24T02:11:09.901011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241274515778833:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:09.901017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:09.901104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241274515778812:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.901148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241274515778819:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:09.902867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241275329903213:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.902910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241275329903116:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:09.902990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275945515075:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:09.904203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241275329903215:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18207
2025-12-24T02:11:09.904426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241275329903213:2123] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.903172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275945515064:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.903191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.903196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275945515064:2104] [node 4] ICP22 created new session: [3:7587241275945515098:2104]
2025-12-24T02:11:09.903203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275945515098:2104] [node 4] ICS09 handshake done sender: [3:7587241275945515075:2117] self: [3:7587241275945515074:0] peer: [4:7587241273196585312:0] socket: 432 qp: -1
2025-12-24T02:11:09.903209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275945515098:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:09.903217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.903530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241275945515077:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:09.903534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241275945515077:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:09.904011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241275945515097:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6269
2025-12-24T02:11:09.904295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241275945515077:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:09.904475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241275329903116:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.904489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.904494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241275329903116:2050] [node 3] ICP22 created new session: [6:7587241275329903216:2050]
2025-12-24T02:11:09.904499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241275329903216:2050] [node 3] ICS09 handshake done sender: [6:7587241275329903213:2123] self: [6:7587241275329903214:0] peer: [3:7587241275945515076:0] socket: 461 qp: -1
2025-12-24T02:11:09.904504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241275329903216:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.904511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.905770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241275945515069:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:09.905787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:09.905791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241275945515069:2107] [node 6] ICP22 created new session: [3:7587241275945515101:2107]
2025-12-24T02:11:09.905796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241275945515101:2107] [node 6] ICS09 handshake done sender: [3:7587241275945515077:2118] self: [3:7587241275945515076:0] peer: [6:7587241275329903214:0] socket: 487 qp: -1
2025-12-24T02:11:09.905801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241275945515101:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:09.905808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:09.906271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241273196585311:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:09.906351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241273196585213:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241273196585311:2121] held: no
2025-12-24T02:11:09.906370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241273196585213:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.906379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.906384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241273196585213:2050] [node 3] ICP22 created new session: [4:7587241273196585314:2050]
2025-12-24T02:11:09.906390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241273196585314:2050] [node 3] ICS09 handshake done sender: [4:7587241273196585311:2121] self: [4:7587241273196585312:0] peer: [3:7587241275945515074:0] socket: 458 qp: -1
2025-12-24T02:11:09.906396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241273196585314:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.906403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.906673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241274515778736:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.906707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:09.906713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241274515778736:2050] [node 3] ICP22 created new session: [5:7587241274515778835:2050]
2025-12-24T02:11:09.906719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241274515778835:2050] [node 3] ICS09 handshake done sender: [5:7587241274515778812:2116] self: [5:7587241274515778822:0] peer: [3:7587241275945515056:0] socket: 460 qp: -1
2025-12-24T02:11:09.906724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241274515778835:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:09.906731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:09.906857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241274515778790:2076] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:09.906866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:09.906870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241274515778790:2076] [node 4] ICP22 created new session: [5:7587241274515778837:2076]
2025-12-24T02:11:09.906874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241274515778837:2076] [node 4] ICS09 handshake done sender: [5:7587241274515778819:2118] self: [5:7587241274515778824:0] peer: [4:7587241273196585275:0] socket: 459 qp: -1
2025-12-24T02:11:09.906878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241274515778837:2076] [node 4] ICS10 traffic start
2025-12-24T02:11:09.906884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:09.919726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241275945515055:2100] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:09.934767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:09.934885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241272482366022:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:09.934899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241272482365927:2048] [node 1] ICP30 unregister session Session# [2:7587241272482366022:2048] VirtualId# [2:7587241272482366020:0]
2025-12-24T02:11:09.934908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241272482366022:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.934965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.934990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.954776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:09.954894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241275945515094:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:09.954904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241275945514998:2048] [node 1] ICP30 unregister session Session# [3:7587241275945515094:2048] VirtualId# [3:7587241275945515085:0]
2025-12-24T02:11:09.954913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241275945515094:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.954961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.954986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.969391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:09.969497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241275329903209:2048] [node 1] ICS01 socket: 444 reason# EndOfStream
2025-12-24T02:11:09.969508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241275329903114:2048] [node 1] ICP30 unregister session Session# [6:7587241275329903209:2048] VirtualId# [6:7587241275329903206:0]
2025-12-24T02:11:09.969516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241275329903209:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.969568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.969595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.970783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:09.970855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241274515778833:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:09.970861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241274515778734:2048] [node 1] ICP30 unregister session Session# [5:7587241274515778833:2048] VirtualId# [5:7587241274515778823:0]
2025-12-24T02:11:09.970867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241274515778833:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.970900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.970919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.973582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:09.973673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241273196585309:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-12-24T02:11:09.973681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241273196585211:2048] [node 1] ICP30 unregister session Session# [4:7587241273196585309:2048] VirtualId# [4:7587241273196585304:0]
2025-12-24T02:11:09.973687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241273196585309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.973739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:09.973761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:09.975217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:09.975294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241275945515091:2049] [node 2] ICS01 socket: 453 reason# EndOfStream
2025-12-24T02:11:09.975301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241275945514999:2049] [node 2] ICP30 unregister session Session# [3:7587241275945515091:2049] VirtualId# [3:7587241275945515081:0]
2025-12-24T02:11:09.975306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241275945515091:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.975332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.975349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.986632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:09.986751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241273196585302:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:09.986761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241273196585212:2049] [node 2] ICP30 unregister session Session# [4:7587241273196585302:2049] VirtualId# [4:7587241273196585297:0]
2025-12-24T02:11:09.986769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241273196585302:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.986817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.986839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.986752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:09.986829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241274515778831:2049] [node 2] ICS01 socket: 484 reason# EndOfStream
2025-12-24T02:11:09.986835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241274515778735:2049] [node 2] ICP30 unregister session Session# [5:7587241274515778831:2049] VirtualId# [5:7587241274515778825:0]
2025-12-24T02:11:09.986841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241274515778831:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.986881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.986899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:09.990073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:09.990156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241275329903198:2049] [node 2] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:09.990164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241275329903115:2049] [node 2] ICP30 unregister session Session# [6:7587241275329903198:2049] VirtualId# [6:7587241275329903196:0]
2025-12-24T02:11:09.990170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241275329903198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:09.990214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:09.990237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.005597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:10.005702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241275329903216:2050] [node 3] ICS01 socket: 461 reason# EndOfStream
2025-12-24T02:11:10.005712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241275329903116:2050] [node 3] ICP30 unregister session Session# [6:7587241275329903216:2050] VirtualId# [6:7587241275329903214:0]
2025-12-24T02:11:10.005721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241275329903216:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.005776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:10.005800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:10.006741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:10.006816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241274515778835:2050] [node 3] ICS01 socket: 460 reason# EndOfStream
2025-12-24T02:11:10.006822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241274515778736:2050] [node 3] ICP30 unregister session Session# [5:7587241274515778835:2050] VirtualId# [5:7587241274515778822:0]
2025-12-24T02:11:10.006828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241274515778835:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.006863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:10.006895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:10.029511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:10.029617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241275329903204:2070] [node 4] ICS01 socket: 446 reason# EndOfStream
2025-12-24T02:11:10.029630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241275329903150:2070] [node 4] ICP30 unregister session Session# [6:7587241275329903204:2070] VirtualId# [6:7587241275329903202:0]
2025-12-24T02:11:10.029639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241275329903204:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.029694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:10.029719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:10.074944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241279862639521:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.078726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241279862639520:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:10.078830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241279285082406:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.078857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241279285082407:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.082974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278365655279:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.082986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278365655280:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.082991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278365655281:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.078817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280224854721:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.078828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280224854722:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.086902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241277220955007:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.086914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241277220955008:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.086919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241277220955009:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.110917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279618858632:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.110936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279618858633:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.114708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279618858631:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.127027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241277220955047:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:10.127084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279618858677:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:10.127178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279618858680:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:10.128732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278365655304:2059] [node 6] ICP01 ready to work
2025-12-24T02:11:10.128845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278365655305:2065] [node 5] ICP01 ready to work
2025-12-24T02:11:10.127155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280224854778:2103] [node 4] ICP01 ready to work
2025-12-24T02:11:10.134955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241279862639584:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:10.134954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241279285082470:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:10.143256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241279862639590:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:10.143377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241279862639591:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.156174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280224854788:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:10.156281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280224854789:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:10.158148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854800:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.158134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241278365655304:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22976858ns) {Configure:104(22976289ns) {ProcessConfigured:135(22662939ns) {StartInitialHandshake:157(22662484ns) {DropHandshakes:470(304ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1201ns) {SwitchToState:179(927ns)}}}}}
2025-12-24T02:11:10.158238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655325:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.158304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655326:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.159141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241279285082475:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:10.160797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241279285082478:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.161234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854801:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.163365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241279285082480:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13085
2025-12-24T02:11:10.163902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241277220955008:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20980627ns) {Configure:104(20979505ns) {ProcessConfigured:135(15527209ns) {StartInitialHandshake:157(15526071ns) {DropHandshakes:470(671ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:477(1207ns) {SwitchToState:179(521ns)}}}}}
2025-12-24T02:11:10.163417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241279285082481:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13085
2025-12-24T02:11:10.163671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241278365655305:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23382657ns) {Configure:104(23382227ns) {ProcessConfigured:135(23160693ns) {StartInitialHandshake:157(23160497ns) {DropHandshakes:470(227ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1053ns) {SwitchToState:179(17ns)}}}}}
2025-12-24T02:11:10.163726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655325:2090] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.163752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278365655281:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.168046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858702:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.169920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854780:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.169937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280224854780:2104] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.174840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279618858702:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.175362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279618858631:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.175613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639586:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.175628Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241279862639586:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.176243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241279862639586:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.176333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241279862639584:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.176349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.176358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241279862639584:2108] [node 6] ICP22 created new session: [1:7587241279862639600:2108]
2025-12-24T02:11:10.176366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241279862639600:2108] [node 6] ICS09 handshake done sender: [1:7587241279862639586:2109] self: [1:7587241279862639585:0] peer: [6:7587241279618858703:0] socket: 444 qp: -1
2025-12-24T02:11:10.176375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241279862639600:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:10.176386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.178793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241277220955007:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51709437ns) {Configure:104(51709051ns) {ProcessConfigured:135(46332731ns) {StartInitialHandshake:157(46331986ns) {DropHandshakes:470(504ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(246ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1102ns) {SwitchToState:179(43ns)}}}}}
2025-12-24T02:11:10.178965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241277220955076:2091] [node 6] ICP01 ready to work
2025-12-24T02:11:10.179711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955077:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.179763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955078:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.179793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955079:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.180195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858704:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.180235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858705:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.180246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858706:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.183827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241279285082478:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22963088ns) {Configure:104(22962042ns) {ProcessConfigured:135(22749124ns) {StartInitialHandshake:157(22748957ns) {DropHandshakes:470(337ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(1314ns) {SwitchToState:179(271ns)}}}}}
2025-12-24T02:11:10.183950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082480:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.183564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655326:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.183645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655360:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.183684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655361:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.183717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655362:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.187062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241279285082487:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13085
2025-12-24T02:11:10.186866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854800:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.187106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241279285082406:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.187511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955077:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.187572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955078:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.187593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955079:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.187642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241277220955007:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.190838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279618858704:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.190917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279618858702:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.191071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279618858631:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.191093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.191101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279618858631:2048] [node 1] ICP22 created new session: [6:7587241279618858707:2048]
2025-12-24T02:11:10.191109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241279618858707:2048] [node 1] ICS09 handshake done sender: [6:7587241279618858702:2112] self: [6:7587241279618858703:0] peer: [1:7587241279862639585:0] socket: 446 qp: -1
2025-12-24T02:11:10.191118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241279618858707:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.191131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.191725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279618858633:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.194503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082481:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.190777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655361:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.190874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278365655279:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.194819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279618858705:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.195207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279618858632:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.195859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858711:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.195136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854791:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.195148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280224854791:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.198827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280224854780:2104] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.198919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082467:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.198931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241279285082467:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.200939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082477:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.200946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241279285082477:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.200886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655365:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.199615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241280224854722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.199685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280224854791:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.202755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639581:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.202769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241279862639581:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.203116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639595:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.203121Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241279862639595:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.203683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639593:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.203687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241279862639593:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.204345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241279862639603:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7166
2025-12-24T02:11:10.204393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241279862639604:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7166
2025-12-24T02:11:10.204557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278365655325:2090] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.204521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241279285082467:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.204545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241277220955009:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40587898ns) {Configure:104(40587020ns) {ProcessConfigured:135(40263040ns) {StartInitialHandshake:157(40262063ns) {DropHandshakes:470(296ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(1639ns) {SwitchToState:179(1090ns)}}}}}
2025-12-24T02:11:10.210814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241279862639595:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.212922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279618858704:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.214987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279618858633:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.215007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.215017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279618858633:2050] [node 3] ICP22 created new session: [6:7587241279618858712:2050]
2025-12-24T02:11:10.215024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241279618858712:2050] [node 3] ICS09 handshake done sender: [6:7587241279618858704:2113] self: [6:7587241279618858709:0] peer: [3:7587241280224854790:0] socket: 455 qp: -1
2025-12-24T02:11:10.215033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241279618858712:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.215041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.218030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858714:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.215038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241277220955009:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.215275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955091:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.215718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241279285082407:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.215737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.215744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241279285082407:2049] [node 3] ICP22 created new session: [2:7587241279285082491:2049]
2025-12-24T02:11:10.215751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241279285082491:2049] [node 3] ICS09 handshake done sender: [2:7587241279285082467:2107] self: [2:7587241279285082466:0] peer: [3:7587241280224854802:0] socket: 433 qp: -1
2025-12-24T02:11:10.215758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241279285082491:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:10.215767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.218785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082487:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.219703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241279862639593:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.219882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279618858705:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.218842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241279285082477:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.218946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241279285082475:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.218959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.218968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241279285082475:2111] [node 6] ICP22 created new session: [2:7587241279285082495:2111]
2025-12-24T02:11:10.218974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241279285082495:2111] [node 6] ICS09 handshake done sender: [2:7587241279285082477:2112] self: [2:7587241279285082476:0] peer: [6:7587241279618858710:0] socket: 458 qp: -1
2025-12-24T02:11:10.218982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241279285082495:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:10.218993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.220180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279618858632:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.220193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.220199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279618858632:2049] [node 2] ICP22 created new session: [6:7587241279618858715:2049]
2025-12-24T02:11:10.220205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241279618858715:2049] [node 2] ICS09 handshake done sender: [6:7587241279618858705:2114] self: [6:7587241279618858710:0] peer: [2:7587241279285082476:0] socket: 469 qp: -1
2025-12-24T02:11:10.220211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241279618858715:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.220217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.223305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955092:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.223594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955092:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.223616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241277220955078:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.223666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241277220955007:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.223678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.223685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241277220955007:2048] [node 1] ICP22 created new session: [5:7587241277220955093:2048]
2025-12-24T02:11:10.223690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241277220955093:2048] [node 1] ICS09 handshake done sender: [5:7587241277220955078:2113] self: [5:7587241277220955088:0] peer: [1:7587241279862639592:0] socket: 464 qp: -1
2025-12-24T02:11:10.223695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241277220955093:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.223702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.223810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241277220955008:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.226347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955097:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.222796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241279862639591:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.222814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.222823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241279862639591:2111] [node 4] ICP22 created new session: [1:7587241279862639607:2111]
2025-12-24T02:11:10.222828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241279862639607:2111] [node 4] ICS09 handshake done sender: [1:7587241279862639595:2113] self: [1:7587241279862639594:0] peer: [4:7587241278365655364:0] socket: 462 qp: -1
2025-12-24T02:11:10.222835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241279862639607:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:10.222844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.224643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082486:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.224655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241279285082486:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.226428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241279285082486:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.226506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241279285082478:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.226519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.226526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241279285082478:2110] [node 5] ICP22 created new session: [2:7587241279285082500:2110]
2025-12-24T02:11:10.226532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241279285082500:2110] [node 5] ICS09 handshake done sender: [2:7587241279285082486:2116] self: [2:7587241279285082479:0] peer: [5:7587241277220955095:0] socket: 487 qp: -1
2025-12-24T02:11:10.226538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241279285082500:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:10.226544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.238898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278365655281:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.238938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.238950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278365655281:2050] [node 3] ICP22 created new session: [4:7587241278365655366:2050]
2025-12-24T02:11:10.238958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278365655366:2050] [node 3] ICS09 handshake done sender: [4:7587241278365655325:2090] self: [4:7587241278365655357:0] peer: [3:7587241280224854779:0] socket: 437 qp: -1
2025-12-24T02:11:10.238969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278365655366:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.238982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.239158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655360:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.241130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655368:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
2025-12-24T02:11:10.239693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280224854778:2103] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.239713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.239722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280224854778:2103] [node 4] ICP22 created new session: [3:7587241280224854805:2103]
2025-12-24T02:11:10.239728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241280224854805:2103] [node 4] ICS09 handshake done sender: [3:7587241280224854780:2104] self: [3:7587241280224854779:0] peer: [4:7587241278365655357:0] socket: 435 qp: -1
2025-12-24T02:11:10.239733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241280224854805:2103] [node 4] ICS10 traffic start
2025-12-24T02:11:10.239740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.239905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854801:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.239917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280224854788:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.239923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.239929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280224854788:2106] [node 6] ICP22 created new session: [3:7587241280224854807:2106]
2025-12-24T02:11:10.239933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241280224854807:2106] [node 6] ICS09 handshake done sender: [3:7587241280224854791:2111] self: [3:7587241280224854790:0] peer: [6:7587241279618858709:0] socket: 448 qp: -1
2025-12-24T02:11:10.239937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241280224854807:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:10.239943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.240127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854793:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.240131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280224854793:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.240714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241280224854778:2103] [node 4] ICP18 handshake (actor [3:7587241280224854801:2116]) is too old
2025-12-24T02:11:10.240815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854809:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
2025-12-24T02:11:10.241270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280224854793:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.241615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280224854789:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.241623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.241627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280224854789:2108] [node 5] ICP22 created new session: [3:7587241280224854813:2108]
2025-12-24T02:11:10.241632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241280224854813:2108] [node 5] ICS09 handshake done sender: [3:7587241280224854793:2112] self: [3:7587241280224854792:0] peer: [5:7587241277220955090:0] socket: 449 qp: -1
2025-12-24T02:11:10.241636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241280224854813:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:10.241642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.241351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241279285082502:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13085
2025-12-24T02:11:10.243258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278365655280:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.243284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278365655361:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.244571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280224854800:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.244001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278365655279:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.244015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.244020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278365655279:2048] [node 1] ICP22 created new session: [4:7587241278365655371:2048]
2025-12-24T02:11:10.244024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278365655371:2048] [node 1] ICS09 handshake done sender: [4:7587241278365655361:2118] self: [4:7587241278365655364:0] peer: [1:7587241279862639594:0] socket: 471 qp: -1
2025-12-24T02:11:10.244029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278365655371:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.244034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.244709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241280224854722:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.244723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.244728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280224854722:2049] [node 2] ICP22 created new session: [3:7587241280224854815:2049]
2025-12-24T02:11:10.244732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241280224854815:2049] [node 2] ICS09 handshake done sender: [3:7587241280224854800:2115] self: [3:7587241280224854802:0] peer: [2:7587241279285082466:0] socket: 434 qp: -1
2025-12-24T02:11:10.244737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241280224854815:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.244743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.245460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.243448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241279862639590:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.243468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.243473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241279862639590:2110] [node 5] ICP22 created new session: [1:7587241279862639612:2110]
2025-12-24T02:11:10.243479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241279862639612:2110] [node 5] ICS09 handshake done sender: [1:7587241279862639593:2112] self: [1:7587241279862639592:0] peer: [5:7587241277220955088:0] socket: 447 qp: -1
2025-12-24T02:11:10.243485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241279862639612:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:10.243492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.243568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241279862639590:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:10.243691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639603:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.244014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241279862639591:2111] [node 4] ICP35 event processing took too much time PendingConnection:279(21180631ns) {HandleHandshakeStatus:329(21180200ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(281ns) SwitchToState:179(87ns) ProcessPendingSessionEvents:495(21091211ns) {ForwardSessionEventToSession:612(21075918ns) ForwardSessionEventToSession:612(2046ns) ForwardSessionEventToSession:612(926ns) ForwardSessionEventToSession:612(1655ns) ForwardSessionEventToSession:612(698ns) ForwardSessionEventToSession:612(869ns) ForwardSessionEventToSession:612(795ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(1053ns) ForwardSessionEventToSession:612(1150ns)}}}
2025-12-24T02:11:10.244044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241279862639591:2111] [node 4] ICP18 handshake (actor [1:7587241279862639603:2115]) is too old
2025-12-24T02:11:10.244117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241279862639610:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7166
2025-12-24T02:11:10.244235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241279862639581:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.250999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241279862639609:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7166
2025-12-24T02:11:10.251207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241279862639520:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.251222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.251234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241279862639520:2048] [node 2] ICP22 created new session: [1:7587241279862639615:2048]
2025-12-24T02:11:10.251242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241279862639615:2048] [node 2] ICS09 handshake done sender: [1:7587241279862639581:2107] self: [1:7587241279862639580:0] peer: [2:7587241279285082488:0] socket: 432 qp: -1
2025-12-24T02:11:10.251250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241279862639615:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:10.251260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.250899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858718:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.251274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279618858718:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.251318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279618858677:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.254722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858717:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.255198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082472:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.255213Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241279285082472:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.255712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241279285082502:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.256038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241279285082407:2049] [node 3] ICP18 handshake (actor [2:7587241279285082502:2118]) is too old
2025-12-24T02:11:10.256280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241279285082504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13085
2025-12-24T02:11:10.259053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241279285082472:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.259171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241279285082480:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.259247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241279285082406:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.259266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.259273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241279285082406:2048] [node 1] ICP22 created new session: [2:7587241279285082505:2048]
2025-12-24T02:11:10.259282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241279285082505:2048] [node 1] ICS09 handshake done sender: [2:7587241279285082480:2113] self: [2:7587241279285082488:0] peer: [1:7587241279862639580:0] socket: 442 qp: -1
2025-12-24T02:11:10.259289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241279285082505:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.259311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.260078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241279285082470:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.260095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.260100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241279285082470:2108] [node 4] ICP22 created new session: [2:7587241279285082507:2108]
2025-12-24T02:11:10.260105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241279285082507:2108] [node 4] ICS09 handshake done sender: [2:7587241279285082472:2109] self: [2:7587241279285082471:0] peer: [4:7587241278365655369:0] socket: 457 qp: -1
2025-12-24T02:11:10.260110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241279285082507:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:10.260115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.267639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279618858717:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.267694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279618858680:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.267181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241277220955092:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.267327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241277220955008:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241277220955092:2119] held: no
2025-12-24T02:11:10.267351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241277220955008:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.267363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.267374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241277220955008:2049] [node 2] ICP22 created new session: [5:7587241277220955100:2049]
2025-12-24T02:11:10.267380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241277220955100:2049] [node 2] ICS09 handshake done sender: [5:7587241277220955092:2119] self: [5:7587241277220955095:0] peer: [2:7587241279285082479:0] socket: 488 qp: -1
2025-12-24T02:11:10.267389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241277220955100:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.267398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.273583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858721:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.273838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955083:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.273865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241277220955083:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.273937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279618858718:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.270889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655352:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.271974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639609:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.273079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639610:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.276192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280224854817:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.276313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241280224854721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.278035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241280224854721:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241280224854817:2118] held: no
2025-12-24T02:11:10.275003Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241279862639520:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23765648ns) {HandleHandshakeStatus:329(23764304ns) {DropIncomingHandshake:442(80ns) DropOutgoingHandshake:457(653ns) SwitchToState:179(60ns) ProcessPendingSessionEvents:495(23652765ns) {ForwardSessionEventToSession:612(23635077ns) ForwardSessionEventToSession:612(2032ns) ForwardSessionEventToSession:612(787ns) ForwardSessionEventToSession:612(1442ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(686ns) ForwardSessionEventToSession:612(734ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(1515ns)}}}
2025-12-24T02:11:10.275086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241279862639520:2048] [node 2] ICP18 handshake (actor [1:7587241279862639609:2117]) is too old
2025-12-24T02:11:10.276457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241279862639598:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.276461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241279862639598:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.277046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241279862639598:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.274390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955098:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.270911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241278365655352:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.277105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241279862639521:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.277118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.273269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655362:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.277123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241279862639521:2049] [node 3] ICP22 created new session: [1:7587241279862639619:2049]
2025-12-24T02:11:10.277128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241279862639619:2049] [node 3] ICS09 handshake done sender: [1:7587241279862639598:2114] self: [1:7587241279862639597:0] peer: [3:7587241280224854818:0] socket: 499 qp: -1
2025-12-24T02:11:10.273299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241278365655280:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241278365655360:2117] held: no
2025-12-24T02:11:10.277133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241279862639619:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:10.274602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955099:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.277140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.274779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241277220955079:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.275281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241277220955083:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.275394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241277220955009:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.275404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.275410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241277220955009:2050] [node 3] ICP22 created new session: [5:7587241277220955104:2050]
2025-12-24T02:11:10.275416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241277220955104:2050] [node 3] ICS09 handshake done sender: [5:7587241277220955079:2114] self: [5:7587241277220955090:0] peer: [3:7587241280224854792:0] socket: 465 qp: -1
2025-12-24T02:11:10.275420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241277220955104:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.275427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.275545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241277220955076:2091] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.275554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: 2025-12-24T02:11:10.273475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278365655374:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55804
::ffff:127.1.0.0
2025-12-24T02:11:10.273796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278365655352:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.274969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241278365655304:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.274991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.275002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278365655304:2059] [node 6] ICP22 created new session: [4:7587241278365655377:2059]
2025-12-24T02:11:10.275324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279618858677:2089] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.275010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278365655377:2059] [node 6] ICS09 handshake done sender: [4:7587241278365655352:2113] self: [4:7587241278365655327:0] peer: [6:7587241279618858719:0] socket: 498 qp: -1
2025-12-24T02:11:10.275341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.275558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241277220955076:2091] [node 6] ICP22 created new session: [5:7587241277220955106:2091]
2025-12-24T02:11:10.275562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241277220955106:2091] [node 6] ICS09 handshake done sender: [5:7587241277220955083:2116] self: [5:7587241277220955082:0] peer: [6:7587241279618858720:0] socket: 489 qp: -1
2025-12-24T02:11:10.275347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279618858677:2089] [node 4] ICP22 created new session: [6:7587241279618858722:2089]
2025-12-24T02:11:10.275566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241277220955106:2091] [node 6] ICS10 traffic start
2025-12-24T02:11:10.275358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241279618858722:2089] [node 4] ICS09 handshake done sender: [6:7587241279618858718:2119] self: [6:7587241279618858719:0] peer: [4:7587241278365655327:0] socket: 500 qp: -1
2025-12-24T02:11:10.275574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.275364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241279618858722:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:10.275374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.278933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279618858724:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6766
2025-12-24T02:11:10.275017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278365655377:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:10.275025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.275162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278365655360:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.279475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854820:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
2025-12-24T02:11:10.279507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280224854819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17524
2025-12-24T02:11:10.282972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280224854817:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.283078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241280224854721:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.283091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.283098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280224854721:2048] [node 1] ICP22 created new session: [3:7587241280224854821:2048]
2025-12-24T02:11:10.283104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241280224854821:2048] [node 1] ICS09 handshake done sender: [3:7587241280224854817:2118] self: [3:7587241280224854818:0] peer: [1:7587241279862639597:0] socket: 452 qp: -1
2025-12-24T02:11:10.283111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241280224854821:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.283118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.282844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278365655280:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.282865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.282872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278365655280:2049] [node 2] ICP22 created new session: [4:7587241278365655379:2049]
2025-12-24T02:11:10.282879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278365655379:2049] [node 2] ICS09 handshake done sender: [4:7587241278365655360:2117] self: [4:7587241278365655369:0] peer: [2:7587241279285082471:0] socket: 459 qp: -1
2025-12-24T02:11:10.282886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278365655379:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.282895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.286891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278365655356:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.286896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241278365655356:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.283102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241277220955099:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.283145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241277220955047:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.286929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279618858680:2092] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24105) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241279618858717:2118] held: no
2025-12-24T02:11:10.286978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279618858717:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.287434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278365655356:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.287502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241278365655305:2065] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.287519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.287527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278365655305:2065] [node 5] ICP22 created new session: [4:7587241278365655382:2065]
2025-12-24T02:11:10.287534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278365655382:2065] [node 5] ICS09 handshake done sender: [4:7587241278365655356:2114] self: [4:7587241278365655343:0] peer: [5:7587241277220955108:0] socket: 493 qp: -1
2025-12-24T02:11:10.287539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278365655382:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:10.287547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.291127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241277220955047:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64729) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241277220955099:2122] held: no
2025-12-24T02:11:10.291182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241277220955109:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10590
2025-12-24T02:11:10.291362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241277220955099:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.291405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241277220955047:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.291415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.291421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241277220955047:2075] [node 4] ICP22 created new session: [5:7587241277220955110:2075]
2025-12-24T02:11:10.291426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241277220955110:2075] [node 4] ICS09 handshake done sender: [5:7587241277220955099:2122] self: [5:7587241277220955108:0] peer: [4:7587241278365655343:0] socket: 460 qp: -1
2025-12-24T02:11:10.291432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241277220955110:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:10.291438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.295451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279618858680:2092] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.295476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.295484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279618858680:2092] [node 5] ICP22 created new session: [6:7587241279618858725:2092]
2025-12-24T02:11:10.295491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241279618858725:2092] [node 5] ICS09 handshake done sender: [6:7587241279618858717:2118] self: [6:7587241279618858720:0] peer: [5:7587241277220955082:0] socket: 492 qp: -1
2025-12-24T02:11:10.295498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241279618858725:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:10.295508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.315499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:10.315592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241280224854821:2048] [node 1] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:10.315602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241280224854721:2048] [node 1] ICP30 unregister session Session# [3:7587241280224854821:2048] VirtualId# [3:7587241280224854818:0]
2025-12-24T02:11:10.315610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241280224854821:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.315646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.315666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.318767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:10.318851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241279285082505:2048] [node 1] ICS01 socket: 442 reason# EndOfStream
2025-12-24T02:11:10.318860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241279285082406:2048] [node 1] ICP30 unregister session Session# [2:7587241279285082505:2048] VirtualId# [2:7587241279285082488:0]
2025-12-24T02:11:10.318867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241279285082505:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.318907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.318930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.325631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:10.325727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241279618858707:2048] [node 1] ICS07 socket disconnect 446 reason# EndOfStream
2025-12-24T02:11:10.325734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241279618858707:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.325785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.325808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241279618858707:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:10.326257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279618858631:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:10.326270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241279618858707:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:10.326276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279618858631:2048] [node 1] ICP30 unregister session Session# [6:7587241279618858707:2048] VirtualId# [6:7587241279618858703:0]
2025-12-24T02:11:10.326293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.326300Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241279618858631:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.326304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.330000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:10.330070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241278365655371:2048] [node 1] ICS07 socket disconnect 471 reason# EndOfStream
2025-12-24T02:11:10.330074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241278365655371:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.330102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.330121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241278365655371:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:10.329794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:10.329871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241277220955093:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-12-24T02:11:10.329878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241277220955007:2048] [node 1] ICP30 unregister session Session# [5:7587241277220955093:2048] VirtualId# [5:7587241277220955088:0]
2025-12-24T02:11:10.329885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241277220955093:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.329933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.329956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.331271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:10.331359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241280224854815:2049] [node 2] ICS01 socket: 434 reason# EndOfStream
2025-12-24T02:11:10.331366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241280224854722:2049] [node 2] ICP30 unregister session Session# [3:7587241280224854815:2049] VirtualId# [3:7587241280224854802:0]
2025-12-24T02:11:10.331372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241280224854815:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.331412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.331433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.334761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241279618858631:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:10.334876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241278365655279:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:10.334890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241278365655371:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:10.334895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241278365655279:2048] [node 1] ICP30 unregister session Session# [4:7587241278365655371:2048] VirtualId# [4:7587241278365655364:0]
2025-12-24T02:11:10.334913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.334919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241278365655279:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.334922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65051) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.338734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:10.338799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241278365655379:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-12-24T02:11:10.338805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241278365655280:2049] [node 2] ICP30 unregister session Session# [4:7587241278365655379:2049] VirtualId# [4:7587241278365655369:0]
2025-12-24T02:11:10.338811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241278365655379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.338850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.338873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.338789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:10.338849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241277220955100:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:10.338854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241277220955008:2049] [node 2] ICP30 unregister session Session# [5:7587241277220955100:2049] VirtualId# [5:7587241277220955095:0]
2025-12-24T02:11:10.338859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241277220955100:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.338887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.338908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.342820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:10.342913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241279618858715:2049] [node 2] ICS01 socket: 469 reason# EndOfStream
2025-12-24T02:11:10.342920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279618858632:2049] [node 2] ICP30 unregister session Session# [6:7587241279618858715:2049] VirtualId# [6:7587241279618858710:0]
2025-12-24T02:11:10.342927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241279618858715:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.342980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.343004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.350731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241278365655279:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:10.378782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:10.378895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241279618858712:2050] [node 3] ICS01 socket: 455 reason# EndOfStream
2025-12-24T02:11:10.378907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279618858633:2050] [node 3] ICP30 unregister session Session# [6:7587241279618858712:2050] VirtualId# [6:7587241279618858709:0]
2025-12-24T02:11:10.378916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241279618858712:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.378982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:10.379010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:10.438817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241277726489525:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.438832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241277726489526:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.431018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277369028715:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:10.431027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277369028716:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.446820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241277467371792:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.446846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241277467371793:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.446829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278410181624:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.446842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278410181625:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.446847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278410181626:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.462762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241279311408707:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.462793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241279311408709:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.466716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241279311408708:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.486662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241277979396698:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.486675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241277979396699:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.486680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241277979396700:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.536951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489585:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
2025-12-24T02:11:10.535051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241277979396711:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:10.535141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277369028777:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:10.535245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277369028778:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.539164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241279311408769:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:10.539219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408768:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.539939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181685:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.539308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241277979396729:2072] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.543391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371852:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-12-24T02:11:10.543067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396744:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.547215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277369028791:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:10.547848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277369028792:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20276
2025-12-24T02:11:10.549654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241277467371859:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:10.549738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241277467371860:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:10.549811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241277467371861:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:10.550885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408768:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.551004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489585:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.551107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241277979396744:2084] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.551133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241277979396711:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.554893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408771:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.554908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241279311408771:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.558888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278410181690:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:10.558938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181686:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.559463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241277726489525:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.559574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241277726489593:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:10.559646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241277726489594:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:10.559716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241277726489595:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:10.560826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.572151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408782:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.574160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028780:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.574185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277369028780:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.572418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241277726489526:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20848264ns) {Configure:104(17300074ns) {ProcessConfigured:135(16953020ns) {StartInitialHandshake:157(16952604ns) {DropHandshakes:470(567ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(361ns) PrepareNewSessionHandshake:477(1208ns) {SwitchToState:179(952ns)}}}}}
2025-12-24T02:11:10.574915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241279311408707:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16123654ns) {Configure:104(16122028ns) {ProcessConfigured:135(15723063ns) {StartInitialHandshake:157(15722667ns) {DropHandshakes:470(633ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(1255ns) {SwitchToState:179(741ns)}}}}}
2025-12-24T02:11:10.574984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241279311408771:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.575182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241279311408769:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.575204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.575209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241279311408769:2105] [node 6] ICP22 created new session: [5:7587241279311408787:2105]
2025-12-24T02:11:10.575215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241279311408787:2105] [node 6] ICS09 handshake done sender: [5:7587241279311408771:2109] self: [5:7587241279311408770:0] peer: [6:7587241277979396773:0] socket: 447 qp: -1
2025-12-24T02:11:10.575221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241279311408787:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:10.575243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.576821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.573564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181692:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.573600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181693:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.578915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241278410181689:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:10.579573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277369028780:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.582787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.586869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277369028716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.586896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.586906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277369028716:2049] [node 3] ICP22 created new session: [1:7587241277369028796:2049]
2025-12-24T02:11:10.586914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277369028796:2049] [node 3] ICS09 handshake done sender: [1:7587241277369028780:2109] self: [1:7587241277369028779:0] peer: [3:7587241277726489592:0] socket: 440 qp: -1
2025-12-24T02:11:10.586922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277369028796:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:10.586932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.591385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489607:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
2025-12-24T02:11:10.591464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241279311408777:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:10.590896Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241278410181624:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31899235ns) {Configure:104(31898805ns) {ProcessConfigured:135(31553152ns) {StartInitialHandshake:157(31552543ns) {DropHandshakes:470(431ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(59ns) PrepareNewSessionHandshake:477(2149ns) {SwitchToState:179(998ns)}}}}}
2025-12-24T02:11:10.590986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181685:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.591020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278410181624:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.603086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241277979396711:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241277979396744:2084] held: no
2025-12-24T02:11:10.603137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241277979396744:2084] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.601373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241278410181689:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22374002ns) {Configure:104(22373092ns) {ProcessConfigured:135(21920281ns) {StartInitialHandshake:157(21919294ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(1713ns) {SwitchToState:179(801ns)}}}}}
2025-12-24T02:11:10.606877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489608:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.618852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241279311408777:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27162299ns) {Configure:104(27160939ns) {ProcessConfigured:135(26758733ns) {StartInitialHandshake:157(26757951ns) {DropHandshakes:470(683ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(2037ns) {SwitchToState:179(1240ns)}}}}}
2025-12-24T02:11:10.619224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371873:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-12-24T02:11:10.619457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241277979396711:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.619483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.619492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241277979396711:2059] [node 5] ICP22 created new session: [6:7587241277979396780:2059]
2025-12-24T02:11:10.619497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241277979396780:2059] [node 5] ICS09 handshake done sender: [6:7587241277979396744:2084] self: [6:7587241277979396773:0] peer: [5:7587241279311408770:0] socket: 452 qp: -1
2025-12-24T02:11:10.619505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241277979396780:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:10.619516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.619898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181686:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.623482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181692:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.623517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278410181625:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.623019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241277369028716:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(36115310ns) {HandleHandshakeStatus:329(36114045ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(321ns) SwitchToState:179(478ns) ProcessPendingSessionEvents:495(28166335ns) {ForwardSessionEventToSession:612(28151923ns) ForwardSessionEventToSession:612(2390ns) ForwardSessionEventToSession:612(1118ns) ForwardSessionEventToSession:612(758ns) ForwardSessionEventToSession:612(1069ns)}}}
2025-12-24T02:11:10.623106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028792:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.623191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277369028798:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20276
2025-12-24T02:11:10.623222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277369028799:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20276
2025-12-24T02:11:10.623235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277369028800:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20276
2025-12-24T02:11:10.623601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371852:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.623669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241277467371792:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.623828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371865:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.623834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241277467371865:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.624517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.624663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241277467371865:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.624905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278410181692:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.625093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278410181625:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.625113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.625120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278410181625:2049] [node 2] ICP22 created new session: [4:7587241278410181710:2049]
2025-12-24T02:11:10.625126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278410181710:2049] [node 2] ICS09 handshake done sender: [4:7587241278410181692:2112] self: [4:7587241278410181708:0] peer: [2:7587241277467371864:0] socket: 461 qp: -1
2025-12-24T02:11:10.625133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278410181710:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.625155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.625767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241277467371859:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.625780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.625788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241277467371859:2109] [node 4] ICP22 created new session: [2:7587241277467371876:2109]
2025-12-24T02:11:10.625792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241277467371876:2109] [node 4] ICS09 handshake done sender: [2:7587241277467371865:2113] self: [2:7587241277467371864:0] peer: [4:7587241278410181708:0] socket: 458 qp: -1
2025-12-24T02:11:10.625796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241277467371876:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:10.625823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.625832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408792:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.626808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028798:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.626899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028784:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.626903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277369028784:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.628399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277369028803:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20276
2025-12-24T02:11:10.628577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277369028784:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.628670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277369028715:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.628689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.628695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277369028715:2048] [node 2] ICP22 created new session: [1:7587241277369028806:2048]
2025-12-24T02:11:10.628701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277369028806:2048] [node 2] ICS09 handshake done sender: [1:7587241277369028784:2111] self: [1:7587241277369028783:0] peer: [2:7587241277467371874:0] socket: 442 qp: -1
2025-12-24T02:11:10.628707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277369028806:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:10.628714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.630986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028782:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.631000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277369028782:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.634799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371872:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-12-24T02:11:10.635122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028800:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.638827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408792:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.638892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241279311408707:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.639104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028786:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.639112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277369028786:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.639641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277369028786:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.639747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277369028778:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.639769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.639777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277369028778:2108] [node 5] ICP22 created new session: [1:7587241277369028811:2108]
2025-12-24T02:11:10.639784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277369028811:2108] [node 5] ICS09 handshake done sender: [1:7587241277369028786:2112] self: [1:7587241277369028785:0] peer: [5:7587241279311408793:0] socket: 485 qp: -1
2025-12-24T02:11:10.639791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277369028811:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:10.639804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.642983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489609:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
2025-12-24T02:11:10.643116Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241279311408707:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241279311408792:2117] held: no
2025-12-24T02:11:10.643190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408794:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
2025-12-24T02:11:10.643202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408795:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.647584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181713:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.648859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181713:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.650854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489607:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.651051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241279311408792:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.651246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241277726489585:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.651334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241277726489525:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.651359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.651368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241277726489525:2048] [node 1] ICP22 created new session: [3:7587241277726489611:2048]
2025-12-24T02:11:10.651375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241277726489611:2048] [node 1] ICS09 handshake done sender: [3:7587241277726489585:2107] self: [3:7587241277726489592:0] peer: [1:7587241277369028779:0] socket: 443 qp: -1
2025-12-24T02:11:10.651383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241277726489611:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.651393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.651742Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277369028782:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.651970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277369028777:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.651984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.651991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277369028777:2107] [node 4] ICP22 created new session: [1:7587241277369028814:2107]
2025-12-24T02:11:10.651997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277369028814:2107] [node 4] ICS09 handshake done sender: [1:7587241277369028782:2110] self: [1:7587241277369028781:0] peer: [4:7587241278410181705:0] socket: 441 qp: -1
2025-12-24T02:11:10.652003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277369028814:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:10.652012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.654836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181693:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.654932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241278410181626:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.655029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181715:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.654856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396783:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.655604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278410181685:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.655682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278410181624:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.655697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.655703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278410181624:2048] [node 1] ICP22 created new session: [4:7587241278410181717:2048]
2025-12-24T02:11:10.655709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278410181717:2048] [node 1] ICS09 handshake done sender: [4:7587241278410181685:2108] self: [4:7587241278410181705:0] peer: [1:7587241277369028781:0] socket: 446 qp: -1
2025-12-24T02:11:10.655714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278410181717:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.655737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.655162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408782:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.655191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241279311408707:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.655206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.655214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241279311408707:2048] [node 1] ICP22 created new session: [5:7587241279311408797:2048]
2025-12-24T02:11:10.655221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241279311408797:2048] [node 1] ICS09 handshake done sender: [5:7587241279311408792:2117] self: [5:7587241279311408793:0] peer: [1:7587241277369028785:0] socket: 472 qp: -1
2025-12-24T02:11:10.655228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241279311408797:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.655242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.655425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241279311408709:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.658868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489597:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.658878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241277726489597:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.659282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489599:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.659286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241277726489599:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.659688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489605:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.659878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241277726489526:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.660156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241277726489597:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.660261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241277726489599:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.660372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241277726489593:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.660387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.660394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241277726489593:2108] [node 4] ICP22 created new session: [3:7587241277726489617:2108]
2025-12-24T02:11:10.660399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241277726489617:2108] [node 4] ICS09 handshake done sender: [3:7587241277726489597:2112] self: [3:7587241277726489596:0] peer: [4:7587241278410181716:0] socket: 463 qp: -1
2025-12-24T02:11:10.660404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241277726489617:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:10.660424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.660471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241277726489594:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.660484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.660487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241277726489594:2109] [node 5] ICP22 created new session: [3:7587241277726489619:2109]
2025-12-24T02:11:10.660490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241277726489619:2109] [node 5] ICS09 handshake done sender: [3:7587241277726489599:2113] self: [3:7587241277726489598:0] peer: [5:7587241279311408799:0] socket: 464 qp: -1
2025-12-24T02:11:10.660493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241277726489619:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:10.660497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.661066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241277979396783:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.661201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241277979396698:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.661278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396784:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.661330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396785:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.656609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241277467371792:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241277467371852:2107] held: no
2025-12-24T02:11:10.656678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371872:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.662897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371863:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.662904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241277467371863:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.663373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.663957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241277979396785:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.664005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241277979396784:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.664042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241277979396700:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.664059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241277979396699:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.666984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371871:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.666998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241277467371871:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.670843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277369028794:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.670864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277369028794:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.670924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241277726489601:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.670938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241277726489601:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.674739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371882:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-12-24T02:11:10.674879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241277467371883:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40458
2025-12-24T02:11:10.675141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241277467371852:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.678884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241277467371863:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.685129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241278410181719:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14928
2025-12-24T02:11:10.685550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181699:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.685562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241278410181699:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.686120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278410181693:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.683623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241277467371792:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.683658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.683670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241277467371792:2048] [node 1] ICP22 created new session: [2:7587241277467371885:2048]
2025-12-24T02:11:10.683678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241277467371885:2048] [node 1] ICS09 handshake done sender: [2:7587241277467371852:2107] self: [2:7587241277467371874:0] peer: [1:7587241277369028783:0] socket: 445 qp: -1
2025-12-24T02:11:10.683688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241277467371885:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.683703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.683916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371882:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.683926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241277467371793:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.683933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.683938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241277467371793:2049] [node 3] ICP22 created new session: [2:7587241277467371887:2049]
2025-12-24T02:11:10.685250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241277979396779:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.683943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241277467371887:2049] [node 3] ICS09 handshake done sender: [2:7587241277467371863:2112] self: [2:7587241277467371862:0] peer: [3:7587241277726489616:0] socket: 457 qp: -1
2025-12-24T02:11:10.683950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241277467371887:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:10.685307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241277979396729:2072] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.683958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.686307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241278410181626:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.686324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.686332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278410181626:2050] [node 3] ICP22 created new session: [4:7587241278410181722:2050]
2025-12-24T02:11:10.686338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278410181722:2050] [node 3] ICS09 handshake done sender: [4:7587241278410181693:2113] self: [4:7587241278410181716:0] peer: [3:7587241277726489596:0] socket: 470 qp: -1
2025-12-24T02:11:10.686345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278410181722:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.686351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.687033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408794:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.687070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241279311408708:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.687117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241279311408709:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241279311408782:2114] held: no
2025-12-24T02:11:10.687202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278410181699:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.687286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241278410181624:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31570408ns) {HandleHandshakeStatus:329(31570163ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(2640ns) SwitchToState:179(109ns) ProcessPendingSessionEvents:495(20118438ns) {ForwardSessionEventToSession:612(20105421ns) ForwardSessionEventToSession:612(1268ns) ForwardSessionEventToSession:612(1295ns) ForwardSessionEventToSession:612(713ns) ForwardSessionEventToSession:612(781ns) ForwardSessionEventToSession:612(1081ns) ForwardSessionEventToSession:612(561ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(1339ns)}}}
2025-12-24T02:11:10.690951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241278410181690:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.690971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.690981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278410181690:2111] [node 6] ICP22 created new session: [4:7587241278410181724:2111]
2025-12-24T02:11:10.690988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278410181724:2111] [node 6] ICS09 handshake done sender: [4:7587241278410181699:2116] self: [4:7587241278410181698:0] peer: [6:7587241277979396789:0] socket: 474 qp: -1
2025-12-24T02:11:10.690995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278410181724:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:10.691004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.691133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241277726489593:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(30728292ns) {HandleHandshakeStatus:329(30727575ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(257ns) SwitchToState:179(62ns) ProcessPendingSessionEvents:495(20252857ns) {ForwardSessionEventToSession:612(20239726ns) ForwardSessionEventToSession:612(1618ns) ForwardSessionEventToSession:612(1357ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(655ns) ForwardSessionEventToSession:612(1614ns) ForwardSessionEventToSession:612(855ns) ForwardSessionEventToSession:612(732ns) ForwardSessionEventToSession:612(597ns) ForwardSessionEventToSession:612(1408ns)}}}
2025-12-24T02:11:10.695413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277369028794:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.695526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277369028791:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.695548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.695555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277369028791:2113] [node 6] ICP22 created new session: [1:7587241277369028817:2113]
2025-12-24T02:11:10.695562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277369028817:2113] [node 6] ICS09 handshake done sender: [1:7587241277369028794:2115] self: [1:7587241277369028793:0] peer: [6:7587241277979396786:0] socket: 492 qp: -1
2025-12-24T02:11:10.695568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277369028817:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:10.695574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.694895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241277979396729:2072] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20538) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241277979396779:2117] held: no
2025-12-24T02:11:10.696465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241277979396698:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241277979396783:2118] held: no
2025-12-24T02:11:10.699824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241277726489601:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:10.698837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371883:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.698878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241277467371793:2049] [node 3] ICP18 handshake (actor [2:7587241277467371883:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.702773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241277467371867:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.702775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241277369028716:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:10.702817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [1:7587241277369028716:2049] [node 3] ICP35 event processing took too much time StateWork:294(31974659ns) {HandleHandshakeStatus:395(31972549ns) {LogHandshakeFail:472(4432ns) {UpdateErrorStateLog:545(999ns)}}}
2025-12-24T02:11:10.702787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241277467371867:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.702792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241277726489595:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.702819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.702829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241277726489595:2110] [node 6] ICP22 created new session: [3:7587241277726489623:2110]
2025-12-24T02:11:10.702837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241277726489623:2110] [node 6] ICS09 handshake done sender: [3:7587241277726489601:2114] self: [3:7587241277726489600:0] peer: [6:7587241277979396787:0] socket: 500 qp: -1
2025-12-24T02:11:10.702845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241277726489623:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:10.702856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.706870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241277467371871:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.715365Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241278410181690:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(24376475ns) {HandleHandshakeStatus:329(24375266ns) {DropIncomingHandshake:442(4318ns) DropOutgoingHandshake:457(127ns) SwitchToState:179(166ns) ProcessPendingSessionEvents:495(24275658ns) {ForwardSessionEventToSession:612(24243257ns) ForwardSessionEventToSession:612(3052ns) ForwardSessionEventToSession:612(684ns) ForwardSessionEventToSession:612(1953ns) ForwardSessionEventToSession:612(821ns) ForwardSessionEventToSession:612(978ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(904ns) ForwardSessionEventToSession:612(1566ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.715187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396791:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.715663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396792:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.715955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241277979396784:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.716130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241277979396699:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.716149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.716157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241277979396699:2049] [node 2] ICP22 created new session: [6:7587241277979396795:2049]
2025-12-24T02:11:10.716163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241277979396795:2049] [node 2] ICS09 handshake done sender: [6:7587241277979396784:2119] self: [6:7587241277979396788:0] peer: [2:7587241277467371868:0] socket: 462 qp: -1
2025-12-24T02:11:10.716172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241277979396795:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.716197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.717028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241277979396779:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.717084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241277979396729:2072] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.717094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.717100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241277979396729:2072] [node 4] ICP22 created new session: [6:7587241277979396797:2072]
2025-12-24T02:11:10.717107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241277979396797:2072] [node 4] ICS09 handshake done sender: [6:7587241277979396779:2117] self: [6:7587241277979396789:0] peer: [4:7587241278410181698:0] socket: 477 qp: -1
2025-12-24T02:11:10.717113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241277979396797:2072] [node 4] ICS10 traffic start
2025-12-24T02:11:10.717119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.717245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396793:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.717469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241277979396794:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53088
2025-12-24T02:11:10.717631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241277979396783:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:10.717676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241277979396785:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.717758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241277979396698:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.717768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.717775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241277979396698:2048] [node 1] ICP22 created new session: [6:7587241277979396799:2048]
2025-12-24T02:11:10.717781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241277979396799:2048] [node 1] ICS09 handshake done sender: [6:7587241277979396783:2118] self: [6:7587241277979396786:0] peer: [1:7587241277369028793:0] socket: 438 qp: -1
2025-12-24T02:11:10.717787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241277979396799:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.717794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.717959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241277979396700:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.717973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.717978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241277979396700:2050] [node 3] ICP22 created new session: [6:7587241277979396801:2050]
2025-12-24T02:11:10.717983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241277979396801:2050] [node 3] ICS09 handshake done sender: [6:7587241277979396785:2120] self: [6:7587241277979396787:0] peer: [3:7587241277726489600:0] socket: 478 qp: -1
2025-12-24T02:11:10.717989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241277979396801:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.717995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.718825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408802:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.720218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241277726489595:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(17394867ns) {HandleHandshakeStatus:329(17393405ns) {DropIncomingHandshake:442(190ns) DropOutgoingHandshake:457(156ns) SwitchToState:179(245ns) ProcessPendingSessionEvents:495(17276446ns) {ForwardSessionEventToSession:612(17265398ns) ForwardSessionEventToSession:612(1726ns) ForwardSessionEventToSession:612(963ns) ForwardSessionEventToSession:612(1131ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(786ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(599ns) ForwardSessionEventToSession:612(663ns) ForwardSessionEventToSession:612(1040ns)}}}
2025-12-24T02:11:10.720260Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241277726489526:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241277726489605:2115] held: no
2025-12-24T02:11:10.720290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241277726489593:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:10.720306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489626:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
2025-12-24T02:11:10.720117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408801:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
2025-12-24T02:11:10.719500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241277467371861:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.719525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:10.719530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241277467371861:2111] [node 6] ICP22 created new session: [2:7587241277467371891:2111]
2025-12-24T02:11:10.719536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241277467371891:2111] [node 6] ICS09 handshake done sender: [2:7587241277467371871:2115] self: [2:7587241277467371868:0] peer: [6:7587241277979396788:0] socket: 455 qp: -1
2025-12-24T02:11:10.719542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241277467371891:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:10.719549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:10.721009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241277467371867:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.721070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241277467371860:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.721081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.721086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241277467371860:2110] [node 5] ICP22 created new session: [2:7587241277467371894:2110]
2025-12-24T02:11:10.721091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241277467371894:2110] [node 5] ICS09 handshake done sender: [2:7587241277467371867:2114] self: [2:7587241277467371866:0] peer: [5:7587241279311408800:0] socket: 489 qp: -1
2025-12-24T02:11:10.721095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241277467371894:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:10.721100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.723078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241277726489595:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:10.723152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241277726489625:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64273
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.723365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241279311408708:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2718) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241279311408794:2118] held: no
2025-12-24T02:11:10.723440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241279311408802:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.723481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408803:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
2025-12-24T02:11:10.726789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241277726489605:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.726972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241279311408777:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.726992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241279311408782:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:10.727045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241277726489526:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.727064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.727071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241277726489526:2049] [node 2] ICP22 created new session: [3:7587241277726489627:2049]
2025-12-24T02:11:10.727077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241277726489627:2049] [node 2] ICS09 handshake done sender: [3:7587241277726489605:2115] self: [3:7587241277726489616:0] peer: [2:7587241277467371862:0] socket: 459 qp: -1
2025-12-24T02:11:10.727082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241277726489627:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.727093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.727424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241279311408709:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.727435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:10.727440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241279311408709:2050] [node 3] ICP22 created new session: [5:7587241279311408805:2050]
2025-12-24T02:11:10.727445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241279311408805:2050] [node 3] ICS09 handshake done sender: [5:7587241279311408782:2114] self: [5:7587241279311408799:0] peer: [3:7587241277726489598:0] socket: 466 qp: -1
2025-12-24T02:11:10.727449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241279311408805:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:10.727455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:10.731000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241279311408794:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:10.731073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241279311408708:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.731088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.731096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241279311408708:2049] [node 2] ICP22 created new session: [5:7587241279311408807:2049]
2025-12-24T02:11:10.731103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241279311408807:2049] [node 2] ICS09 handshake done sender: [5:7587241279311408794:2118] self: [5:7587241279311408800:0] peer: [2:7587241277467371866:0] socket: 493 qp: -1
2025-12-24T02:11:10.731109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241279311408807:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:10.731117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.730856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241278410181706:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.730870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241278410181706:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.734955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241279311408809:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3709
2025-12-24T02:11:10.738816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241278410181706:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.739082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241279311408802:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:10.739139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241279311408777:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.739154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:10.739162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241279311408777:2111] [node 4] ICP22 created new session: [5:7587241279311408810:2111]
2025-12-24T02:11:10.739169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241279311408810:2111] [node 4] ICS09 handshake done sender: [5:7587241279311408802:2121] self: [5:7587241279311408804:0] peer: [4:7587241278410181703:0] socket: 509 qp: -1
2025-12-24T02:11:10.739176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241279311408810:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:10.739184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:10.742821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241278410181689:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.742847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.742854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241278410181689:2110] [node 5] ICP22 created new session: [4:7587241278410181727:2110]
2025-12-24T02:11:10.742860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241278410181727:2110] [node 5] ICS09 handshake done sender: [4:7587241278410181706:2118] self: [4:7587241278410181703:0] peer: [5:7587241279311408804:0] socket: 488 qp: -1
2025-12-24T02:11:10.742866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241278410181727:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:10.742874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.762856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:10.762968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241277467371885:2048] [node 1] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:10.762980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241277467371792:2048] [node 1] ICP30 unregister session Session# [2:7587241277467371885:2048] VirtualId# [2:7587241277467371874:0]
2025-12-24T02:11:10.762989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241277467371885:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.763029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.763053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.766774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:10.766882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241277726489611:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-12-24T02:11:10.766890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241277726489525:2048] [node 1] ICP30 unregister session Session# [3:7587241277726489611:2048] VirtualId# [3:7587241277726489592:0]
2025-12-24T02:11:10.766897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241277726489611:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.766947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.766975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.774822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:10.774916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241278410181717:2048] [node 1] ICS07 socket disconnect 446 reason# EndOfStream
2025-12-24T02:11:10.774923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241278410181717:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.774975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.774996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241278410181717:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:10.775232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:10.775296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241279311408797:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:10.775313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241279311408707:2048] [node 1] ICP30 unregister session Session# [5:7587241279311408797:2048] VirtualId# [5:7587241279311408793:0]
2025-12-24T02:11:10.775320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241279311408797:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.775348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.775370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.775661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241278410181624:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:10.775673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241278410181717:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:10.775677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241278410181624:2048] [node 1] ICP30 unregister session Session# [4:7587241278410181717:2048] VirtualId# [4:7587241278410181705:0]
2025-12-24T02:11:10.775690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.775696Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241278410181624:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.775699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13391) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:10.778673Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:10.778860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241277979396799:2048] [node 1] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:10.778873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241277979396698:2048] [node 1] ICP30 unregister session Session# [6:7587241277979396799:2048] VirtualId# [6:7587241277979396786:0]
2025-12-24T02:11:10.778879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241277979396799:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.778929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:10.778951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:10.783341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:10.783441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241277726489627:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-12-24T02:11:10.783450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241277726489526:2049] [node 2] ICP30 unregister session Session# [3:7587241277726489627:2049] VirtualId# [3:7587241277726489616:0]
2025-12-24T02:11:10.783456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241277726489627:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.783497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.783517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.790757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241278410181624:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:10.793428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:10.793530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241277979396795:2049] [node 2] ICS01 socket: 462 reason# EndOfStream
2025-12-24T02:11:10.793540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241277979396699:2049] [node 2] ICP30 unregister session Session# [6:7587241277979396795:2049] VirtualId# [6:7587241277979396788:0]
2025-12-24T02:11:10.793549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241277979396795:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.793607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.793633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.798270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:10.798373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241279311408807:2049] [node 2] ICS01 socket: 493 reason# EndOfStream
2025-12-24T02:11:10.798383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241279311408708:2049] [node 2] ICP30 unregister session Session# [5:7587241279311408807:2049] VirtualId# [5:7587241279311408800:0]
2025-12-24T02:11:10.798391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241279311408807:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.798461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.798488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.798589Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:10.798664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241278410181710:2049] [node 2] ICS01 socket: 461 reason# EndOfStream
2025-12-24T02:11:10.798670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241278410181625:2049] [node 2] ICP30 unregister session Session# [4:7587241278410181710:2049] VirtualId# [4:7587241278410181708:0]
2025-12-24T02:11:10.798676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241278410181710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.798729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:10.798753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:10.813253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:10.813377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241277979396801:2050] [node 3] ICS01 socket: 478 reason# EndOfStream
2025-12-24T02:11:10.813388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241277979396700:2050] [node 3] ICP30 unregister session Session# [6:7587241277979396801:2050] VirtualId# [6:7587241277979396787:0]
2025-12-24T02:11:10.813399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241277979396801:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.813453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:10.813481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:10.813593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:10.813665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241279311408805:2050] [node 3] ICS01 socket: 466 reason# EndOfStream
2025-12-24T02:11:10.813670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241279311408709:2050] [node 3] ICP30 unregister session Session# [5:7587241279311408805:2050] VirtualId# [5:7587241279311408799:0]
2025-12-24T02:11:10.813675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241279311408805:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.813705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:10.813725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:10.826190Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:10.826299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241277979396797:2072] [node 4] ICS01 socket: 477 reason# EndOfStream
2025-12-24T02:11:10.826311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241277979396729:2072] [node 4] ICP30 unregister session Session# [6:7587241277979396797:2072] VirtualId# [6:7587241277979396789:0]
2025-12-24T02:11:10.826320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241277979396797:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.826396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:10.826422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:10.830762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:10.830856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241277979396780:2059] [node 5] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:10.830865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241277979396711:2059] [node 5] ICP30 unregister session Session# [6:7587241277979396780:2059] VirtualId# [6:7587241277979396773:0]
2025-12-24T02:11:10.830873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241277979396780:2059] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:10.830910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:10.830930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:10.855035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277100383074:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:10.855049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277100383075:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.862872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280230000468:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.862893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280230000469:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.866760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241280425772633:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.866788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241280425772634:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:10.872837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241277561317108:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.872849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241277561317109:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.872856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241277561317110:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.870819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241280275535370:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.870829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241280275535371:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.870833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241280275535372:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.886738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279143879844:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:10.886780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279143879845:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:10.886787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279143879846:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:10.927063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241280275535394:2063] [node 6] ICP01 ready to work
2025-12-24T02:11:10.927041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241277561317138:2071] [node 5] ICP01 ready to work
2025-12-24T02:11:10.927187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241280275535399:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:10.927035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280230000510:2080] [node 5] ICP01 ready to work
2025-12-24T02:11:10.927196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280230000511:2082] [node 4] ICP01 ready to work
2025-12-24T02:11:10.927280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241280230000512:2078] [node 6] ICP01 ready to work
2025-12-24T02:11:10.928285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241277561317154:2079] [node 6] ICP01 ready to work
2025-12-24T02:11:10.934881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279143879872:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:10.934870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241280425772698:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:10.935291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277100383136:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.940638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277100383149:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.941254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241280275535431:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:10.944609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241280275535431:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.944648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241280275535370:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.946754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383149:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.946797Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241279143879845:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19666087ns) {Configure:104(19665017ns) {ProcessConfigured:135(18532752ns) {StartInitialHandshake:157(18531773ns) {DropHandshakes:470(313ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(1153ns) {SwitchToState:179(666ns)}}}}}
2025-12-24T02:11:10.946940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279143879886:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:10.947166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279143879885:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.947276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241277561317110:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18934695ns) {Configure:104(18934095ns) {ProcessConfigured:135(18609033ns) {StartInitialHandshake:157(18608389ns) {DropHandshakes:470(425ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(110ns) PrepareNewSessionHandshake:477(440ns) {SwitchToState:179(267ns)}}}}}
2025-12-24T02:11:10.947532Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241280230000510:2080] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20286350ns) {Configure:104(20285718ns) {ProcessConfigured:135(18818270ns) {StartInitialHandshake:157(18817746ns) {DropHandshakes:470(350ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(936ns) {SwitchToState:179(121ns)}}}}}
2025-12-24T02:11:10.948240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280230000525:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:10.948292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280230000526:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:10.948331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280230000527:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:10.950914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383144:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.950925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277100383144:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:10.953149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241280275535399:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22368456ns) {Configure:104(308045ns) {ProcessConfigured:135(32710ns) {StartInitialHandshake:157(31177ns) {DropHandshakes:470(627ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(248ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.950813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241280425772695:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.953602Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241280275535371:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22848036ns) {Configure:104(18847758ns) {ProcessConfigured:135(18533671ns) {StartInitialHandshake:157(18533296ns) {DropHandshakes:470(752ns) {DropIncomingHandshake:442(264ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(371ns) {SwitchToState:179(87ns)}}}}}
2025-12-24T02:11:10.955981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241280275535451:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:10.953702Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241280230000512:2078] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18915288ns) {Configure:104(18914474ns) {ProcessConfigured:135(18587877ns) {StartInitialHandshake:157(18587518ns) {DropHandshakes:470(547ns) {DropIncomingHandshake:442(159ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(548ns) {SwitchToState:179(231ns)}}}}}
2025-12-24T02:11:10.955019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241280425772634:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20105636ns) {Configure:104(20105242ns) {ProcessConfigured:135(19746326ns) {StartInitialHandshake:157(19746007ns) {DropHandshakes:470(202ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(54ns) PrepareNewSessionHandshake:477(671ns) {SwitchToState:179(484ns)}}}}}
2025-12-24T02:11:10.953930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241279143879872:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19007287ns) {Configure:104(19006861ns) {ProcessConfigured:135(18783526ns) {StartInitialHandshake:157(18783392ns) {DropHandshakes:470(313ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(47ns) PrepareNewSessionHandshake:477(451ns) {SwitchToState:179(146ns)}}}}}
2025-12-24T02:11:10.953750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000525:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.954771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241279143879846:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19926996ns) {Configure:104(19926463ns) {ProcessConfigured:135(19606862ns) {StartInitialHandshake:157(19606543ns) {DropHandshakes:470(335ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(579ns) {SwitchToState:179(274ns)}}}}}
2025-12-24T02:11:10.954951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241279143879921:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:10.962724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241279143879883:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:10.962741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277100383141:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:10.963001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241277100383142:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:10.963449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241280425772696:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:10.962821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241280275535449:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:10.970801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241277561317109:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(43673904ns) {Configure:104(43672384ns) {ProcessConfigured:135(42564005ns) {StartInitialHandshake:157(42563239ns) {DropHandshakes:470(219ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(987ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.978825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241280425772698:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27914225ns) {Configure:104(27913162ns) {ProcessConfigured:135(27482650ns) {StartInitialHandshake:157(27481587ns) {DropHandshakes:470(1183ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:477(2347ns) {SwitchToState:179(1450ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.979028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241277561317138:2071] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(48213932ns) {Configure:104(26562323ns) {ProcessConfigured:135(26254540ns) {StartInitialHandshake:157(26252914ns) {DropHandshakes:470(197ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(813ns) {SwitchToState:179(64ns)}}}}}
2025-12-24T02:11:10.979117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241277561317185:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:10.979314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241280425772711:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
2025-12-24T02:11:10.979567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241280425772712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
2025-12-24T02:11:10.979583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241280425772713:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
2025-12-24T02:11:10.978917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241280275535452:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:10.980026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277100383144:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:10.980162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241280275535431:2100] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.982822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241280275535370:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:10.982853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:10.982863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241280275535370:2048] [node 1] ICP22 created new session: [5:7587241280275535453:2048]
2025-12-24T02:11:10.982872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241280275535453:2048] [node 1] ICS09 handshake done sender: [5:7587241280275535431:2100] self: [5:7587241280275535433:0] peer: [1:7587241277100383143:0] socket: 452 qp: -1
2025-12-24T02:11:10.982883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241280275535453:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:10.983174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241280230000511:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55835435ns) {Configure:104(55835054ns) {ProcessConfigured:135(54382464ns) {StartInitialHandshake:157(54381900ns) {DropHandshakes:470(199ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(87ns) PrepareNewSessionHandshake:477(1112ns) {SwitchToState:179(545ns)}}}}}
2025-12-24T02:11:10.982909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:10.983238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000526:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.983415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241280230000550:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:10.986846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241279143879883:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23962184ns) {Configure:104(23960778ns) {ProcessConfigured:135(23533406ns) {StartInitialHandshake:157(23532740ns) {DropHandshakes:470(689ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1455ns) {SwitchToState:179(821ns)}}}}}
2025-12-24T02:11:10.986903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279143879885:2082] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.986935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279143879872:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.990783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241277100383141:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27739584ns) {Configure:104(27738514ns) {ProcessConfigured:135(27339164ns) {StartInitialHandshake:157(27338771ns) {DropHandshakes:470(365ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(1069ns) {SwitchToState:179(111ns)}}}}}
2025-12-24T02:11:10.990823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277100383136:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.990846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:10.990856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277100383136:2107] [node 5] ICP22 created new session: [1:7587241277100383155:2107]
2025-12-24T02:11:10.990862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277100383155:2107] [node 5] ICS09 handshake done sender: [1:7587241277100383144:2111] self: [1:7587241277100383143:0] peer: [5:7587241280275535433:0] socket: 445 qp: -1
2025-12-24T02:11:10.990872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277100383155:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:10.990908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:10.991016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772711:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.991062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241280425772633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:10.991283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383140:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:10.991293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277100383140:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.994854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241280425772715:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
2025-12-24T02:11:10.994946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241280425772696:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31394296ns) {Configure:104(31393879ns) {ProcessConfigured:135(31131705ns) {StartInitialHandshake:157(31131599ns) {DropHandshakes:470(125ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(783ns) {SwitchToState:179(383ns)}}}}}
2025-12-24T02:11:10.994811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241277100383142:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31732329ns) {Configure:104(31731558ns) {ProcessConfigured:135(31363056ns) {StartInitialHandshake:157(31362559ns) {DropHandshakes:470(415ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(1765ns) {SwitchToState:179(819ns)}}}}}
2025-12-24T02:11:10.998865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277100383140:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:10.999089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277100383074:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:10.999111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:10.999121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277100383074:2048] [node 2] ICP22 created new session: [1:7587241277100383163:2048]
2025-12-24T02:11:10.999128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241277100383163:2048] [node 2] ICS09 handshake done sender: [1:7587241277100383140:2110] self: [1:7587241277100383134:0] peer: [2:7587241280425772714:0] socket: 444 qp: -1
2025-12-24T02:11:10.999135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241277100383163:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:10.999148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:10.999453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277100383160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24603
2025-12-24T02:11:10.999485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277100383161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24603
2025-12-24T02:11:10.999496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241277100383162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24603
2025-12-24T02:11:11.002810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241280425772711:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.002911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241280425772633:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.002928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.002934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241280425772633:2048] [node 1] ICP22 created new session: [2:7587241284720740013:2048]
2025-12-24T02:11:11.002942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284720740013:2048] [node 1] ICS09 handshake done sender: [2:7587241280425772711:2114] self: [2:7587241280425772714:0] peer: [1:7587241277100383134:0] socket: 470 qp: -1
2025-12-24T02:11:11.002951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284720740013:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.002973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.006932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241280275535430:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.006951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241280275535430:2099] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.007584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241280275535430:2099] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.007662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241280275535394:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.007674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.007683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241280275535394:2063] [node 6] ICP22 created new session: [5:7587241284570502752:2063]
2025-12-24T02:11:11.007690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284570502752:2063] [node 6] ICS09 handshake done sender: [5:7587241280275535430:2099] self: [5:7587241280275535407:0] peer: [6:7587241279143879925:0] socket: 453 qp: -1
2025-12-24T02:11:11.007696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284570502752:2063] [node 6] ICS10 traffic start
2025-12-24T02:11:11.007716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.012322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.012355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.015073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000527:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.015335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279143879886:2083] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.015374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279143879883:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.014917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241281856284484:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.018750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241277561317169:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.018754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241277561317169:2102] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.022870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383161:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.024895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241280230000468:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(50725438ns) {Configure:104(50724696ns) {ProcessConfigured:135(50377548ns) {StartInitialHandshake:157(50376915ns) {DropHandshakes:470(408ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1609ns) {SwitchToState:179(92ns)}}}}}
2025-12-24T02:11:11.024965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241280230000468:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.026784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383160:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.026947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241280275535370:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(44081627ns) {HandleHandshakeStatus:329(44080322ns) {DropIncomingHandshake:442(641ns) DropOutgoingHandshake:457(3317ns) SwitchToState:179(95ns) ProcessPendingSessionEvents:495(24055851ns) {ForwardSessionEventToSession:612(24040371ns) ForwardSessionEventToSession:612(2608ns) ForwardSessionEventToSession:612(1346ns) ForwardSessionEventToSession:612(927ns) ForwardSessionEventToSession:612(735ns) ForwardSessionEventToSession:612(1618ns) ForwardSessionEventToSession:612(871ns) ForwardSessionEventToSession:612(759ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(1002ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.027391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383147:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.027400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277100383147:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.028828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277100383147:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.029059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277100383075:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.029072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.029079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277100383075:2049] [node 3] ICP22 created new session: [1:7587241281395350465:2049]
No RDMA context found for GID: 2025-12-24T02:11:11.029085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241281395350465:2049] [node 3] ICS09 handshake done sender: [1:7587241277100383147:2112] self: [1:7587241277100383137:0] peer: [3:7587241284524967849:0] socket: 448 qp: -1
::ffff:127.1.0.0
2025-12-24T02:11:11.029091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241281395350465:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:11.029098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.028748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284720740016:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
2025-12-24T02:11:11.030870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284524967850:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:11.035142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280230000526:2098] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.035528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241280230000468:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.035548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.035555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280230000468:2048] [node 1] ICP22 created new session: [3:7587241284524967851:2048]
2025-12-24T02:11:11.035561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284524967851:2048] [node 1] ICS09 handshake done sender: [3:7587241280230000526:2098] self: [3:7587241284524967849:0] peer: [1:7587241277100383137:0] socket: 464 qp: -1
2025-12-24T02:11:11.035568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284524967851:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.035591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.035233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772712:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.035460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284720740016:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.039473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241280275535451:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.039514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241280275535394:2063] [node 6] ICP18 handshake (actor [5:7587241280275535451:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.040666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284524967854:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
2025-12-24T02:11:11.041709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284524967854:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.039098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847223:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.039153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847224:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.039361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847225:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.039427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279143879885:2082] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.040063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279143879872:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21438) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241279143879885:2082] held: no
2025-12-24T02:11:11.040081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279143879872:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.040089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.040094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279143879872:2068] [node 5] ICP22 created new session: [6:7587241283438847226:2068]
2025-12-24T02:11:11.040098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241283438847226:2068] [node 5] ICS09 handshake done sender: [6:7587241279143879885:2082] self: [6:7587241279143879925:0] peer: [5:7587241280275535407:0] socket: 458 qp: -1
2025-12-24T02:11:11.040104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241283438847226:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:11.040110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.041496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283438847223:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.041519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279143879844:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.047042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383159:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.047057Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277100383159:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.054920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241280425772633:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(51963054ns) {HandleHandshakeStatus:329(51962238ns) {DropIncomingHandshake:442(343ns) DropOutgoingHandshake:457(2719ns) SwitchToState:179(67ns) ProcessPendingSessionEvents:495(35555047ns) {ForwardSessionEventToSession:612(35539330ns) ForwardSessionEventToSession:612(2590ns) ForwardSessionEventToSession:612(1435ns) ForwardSessionEventToSession:612(644ns) ForwardSessionEventToSession:612(1292ns) ForwardSessionEventToSession:612(1678ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(761ns) ForwardSessionEventToSession:612(852ns) ForwardSessionEventToSession:612(1320ns)}}}
2025-12-24T02:11:11.054805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241277561317169:2102] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.058912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241277561317154:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.058928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.058938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241277561317154:2079] [node 6] ICP22 created new session: [4:7587241281856284487:2079]
2025-12-24T02:11:11.058948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241281856284487:2079] [node 6] ICS09 handshake done sender: [4:7587241277561317169:2102] self: [4:7587241277561317168:0] peer: [6:7587241283438847222:0] socket: 455 qp: -1
2025-12-24T02:11:11.058956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241281856284487:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:11.058967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.063684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277100383159:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.063891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277100383142:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.063919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.063930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277100383142:2108] [node 6] ICP22 created new session: [1:7587241281395350468:2108]
2025-12-24T02:11:11.063936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241281395350468:2108] [node 6] ICS09 handshake done sender: [1:7587241277100383159:2115] self: [1:7587241277100383152:0] peer: [6:7587241283438847229:0] socket: 492 qp: -1
2025-12-24T02:11:11.063944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241281395350468:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:11.063953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.064491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284720740019:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53530
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.067767Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279143879844:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241283438847223:2118] held: no
2025-12-24T02:11:11.067077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241281856284483:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.067144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284490:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.067164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241277561317108:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.070857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241277100383154:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.070870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241277100383154:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.071749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241277100383154:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.071852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241277100383141:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.071875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.071884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241277100383141:2109] [node 4] ICP22 created new session: [1:7587241281395350471:2109]
2025-12-24T02:11:11.071892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241281395350471:2109] [node 4] ICS09 handshake done sender: [1:7587241277100383154:2114] self: [1:7587241277100383151:0] peer: [4:7587241281856284491:0] socket: 486 qp: -1
2025-12-24T02:11:11.071898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241281395350471:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:11.071907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.073095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241277561317108:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241281856284483:2116] held: no
2025-12-24T02:11:11.073152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284492:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.074945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284570502756:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.075085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.075180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847232:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.073703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241281856284483:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.073769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241277561317108:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.073778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.073784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241277561317108:2048] [node 1] ICP22 created new session: [4:7587241281856284493:2048]
2025-12-24T02:11:11.073790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241281856284493:2048] [node 1] ICS09 handshake done sender: [4:7587241281856284483:2116] self: [4:7587241281856284491:0] peer: [1:7587241277100383151:0] socket: 489 qp: -1
2025-12-24T02:11:11.073795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241281856284493:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.073812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.075168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.079080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284720740019:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.082807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283438847225:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.082841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279143879886:2083] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.082887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279143879845:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.082739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000550:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.082847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241280230000469:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.082957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241280275535449:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.082999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241280275535372:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.085090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241280275535372:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241280275535449:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.083445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000522:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.083455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280230000522:2095] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.084047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280230000522:2095] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.084112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280230000510:2080] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.084129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.083060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283438847223:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.086775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279143879844:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.086791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.086797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279143879844:2048] [node 1] ICP22 created new session: [6:7587241283438847234:2048]
2025-12-24T02:11:11.086807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241283438847234:2048] [node 1] ICS09 handshake done sender: [6:7587241283438847223:2118] self: [6:7587241283438847229:0] peer: [1:7587241277100383152:0] socket: 493 qp: -1
2025-12-24T02:11:11.086813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241283438847234:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.086820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.086821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279143879883:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.086834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.086840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279143879883:2075] [node 4] ICP22 created new session: [6:7587241283438847236:2075]
2025-12-24T02:11:11.086847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241283438847236:2075] [node 4] ICS09 handshake done sender: [6:7587241279143879886:2083] self: [6:7587241283438847222:0] peer: [4:7587241277561317168:0] socket: 459 qp: -1
2025-12-24T02:11:11.086855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241283438847236:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:11.086862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.084135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280230000510:2080] [node 5] ICP22 created new session: [3:7587241284524967858:2080]
2025-12-24T02:11:11.084141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284524967858:2080] [node 5] ICS09 handshake done sender: [3:7587241280230000522:2095] self: [3:7587241280230000513:0] peer: [5:7587241284570502757:0] socket: 466 qp: -1
2025-12-24T02:11:11.084148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284524967858:2080] [node 5] ICS10 traffic start
2025-12-24T02:11:11.084165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.086878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772701:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.086888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241280425772701:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.087353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772710:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.087357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241280425772710:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.088153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241280425772701:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.088296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241280425772710:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.088414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241280425772634:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.088430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.088437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241280425772634:2049] [node 3] ICP22 created new session: [2:7587241284720740023:2049]
2025-12-24T02:11:11.088444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284720740023:2049] [node 3] ICS09 handshake done sender: [2:7587241280425772701:2110] self: [2:7587241280425772699:0] peer: [3:7587241284524967856:0] socket: 475 qp: -1
2025-12-24T02:11:11.088450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284720740023:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:11.088457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.088575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241280425772698:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.088588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.088592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241280425772698:2109] [node 6] ICP22 created new session: [2:7587241284720740025:2109]
2025-12-24T02:11:11.088597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284720740025:2109] [node 6] ICS09 handshake done sender: [2:7587241280425772710:2113] self: [2:7587241280425772700:0] peer: [6:7587241283438847233:0] socket: 477 qp: -1
2025-12-24T02:11:11.088601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284720740025:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:11.088606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.090802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241277561317185:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.090840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241277561317109:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.090993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241281856284490:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.091010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241277561317110:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.091105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000549:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.091113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280230000549:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.094714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284570502758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:11.094929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241280275535449:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.094993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279143879845:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6865) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241283438847225:2120] held: no
2025-12-24T02:11:11.095053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241280275535372:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.095068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.095075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241280275535372:2050] [node 3] ICP22 created new session: [5:7587241284570502759:2050]
2025-12-24T02:11:11.095082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284570502759:2050] [node 3] ICS09 handshake done sender: [5:7587241280275535449:2115] self: [5:7587241284570502757:0] peer: [3:7587241280230000513:0] socket: 471 qp: -1
2025-12-24T02:11:11.095094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284570502759:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.095106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.095492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241279143879921:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.095519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241279143879846:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.095171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772713:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.098733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241280425772634:2049] [node 3] ICP18 handshake (actor [2:7587241280425772713:2116]) is too old
2025-12-24T02:11:11.098762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284570502756:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.098789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241280275535399:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.098782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772703:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.098793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241280425772703:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.099660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241280425772703:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.102709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241280230000546:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.102719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241280230000546:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.102712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241280425772695:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.102733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.102739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241280425772695:2107] [node 4] ICP22 created new session: [2:7587241284720740029:2107]
2025-12-24T02:11:11.102744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284720740029:2107] [node 4] ICS09 handshake done sender: [2:7587241280425772703:2111] self: [2:7587241280425772702:0] peer: [4:7587241281856284497:0] socket: 476 qp: -1
2025-12-24T02:11:11.102750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284720740029:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:11.102757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.102842Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241277561317154:2079] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:11.104562Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241277561317109:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6865) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241277561317185:2114] held: no
2025-12-24T02:11:11.106745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241280230000469:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6865) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241280230000550:2117] held: no
2025-12-24T02:11:11.110804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280230000546:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.110832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241277561317186:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.110843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241277561317186:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.111882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241277561317186:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.111959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241277561317138:2071] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.111974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.111979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241277561317138:2071] [node 5] ICP22 created new session: [4:7587241281856284500:2071]
2025-12-24T02:11:11.111984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241281856284500:2071] [node 5] ICS09 handshake done sender: [4:7587241277561317186:2115] self: [4:7587241277561317183:0] peer: [5:7587241284570502761:0] socket: 454 qp: -1
2025-12-24T02:11:11.111988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241281856284500:2071] [node 5] ICS10 traffic start
2025-12-24T02:11:11.111994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.111461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280230000512:2078] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.111483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.111494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280230000512:2078] [node 6] ICP22 created new session: [3:7587241284524967862:2078]
2025-12-24T02:11:11.111501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284524967862:2078] [node 6] ICS09 handshake done sender: [3:7587241280230000546:2115] self: [3:7587241280230000518:0] peer: [6:7587241283438847238:0] socket: 472 qp: -1
2025-12-24T02:11:11.111506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284524967862:2078] [node 6] ICS10 traffic start
2025-12-24T02:11:11.111515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.114773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241280275535399:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28671) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241284570502756:2119] held: no
2025-12-24T02:11:11.114792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241279143879846:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241279143879921:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.118875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284502:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.122917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241277561317185:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.123041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241277561317109:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.123060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.123070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241277561317109:2049] [node 2] ICP22 created new session: [4:7587241281856284503:2049]
2025-12-24T02:11:11.123079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241281856284503:2049] [node 2] ICS09 handshake done sender: [4:7587241277561317185:2114] self: [4:7587241281856284497:0] peer: [2:7587241280425772702:0] socket: 481 qp: -1
2025-12-24T02:11:11.123088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241281856284503:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.123102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.155021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284570502763:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.155185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847239:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.155369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283438847240:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12136
2025-12-24T02:11:11.155287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284524967864:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23360
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.159239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241281856284505:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65391
2025-12-24T02:11:11.159822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241281856284490:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.159372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241280425772716:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.159386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241280425772716:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.160197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241280425772716:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.160455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241280425772696:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.160468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.160472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241280425772696:2108] [node 5] ICP22 created new session: [2:7587241284720740032:2108]
2025-12-24T02:11:11.160476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284720740032:2108] [node 5] ICS09 handshake done sender: [2:7587241280425772716:2118] self: [2:7587241280425772709:0] peer: [5:7587241284570502764:0] socket: 484 qp: -1
2025-12-24T02:11:11.160480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284720740032:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:11.160485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.159925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241277561317110:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.159942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.159947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241277561317110:2050] [node 3] ICP22 created new session: [4:7587241281856284506:2050]
2025-12-24T02:11:11.159952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241281856284506:2050] [node 3] ICS09 handshake done sender: [4:7587241281856284490:2118] self: [4:7587241281856284498:0] peer: [3:7587241280230000514:0] socket: 499 qp: -1
2025-12-24T02:11:11.159958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241281856284506:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.159963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.158751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284570502762:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:11.159042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284570502756:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.159195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284570502762:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.159228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241280275535371:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.160120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284570502765:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48723
2025-12-24T02:11:11.160307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284570502762:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.160403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241280275535399:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.160412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.160417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241280275535399:2067] [node 4] ICP22 created new session: [5:7587241284570502766:2067]
2025-12-24T02:11:11.160423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284570502766:2067] [node 4] ICS09 handshake done sender: [5:7587241284570502756:2119] self: [5:7587241284570502761:0] peer: [4:7587241277561317183:0] socket: 460 qp: -1
2025-12-24T02:11:11.160430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284570502766:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:11.160436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.160832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241280275535371:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.160842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.160847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241280275535371:2049] [node 2] ICP22 created new session: [5:7587241284570502768:2049]
2025-12-24T02:11:11.160851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284570502768:2049] [node 2] ICS09 handshake done sender: [5:7587241284570502762:2121] self: [5:7587241284570502764:0] peer: [2:7587241280425772709:0] socket: 474 qp: -1
2025-12-24T02:11:11.160855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284570502768:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.160860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.158936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241279143879921:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.159046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279143879846:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.159069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.159078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279143879846:2050] [node 3] ICP22 created new session: [6:7587241283438847241:2050]
2025-12-24T02:11:11.159085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241283438847241:2050] [node 3] ICS09 handshake done sender: [6:7587241279143879921:2116] self: [6:7587241283438847238:0] peer: [3:7587241280230000518:0] socket: 473 qp: -1
2025-12-24T02:11:11.159094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241283438847241:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.159123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.162948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283438847225:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.159116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280230000549:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.159452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241280230000550:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.159526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241280230000469:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.159542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.159548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280230000469:2049] [node 2] ICP22 created new session: [3:7587241284524967865:2049]
2025-12-24T02:11:11.159554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284524967865:2049] [node 2] ICS09 handshake done sender: [3:7587241280230000550:2117] self: [3:7587241284524967856:0] peer: [2:7587241280425772699:0] socket: 479 qp: -1
2025-12-24T02:11:11.159560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284524967865:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.159566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.160747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241280230000511:2082] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.160760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.160764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241280230000511:2082] [node 4] ICP22 created new session: [3:7587241284524967867:2082]
2025-12-24T02:11:11.160769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284524967867:2082] [node 4] ICS09 handshake done sender: [3:7587241280230000549:2116] self: [3:7587241280230000514:0] peer: [4:7587241281856284498:0] socket: 446 qp: -1
2025-12-24T02:11:11.160773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284524967867:2082] [node 4] ICS10 traffic start
2025-12-24T02:11:11.160779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.163141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241279143879845:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.163152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.163157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241279143879845:2049] [node 2] ICP22 created new session: [6:7587241283438847243:2049]
2025-12-24T02:11:11.163161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241283438847243:2049] [node 2] ICS09 handshake done sender: [6:7587241283438847225:2120] self: [6:7587241283438847233:0] peer: [2:7587241280425772700:0] socket: 501 qp: -1
2025-12-24T02:11:11.163166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241283438847243:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.163173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.190778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:11.190897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241284720740013:2048] [node 1] ICS07 socket disconnect 470 reason# EndOfStream
2025-12-24T02:11:11.190908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241284720740013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.190958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.190978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241284720740013:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:11.191441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241280425772633:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:11.191453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241284720740013:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:11.191459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241280425772633:2048] [node 1] ICP30 unregister session Session# [2:7587241284720740013:2048] VirtualId# [2:7587241280425772714:0]
2025-12-24T02:11:11.191475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.191481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241280425772633:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.191484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.194757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:11.194855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241284524967851:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-12-24T02:11:11.194863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241280230000468:2048] [node 1] ICP30 unregister session Session# [3:7587241284524967851:2048] VirtualId# [3:7587241284524967849:0]
2025-12-24T02:11:11.194870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241284524967851:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.194925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.194954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.198751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:11.198839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241280275535453:2048] [node 1] ICS07 socket disconnect 452 reason# EndOfStream
2025-12-24T02:11:11.198932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:11.199003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241283438847234:2048] [node 1] ICS01 socket: 493 reason# EndOfStream
2025-12-24T02:11:11.199008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279143879844:2048] [node 1] ICP30 unregister session Session# [6:7587241283438847234:2048] VirtualId# [6:7587241283438847229:0]
2025-12-24T02:11:11.199014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241283438847234:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.199057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.199080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.198845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241280275535453:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.198880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.198900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241280275535453:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:11.199288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241280275535370:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:11.199299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241280275535453:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:11.199303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241280275535370:2048] [node 1] ICP30 unregister session Session# [5:7587241280275535453:2048] VirtualId# [5:7587241280275535433:0]
2025-12-24T02:11:11.199326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.199331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241280275535370:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.199333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7008) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.200811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:11.200890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241281856284493:2048] [node 1] ICS01 socket: 489 reason# EndOfStream
2025-12-24T02:11:11.200896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241277561317108:2048] [node 1] ICP30 unregister session Session# [4:7587241281856284493:2048] VirtualId# [4:7587241281856284491:0]
2025-12-24T02:11:11.200902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241281856284493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.200942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.200962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.202811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:11.202878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241284524967865:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-12-24T02:11:11.202883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241280230000469:2049] [node 2] ICP30 unregister session Session# [3:7587241284524967865:2049] VirtualId# [3:7587241284524967856:0]
2025-12-24T02:11:11.202887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241284524967865:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.202914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.202931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.210803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:11.210894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241284570502768:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-12-24T02:11:11.210901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241280275535371:2049] [node 2] ICP30 unregister session Session# [5:7587241284570502768:2049] VirtualId# [5:7587241284570502764:0]
2025-12-24T02:11:11.210907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241284570502768:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.210961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.210985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.210820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:11.210893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241283438847243:2049] [node 2] ICS01 socket: 501 reason# EndOfStream
2025-12-24T02:11:11.210900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279143879845:2049] [node 2] ICP30 unregister session Session# [6:7587241283438847243:2049] VirtualId# [6:7587241283438847233:0]
2025-12-24T02:11:11.210906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241283438847243:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.210952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.210973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.211118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241280275535370:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:11.212475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:11.212559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241281856284503:2049] [node 2] ICS01 socket: 481 reason# EndOfStream
2025-12-24T02:11:11.212564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241277561317109:2049] [node 2] ICP30 unregister session Session# [4:7587241281856284503:2049] VirtualId# [4:7587241281856284497:0]
2025-12-24T02:11:11.212569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241281856284503:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.212605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.212625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.226827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:11.226914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241283438847241:2050] [node 3] ICS01 socket: 473 reason# EndOfStream
2025-12-24T02:11:11.226924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279143879846:2050] [node 3] ICP30 unregister session Session# [6:7587241283438847241:2050] VirtualId# [6:7587241283438847238:0]
2025-12-24T02:11:11.226931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241283438847241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.226977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:11.226999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:11.230305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:11.230378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241284570502759:2050] [node 3] ICS01 socket: 471 reason# EndOfStream
2025-12-24T02:11:11.230387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241280275535372:2050] [node 3] ICP30 unregister session Session# [5:7587241284570502759:2050] VirtualId# [5:7587241284570502757:0]
2025-12-24T02:11:11.230392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241284570502759:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.230430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:11.230451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:11.246887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:11.246982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241283438847236:2075] [node 4] ICS01 socket: 459 reason# EndOfStream
2025-12-24T02:11:11.246991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241279143879883:2075] [node 4] ICP30 unregister session Session# [6:7587241283438847236:2075] VirtualId# [6:7587241283438847222:0]
2025-12-24T02:11:11.246998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241283438847236:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.247052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:11.247076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:11.274824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241284527047060:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:11.274840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241284527047061:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:11.286763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241284300581791:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.286780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241284300581792:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.282756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241284648152477:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.282765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241284648152478:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:11.294898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284614269757:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.294916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284614269758:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.306719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284614269756:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.294714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241284261256838:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.294726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241284261256839:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.294732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241284261256840:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.321981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241284300581822:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:11.323178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241284648152501:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:11.323261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241284648152502:2070] [node 6] ICP01 ready to work
2025-12-24T02:11:11.338842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241284300581822:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16286834ns) {Configure:104(16280845ns) {ProcessConfigured:135(16059275ns) {StartInitialHandshake:157(16058304ns) {DropHandshakes:470(265ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(2149ns) {SwitchToState:179(1108ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.343314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269792:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.346803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241284300581792:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24117812ns) {Configure:104(24116447ns) {ProcessConfigured:135(23780063ns) {StartInitialHandshake:157(23779485ns) {DropHandshakes:470(486ns) {DropIncomingHandshake:442(98ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1428ns) {SwitchToState:179(818ns)}}}}}
2025-12-24T02:11:11.316852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241282762471672:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.316865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241282762471673:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.316871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241282762471674:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.319992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241282762471693:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:11.320083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241282762471694:2062] [node 5] ICP01 ready to work
2025-12-24T02:11:11.350667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241284527047085:2071] [node 4] ICP01 ready to work
2025-12-24T02:11:11.350857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241284527047110:2074] [node 6] ICP01 ready to work
2025-12-24T02:11:11.351356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241284527047088:2073] [node 5] ICP01 ready to work
2025-12-24T02:11:11.353280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241284300581863:2089] [node 6] ICP01 ready to work
2025-12-24T02:11:11.351499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284614269818:2063] [node 5] ICP01 ready to work
2025-12-24T02:11:11.352418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241284261256898:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:11.358109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241284648152502:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(34350853ns) {Configure:104(34349236ns) {ProcessConfigured:135(34150090ns) {StartInitialHandshake:157(34149619ns) {DropHandshakes:470(267ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1315ns) {SwitchToState:179(433ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.362916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241284527047128:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36189
2025-12-24T02:11:11.362951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241284527047129:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36189
2025-12-24T02:11:11.366918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256905:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.366940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256906:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.366950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256907:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.367360Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241284261256839:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40486385ns) {Configure:104(40485096ns) {ProcessConfigured:135(40093738ns) {StartInitialHandshake:157(40093371ns) {DropHandshakes:470(641ns) {DropIncomingHandshake:442(106ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(698ns) {SwitchToState:179(299ns)}}}}}
2025-12-24T02:11:11.367517Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241284261256840:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40605276ns) {Configure:104(40603681ns) {ProcessConfigured:135(40054089ns) {StartInitialHandshake:157(40053948ns) {DropHandshakes:470(313ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(1086ns) {SwitchToState:179(828ns)}}}}}
2025-12-24T02:11:11.367074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241284648152477:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40306256ns) {Configure:104(40305579ns) {ProcessConfigured:135(39991844ns) {StartInitialHandshake:157(39991178ns) {DropHandshakes:470(548ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(2063ns) {SwitchToState:179(1032ns)}}}}}
2025-12-24T02:11:11.367604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241284527047060:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44826164ns) {Configure:104(44825198ns) {ProcessConfigured:135(39284812ns) {StartInitialHandshake:157(39284116ns) {DropHandshakes:470(413ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(32ns) PrepareNewSessionHandshake:477(520ns) {SwitchToState:179(296ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.378852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241282762471673:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31111450ns) {Configure:104(31110693ns) {ProcessConfigured:135(30749333ns) {StartInitialHandshake:157(30748378ns) {DropHandshakes:470(809ns) {DropIncomingHandshake:442(94ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1498ns) {SwitchToState:179(539ns)}}}}}
2025-12-24T02:11:11.379014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471749:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.383264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152553:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
2025-12-24T02:11:11.383284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152554:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
2025-12-24T02:11:11.383296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152555:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
2025-12-24T02:11:11.383316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152556:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.383503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269833:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.383523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269834:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.386940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241284527047110:2074] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36011332ns) {Configure:104(36010585ns) {ProcessConfigured:135(35603608ns) {StartInitialHandshake:157(35602395ns) {DropHandshakes:470(669ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(108ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1861ns) {SwitchToState:179(1317ns)}}}}}
2025-12-24T02:11:11.387068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241284527047140:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36189
2025-12-24T02:11:11.387929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241284648152515:2072] [node 4] ICP01 ready to work
2025-12-24T02:11:11.390831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241284300581861:2087] [node 5] ICP01 ready to work
2025-12-24T02:11:11.390959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241284261256886:2069] [node 6] ICP01 ready to work
2025-12-24T02:11:11.394218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241282762471672:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(42951387ns) {Configure:104(42950530ns) {ProcessConfigured:135(42646454ns) {StartInitialHandshake:157(42645854ns) {DropHandshakes:470(383ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:477(1332ns) {SwitchToState:179(573ns)}}}}}
2025-12-24T02:11:11.394312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241284648152501:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(71001545ns) {Configure:104(71001266ns) {ProcessConfigured:135(70661721ns) {StartInitialHandshake:157(70661532ns) {DropHandshakes:470(295ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:477(1181ns) {SwitchToState:179(878ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.394725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284614269817:2061] [node 6] ICP01 ready to work
2025-12-24T02:11:11.394827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269792:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.402816Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241284527047061:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(79962823ns) {Configure:104(79961745ns) {ProcessConfigured:135(74425562ns) {StartInitialHandshake:157(74424323ns) {DropHandshakes:470(385ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(2341ns) {SwitchToState:179(1653ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.406827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581869:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.407232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241284614269756:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(56212660ns) {Configure:104(56212007ns) {ProcessConfigured:135(55838238ns) {StartInitialHandshake:157(55838112ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(1097ns) {SwitchToState:179(511ns)}}}}}
2025-12-24T02:11:11.407078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581871:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.407105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581872:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.414745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269832:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.418621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581871:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.421855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256906:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.428112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256921:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.426722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047125:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.426747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241284527047125:2104] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.428288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241284527047125:2104] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.428619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241284527047088:2073] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.428635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.428644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241284527047088:2073] [node 5] ICP22 created new session: [1:7587241284527047143:2073]
2025-12-24T02:11:11.428652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241284527047143:2073] [node 5] ICS09 handshake done sender: [1:7587241284527047125:2104] self: [1:7587241284527047124:0] peer: [5:7587241284261256920:0] socket: 467 qp: -1
2025-12-24T02:11:11.428661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241284527047143:2073] [node 5] ICS10 traffic start
2025-12-24T02:11:11.428672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.430966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269833:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.431210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047123:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.431226Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241284527047123:2103] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.434919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241284261256838:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.434975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284261256906:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.438732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284614269756:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.439025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152554:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.442814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241284261256838:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.442840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.442849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241284261256838:2048] [node 1] ICP22 created new session: [5:7587241284261256922:2048]
2025-12-24T02:11:11.442856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284261256922:2048] [node 1] ICS09 handshake done sender: [5:7587241284261256906:2107] self: [5:7587241284261256920:0] peer: [1:7587241284527047124:0] socket: 478 qp: -1
2025-12-24T02:11:11.442865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284261256922:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.442890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.451015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581869:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.451182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256905:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.450836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241284527047123:2103] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.451685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241284527047085:2071] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.451699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.451708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241284527047085:2071] [node 4] ICP22 created new session: [1:7587241284527047147:2071]
2025-12-24T02:11:11.451715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241284527047147:2071] [node 4] ICS09 handshake done sender: [1:7587241284527047123:2103] self: [1:7587241284527047122:0] peer: [4:7587241284614269839:0] socket: 466 qp: -1
2025-12-24T02:11:11.451723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241284527047147:2071] [node 4] ICS10 traffic start
2025-12-24T02:11:11.451759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.451215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.451597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284614269833:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.454972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581875:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.455001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581876:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.455176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047139:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.455186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241284527047139:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.455614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047128:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.455841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152556:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.455871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241284648152477:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.455054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284614269756:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.455081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.455087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284614269756:2048] [node 1] ICP22 created new session: [4:7587241284614269841:2048]
2025-12-24T02:11:11.455093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241284614269841:2048] [node 1] ICS09 handshake done sender: [4:7587241284614269833:2115] self: [4:7587241284614269839:0] peer: [1:7587241284527047122:0] socket: 487 qp: -1
2025-12-24T02:11:11.455099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241284614269841:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.455108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.447379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471751:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.454837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241282762471751:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.454889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241282762471672:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.458956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241284527047139:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.459135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047130:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.459139Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241284527047130:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.459913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241284527047130:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.460033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241284527047110:2074] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.460051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.460062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241284527047110:2074] [node 6] ICP22 created new session: [1:7587241284527047153:2074]
2025-12-24T02:11:11.460069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241284527047153:2074] [node 6] ICS09 handshake done sender: [1:7587241284527047139:2115] self: [1:7587241284527047118:0] peer: [6:7587241282762471752:0] socket: 494 qp: -1
2025-12-24T02:11:11.460079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241284527047153:2074] [node 6] ICS10 traffic start
2025-12-24T02:11:11.460088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.460199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241284527047060:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.460206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.460210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241284527047060:2048] [node 2] ICP22 created new session: [1:7587241284527047155:2048]
2025-12-24T02:11:11.460215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241284527047155:2048] [node 2] ICS09 handshake done sender: [1:7587241284527047130:2107] self: [1:7587241284527047095:0] peer: [2:7587241284648152561:0] socket: 482 qp: -1
2025-12-24T02:11:11.460219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241284527047155:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:11.460225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.458930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471753:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.459009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471754:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.462833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
2025-12-24T02:11:11.466793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241284648152556:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.466976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241284648152477:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.467000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.467009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241284648152477:2048] [node 1] ICP22 created new session: [2:7587241284648152563:2048]
2025-12-24T02:11:11.467016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284648152563:2048] [node 1] ICS09 handshake done sender: [2:7587241284648152556:2116] self: [2:7587241284648152561:0] peer: [1:7587241284527047095:0] socket: 486 qp: -1
2025-12-24T02:11:11.467024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284648152563:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.467045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.466779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241282762471751:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.470744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241282762471672:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.470768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.470778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241282762471672:2048] [node 1] ICP22 created new session: [6:7587241282762471755:2048]
2025-12-24T02:11:11.470784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241282762471755:2048] [node 1] ICS09 handshake done sender: [6:7587241282762471751:2116] self: [6:7587241282762471752:0] peer: [1:7587241284527047118:0] socket: 495 qp: -1
2025-12-24T02:11:11.470792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241282762471755:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.470817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.471194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152553:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.474805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269834:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.474929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256907:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.474973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241284261256898:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.478774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152555:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.478926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241284648152566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59496
2025-12-24T02:11:11.478806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269823:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.478817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241284614269823:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.480637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284614269823:2107] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.481042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241284614269818:2063] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.481063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.481070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284614269818:2063] [node 5] ICP22 created new session: [4:7587241284614269846:2063]
2025-12-24T02:11:11.481076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241284614269846:2063] [node 5] ICS09 handshake done sender: [4:7587241284614269823:2107] self: [4:7587241284614269822:0] peer: [5:7587241284261256926:0] socket: 470 qp: -1
2025-12-24T02:11:11.481083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241284614269846:2063] [node 5] ICS10 traffic start
2025-12-24T02:11:11.481093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.483272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269832:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.483324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284614269758:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.484340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581875:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.484387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581842:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.484391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241284300581842:2093] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.485044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269849:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.485214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241284300581842:2093] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.485388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284614269832:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.485477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241284300581822:2075] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.485496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.485502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241284300581822:2075] [node 4] ICP22 created new session: [3:7587241284300581879:2075]
2025-12-24T02:11:11.485519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284300581879:2075] [node 4] ICS09 handshake done sender: [3:7587241284300581842:2093] self: [3:7587241284300581824:0] peer: [4:7587241284614269848:0] socket: 462 qp: -1
2025-12-24T02:11:11.485525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284300581879:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:11.485531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.480824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256927:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.480868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256928:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.481166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284261256907:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.481225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241284261256898:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.481234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.481240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241284261256898:2064] [node 4] ICP22 created new session: [5:7587241284261256929:2064]
2025-12-24T02:11:11.481244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284261256929:2064] [node 4] ICS09 handshake done sender: [5:7587241284261256907:2108] self: [5:7587241284261256926:0] peer: [4:7587241284614269822:0] socket: 479 qp: -1
2025-12-24T02:11:11.481249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284261256929:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:11.481265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.490717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284614269758:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.490739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.490745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284614269758:2050] [node 3] ICP22 created new session: [4:7587241284614269850:2050]
2025-12-24T02:11:11.490751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241284614269850:2050] [node 3] ICS09 handshake done sender: [4:7587241284614269832:2114] self: [4:7587241284614269848:0] peer: [3:7587241284300581824:0] socket: 465 qp: -1
2025-12-24T02:11:11.490757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241284614269850:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.490780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.491020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241284614269758:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:11.494748Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241284527047060:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:11.490822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471758:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.490901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471759:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.494778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241282762471749:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.494779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241282762471759:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.494816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241282762471673:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.494826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241282762471674:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.494962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241282762471758:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.495005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241282762471694:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.494949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152537:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.494962Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241284648152537:2101] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.499265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152566:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.502826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256919:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.502843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241284261256919:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.506723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581865:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.506748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241284300581865:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.506756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256928:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.506804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241284261256839:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.509409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241284300581865:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.509510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241284300581863:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.509535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.509544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241284300581863:2089] [node 6] ICP22 created new session: [3:7587241284300581883:2089]
2025-12-24T02:11:11.509551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284300581883:2089] [node 6] ICS09 handshake done sender: [3:7587241284300581865:2111] self: [3:7587241284300581864:0] peer: [6:7587241282762471761:0] socket: 513 qp: -1
2025-12-24T02:11:11.509559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284300581883:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:11.509570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.510858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241282762471754:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.510884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241282762471674:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241282762471759:2120] held: no
2025-12-24T02:11:11.510904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241282762471693:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.510916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284261256919:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.511075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241284261256886:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.511091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.511095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241284261256886:2069] [node 6] ICP22 created new session: [5:7587241284261256935:2069]
2025-12-24T02:11:11.511102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284261256935:2069] [node 6] ICS09 handshake done sender: [5:7587241284261256919:2118] self: [5:7587241284261256918:0] peer: [6:7587241282762471762:0] socket: 488 qp: -1
2025-12-24T02:11:11.511108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284261256935:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:11.511114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.511366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256934:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.514808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241284648152537:2101] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.518736Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241282762471694:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24122) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241282762471758:2119] held: no
2025-12-24T02:11:11.518760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152558:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.518777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241284648152558:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.520306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241284648152558:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.522723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269837:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.522740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241284614269837:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.526773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241284261256934:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.526840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241284261256840:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.527531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581870:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.527543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241284300581870:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.528051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241284300581870:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.528120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241284300581861:2087] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.528138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.528144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241284300581861:2087] [node 5] ICP22 created new session: [3:7587241284300581887:2087]
2025-12-24T02:11:11.528150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284300581887:2087] [node 5] ICS09 handshake done sender: [3:7587241284300581870:2115] self: [3:7587241284300581868:0] peer: [5:7587241284261256937:0] socket: 456 qp: -1
2025-12-24T02:11:11.528155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284300581887:2087] [node 5] ICS10 traffic start
2025-12-24T02:11:11.528161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.526767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241284648152501:2068] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.526798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.526808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241284648152501:2068] [node 5] ICP22 created new session: [2:7587241284648152573:2068]
2025-12-24T02:11:11.526817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284648152573:2068] [node 5] ICS09 handshake done sender: [2:7587241284648152558:2117] self: [2:7587241284648152503:0] peer: [5:7587241284261256932:0] socket: 509 qp: -1
2025-12-24T02:11:11.526826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284648152573:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:11.526852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.530762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284614269837:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:11.531948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241284261256840:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4331) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241284261256934:2122] held: no
2025-12-24T02:11:11.531986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241284261256839:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241284261256928:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.536057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471764:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.536199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256938:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.536280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241284261256939:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14942
2025-12-24T02:11:11.534831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241284614269817:2061] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.534861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:11.534837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241284648152502:2070] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.534870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284614269817:2061] [node 6] ICP22 created new session: [4:7587241284614269853:2061]
2025-12-24T02:11:11.534871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.534879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241284648152502:2070] [node 6] ICP22 created new session: [2:7587241284648152575:2070]
2025-12-24T02:11:11.534887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284648152575:2070] [node 6] ICS09 handshake done sender: [2:7587241284648152537:2101] self: [2:7587241284648152505:0] peer: [6:7587241282762471760:0] socket: 473 qp: -1
2025-12-24T02:11:11.534893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284648152575:2070] [node 6] ICS10 traffic start
2025-12-24T02:11:11.534907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:11.534876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241284614269853:2061] [node 6] ICS09 handshake done sender: [4:7587241284614269837:2118] self: [4:7587241284614269836:0] peer: [6:7587241282762471763:0] socket: 502 qp: -1
2025-12-24T02:11:11.534883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241284614269853:2061] [node 6] ICS10 traffic start
No RDMA context found for GID: 2025-12-24T02:11:11.534892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.540374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581872:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.540461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241284300581792:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.539019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284261256928:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.539130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241284261256934:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.539203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241284261256840:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.539216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.539222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241284261256840:2050] [node 3] ICP22 created new session: [5:7587241284261256940:2050]
2025-12-24T02:11:11.539227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284261256940:2050] [node 3] ICS09 handshake done sender: [5:7587241284261256934:2122] self: [5:7587241284261256937:0] peer: [3:7587241284300581868:0] socket: 516 qp: -1
2025-12-24T02:11:11.539233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284261256940:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.539241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.539443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241284261256839:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.539452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.539459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241284261256839:2049] [node 2] ICP22 created new session: [5:7587241284261256942:2049]
2025-12-24T02:11:11.539465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241284261256942:2049] [node 2] ICS09 handshake done sender: [5:7587241284261256928:2121] self: [5:7587241284261256932:0] peer: [2:7587241284648152503:0] socket: 511 qp: -1
2025-12-24T02:11:11.539470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241284261256942:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.539476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.538794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471765:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.538830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471766:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.538847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241282762471767:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45061
2025-12-24T02:11:11.542763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152539:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.542775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241284648152539:2102] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.546776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241284648152539:2102] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:11.546910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241284648152478:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.546928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.546936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241284648152478:2049] [node 3] ICP22 created new session: [2:7587241284648152578:2049]
2025-12-24T02:11:11.546942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284648152578:2049] [node 3] ICS09 handshake done sender: [2:7587241284648152539:2102] self: [2:7587241284648152538:0] peer: [3:7587241284300581889:0] socket: 474 qp: -1
2025-12-24T02:11:11.546946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284648152578:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:11.546953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.551029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241284614269817:2061] [node 6] ICP35 event processing took too much time PendingConnection:279(16169280ns) {HandleHandshakeStatus:329(16168572ns) {DropIncomingHandshake:442(8715ns) DropOutgoingHandshake:457(82ns) SwitchToState:179(313ns) ProcessPendingSessionEvents:495(16052125ns) {ForwardSessionEventToSession:612(16040292ns) ForwardSessionEventToSession:612(1404ns) ForwardSessionEventToSession:612(734ns) ForwardSessionEventToSession:612(1716ns) ForwardSessionEventToSession:612(581ns) ForwardSessionEventToSession:612(1027ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(771ns) ForwardSessionEventToSession:612(689ns) ForwardSessionEventToSession:612(1245ns)}}}
2025-12-24T02:11:11.550944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241282762471759:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.554914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241282762471749:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.555068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581890:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.558791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241282762471758:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.558778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241284300581872:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.558821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241284300581792:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.558844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.558851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241284300581792:2049] [node 2] ICP22 created new session: [3:7587241284300581891:2049]
2025-12-24T02:11:11.558859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284300581891:2049] [node 2] ICS09 handshake done sender: [3:7587241284300581872:2117] self: [3:7587241284300581889:0] peer: [2:7587241284648152538:0] socket: 491 qp: -1
2025-12-24T02:11:11.558867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284300581891:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.558877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.562771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241284300581876:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.562834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241284300581791:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.563115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241282762471754:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.567990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241282762471694:2062] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.568013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.568024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241282762471694:2062] [node 5] ICP22 created new session: [6:7587241282762471768:2062]
2025-12-24T02:11:11.568030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241282762471768:2062] [node 5] ICS09 handshake done sender: [6:7587241282762471758:2119] self: [6:7587241282762471762:0] peer: [5:7587241284261256918:0] socket: 514 qp: -1
2025-12-24T02:11:11.568038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241282762471768:2062] [node 5] ICS10 traffic start
2025-12-24T02:11:11.568049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:11.563002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269855:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.566965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284614269855:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.567010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284614269757:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.568784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241284648152502:2070] [node 6] ICP35 event processing took too much time PendingConnection:279(50027880ns) {HandleHandshakeStatus:329(50027071ns) {DropIncomingHandshake:442(11814ns) DropOutgoingHandshake:457(218ns) SwitchToState:179(363ns) ProcessPendingSessionEvents:495(17996922ns) {ForwardSessionEventToSession:612(17984368ns) ForwardSessionEventToSession:612(1802ns) ForwardSessionEventToSession:612(1086ns) ForwardSessionEventToSession:612(754ns) ForwardSessionEventToSession:612(619ns) ForwardSessionEventToSession:612(1221ns) ForwardSessionEventToSession:612(788ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(620ns) ForwardSessionEventToSession:612(2122ns)}}}
2025-12-24T02:11:11.570912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241282762471674:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.570929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.570936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241282762471674:2050] [node 3] ICP22 created new session: [6:7587241282762471770:2050]
2025-12-24T02:11:11.570942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241282762471770:2050] [node 3] ICS09 handshake done sender: [6:7587241282762471759:2120] self: [6:7587241282762471761:0] peer: [3:7587241284300581864:0] socket: 490 qp: -1
2025-12-24T02:11:11.570948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241282762471770:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:11.570954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.571043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241282762471673:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.571050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.571054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241282762471673:2049] [node 2] ICP22 created new session: [6:7587241282762471772:2049]
2025-12-24T02:11:11.571059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241282762471772:2049] [node 2] ICS09 handshake done sender: [6:7587241282762471749:2114] self: [6:7587241282762471760:0] peer: [2:7587241284648152505:0] socket: 483 qp: -1
2025-12-24T02:11:11.571064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241282762471772:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.571068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.571472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241282762471693:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.571489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.571497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241282762471693:2059] [node 4] ICP22 created new session: [6:7587241282762471774:2059]
2025-12-24T02:11:11.571503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241282762471774:2059] [node 4] ICS09 handshake done sender: [6:7587241282762471754:2118] self: [6:7587241282762471763:0] peer: [4:7587241284614269836:0] socket: 504 qp: -1
2025-12-24T02:11:11.571509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241282762471774:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:11.571516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.571602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241282762471693:2059] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:11.574834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241282762471673:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:11.574743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241284527047141:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.574761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241284527047141:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.578794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241284300581792:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19934323ns) {HandleHandshakeStatus:329(19933456ns) {DropIncomingHandshake:442(261ns) DropOutgoingHandshake:457(4751ns) SwitchToState:179(352ns) ProcessPendingSessionEvents:495(11995480ns) {ForwardSessionEventToSession:612(11982619ns) ForwardSessionEventToSession:612(1574ns) ForwardSessionEventToSession:612(796ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(1114ns) ForwardSessionEventToSession:612(793ns) ForwardSessionEventToSession:612(665ns) ForwardSessionEventToSession:612(730ns) ForwardSessionEventToSession:612(1204ns)}}}
2025-12-24T02:11:11.574734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241284648152559:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.574750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241284648152559:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.586821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241284527047141:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.586937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241284527047061:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.586965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:11.586973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241284527047061:2049] [node 3] ICP22 created new session: [1:7587241284527047158:2049]
2025-12-24T02:11:11.586981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241284527047158:2049] [node 3] ICS09 handshake done sender: [1:7587241284527047141:2117] self: [1:7587241284527047096:0] peer: [3:7587241284300581893:0] socket: 471 qp: -1
2025-12-24T02:11:11.586988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241284527047158:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:11.587001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:11.586960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241284300581894:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60176
2025-12-24T02:11:11.587468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241284300581876:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:11.587581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241284300581791:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241284300581876:2119] held: no
2025-12-24T02:11:11.587703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241284300581791:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.587712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.587716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241284300581791:2048] [node 1] ICP22 created new session: [3:7587241284300581895:2048]
2025-12-24T02:11:11.587721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241284300581895:2048] [node 1] ICS09 handshake done sender: [3:7587241284300581876:2119] self: [3:7587241284300581893:0] peer: [1:7587241284527047096:0] socket: 500 qp: -1
2025-12-24T02:11:11.587726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241284300581895:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.587747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.586820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241284648152559:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:11.587135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241284648152515:2072] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.587157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:11.587165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241284648152515:2072] [node 4] ICP22 created new session: [2:7587241284648152581:2072]
2025-12-24T02:11:11.587172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241284648152581:2072] [node 4] ICS09 handshake done sender: [2:7587241284648152559:2118] self: [2:7587241284648152557:0] peer: [4:7587241284614269856:0] socket: 455 qp: -1
2025-12-24T02:11:11.587177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241284648152581:2072] [node 4] ICS10 traffic start
2025-12-24T02:11:11.587185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:11.586873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284614269857:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15924
2025-12-24T02:11:11.587096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284614269855:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.587735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241284614269757:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241284614269855:2121] held: no
2025-12-24T02:11:11.587832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284614269757:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.587840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.587845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284614269757:2049] [node 2] ICP22 created new session: [4:7587241284614269858:2049]
2025-12-24T02:11:11.587851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241284614269858:2049] [node 2] ICS09 handshake done sender: [4:7587241284614269855:2121] self: [4:7587241284614269856:0] peer: [2:7587241284648152557:0] socket: 517 qp: -1
2025-12-24T02:11:11.587856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241284614269858:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:11.587864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:11.610768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:11.610879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241284648152563:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-12-24T02:11:11.610889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241284648152477:2048] [node 1] ICP30 unregister session Session# [2:7587241284648152563:2048] VirtualId# [2:7587241284648152561:0]
2025-12-24T02:11:11.610898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241284648152563:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.610943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.610970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.611186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:11.611271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241284300581895:2048] [node 1] ICS01 socket: 500 reason# EndOfStream
2025-12-24T02:11:11.611280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241284300581791:2048] [node 1] ICP30 unregister session Session# [3:7587241284300581895:2048] VirtualId# [3:7587241284300581893:0]
2025-12-24T02:11:11.611286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241284300581895:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.611332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.611351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.618757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:11.618757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:11.618855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241284614269841:2048] [node 1] ICS01 socket: 487 reason# EndOfStream
2025-12-24T02:11:11.618863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241284614269756:2048] [node 1] ICP30 unregister session Session# [4:7587241284614269841:2048] VirtualId# [4:7587241284614269839:0]
2025-12-24T02:11:11.618870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241284614269841:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.618920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.618942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.619071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241282762471755:2048] [node 1] ICS01 socket: 495 reason# EndOfStream
2025-12-24T02:11:11.619082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241282762471672:2048] [node 1] ICP30 unregister session Session# [6:7587241282762471755:2048] VirtualId# [6:7587241282762471752:0]
2025-12-24T02:11:11.619088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241282762471755:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.619115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.619137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.620650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:11.620738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241284261256922:2048] [node 1] ICS07 socket disconnect 478 reason# EndOfStream
2025-12-24T02:11:11.620743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241284261256922:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.620795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:11.620816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241284261256922:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:11.621117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241284261256838:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:11.621129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241284261256922:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:11.621135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241284261256838:2048] [node 1] ICP30 unregister session Session# [5:7587241284261256922:2048] VirtualId# [5:7587241284261256920:0]
2025-12-24T02:11:11.621156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:11.621170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241284261256838:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.621173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.630767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:11.630882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241284300581891:2049] [node 2] ICS07 socket disconnect 491 reason# EndOfStream
2025-12-24T02:11:11.630894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241284300581891:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.630947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.630969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241284300581891:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:11.631408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241284300581792:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:11.631417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241284300581891:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:11.631423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241284300581792:2049] [node 2] ICP30 unregister session Session# [3:7587241284300581891:2049] VirtualId# [3:7587241284300581889:0]
2025-12-24T02:11:11.631442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.631448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241284300581792:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.631451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.634725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241284261256838:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:11.641341Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:11.641427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241284614269858:2049] [node 2] ICS01 socket: 517 reason# EndOfStream
2025-12-24T02:11:11.641435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241284614269757:2049] [node 2] ICP30 unregister session Session# [4:7587241284614269858:2049] VirtualId# [4:7587241284614269856:0]
2025-12-24T02:11:11.641442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241284614269858:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.641491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.641513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.641592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:11.641650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241282762471772:2049] [node 2] ICS07 socket disconnect 483 reason# EndOfStream
2025-12-24T02:11:11.641656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241282762471772:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.641679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.641697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241282762471772:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:11.642139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241282762471673:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:11.642148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241282762471772:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:11.642152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241282762471673:2049] [node 2] ICP30 unregister session Session# [6:7587241282762471772:2049] VirtualId# [6:7587241282762471760:0]
2025-12-24T02:11:11.642165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.642170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241282762471673:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.642173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26856) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:11.642730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:11.642806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241284261256942:2049] [node 2] ICS01 socket: 511 reason# EndOfStream
2025-12-24T02:11:11.642813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241284261256839:2049] [node 2] ICP30 unregister session Session# [5:7587241284261256942:2049] VirtualId# [5:7587241284261256932:0]
2025-12-24T02:11:11.642819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241284261256942:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.642871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:11.642894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:11.650939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241282762471673:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:11.662753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:11.662771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:11.662860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241282762471770:2050] [node 3] ICS01 socket: 490 reason# EndOfStream
2025-12-24T02:11:11.662859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241284261256940:2050] [node 3] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:11.662871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241282762471674:2050] [node 3] ICP30 unregister session Session# [6:7587241282762471770:2050] VirtualId# [6:7587241282762471761:0]
2025-12-24T02:11:11.662871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241284261256840:2050] [node 3] ICP30 unregister session Session# [5:7587241284261256940:2050] VirtualId# [5:7587241284261256937:0]
2025-12-24T02:11:11.662881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241282762471770:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.662880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241284261256940:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.662921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:11.662928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:11.662943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:11.662952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:11.686773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:11.686889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241282762471774:2059] [node 4] ICS01 socket: 504 reason# EndOfStream
2025-12-24T02:11:11.686900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241282762471693:2059] [node 4] ICP30 unregister session Session# [6:7587241282762471774:2059] VirtualId# [6:7587241282762471763:0]
2025-12-24T02:11:11.686909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241282762471774:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:11.686975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:11.687005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:11.749442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241281868009798:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.749467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241281868009799:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:11.762970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241282926736307:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:11.762987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241282926736308:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:11.762720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241283919801253:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.762740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241283919801254:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.770876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285007864738:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.770895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285007864739:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.770902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285007864740:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.770831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284129209605:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.770851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284129209606:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.770858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284129209607:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.778863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241283743320025:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:11.778885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241283743320026:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:11.778892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241283743320027:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:11.846002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285007864789:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:11.846895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241283743320068:2081] [node 5] ICP01 ready to work
2025-12-24T02:11:11.847025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241283743320071:2086] [node 4] ICP01 ready to work
2025-12-24T02:11:11.847460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285007864801:2101] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.850853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284129209663:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:11.850868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241282926736367:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:11.854888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241284129209668:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.858968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241281868009861:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:11.859034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241281868009859:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40305
2025-12-24T02:11:11.859049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241281868009860:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40305
2025-12-24T02:11:11.858836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241282926736379:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:11.859300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241282926736380:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34914
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.863583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284129209669:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:11.883128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241281868009866:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.886996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241283919801254:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34743764ns) {Configure:104(34742548ns) {ProcessConfigured:135(34356109ns) {StartInitialHandshake:157(34355568ns) {DropHandshakes:470(227ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(38ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1872ns) {SwitchToState:179(1614ns)}}}}}
2025-12-24T02:11:11.887378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241283919801320:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:11.887463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241283919801321:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:11.887533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241283919801322:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:11.887571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241283919801319:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.919405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241282926736385:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:11.919480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241282926736384:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34914
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.935908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285007864810:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:11.935939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285007864811:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.939836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736369:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.934984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241281868009798:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32159388ns) {Configure:104(32158065ns) {ProcessConfigured:135(31745116ns) {StartInitialHandshake:157(31744201ns) {DropHandshakes:470(1473ns) {DropIncomingHandshake:442(191ns) DropOutgoingHandshake:457(361ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(843ns)}}}}}
2025-12-24T02:11:11.939849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241282926736369:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.940361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241282926736388:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34914
2025-12-24T02:11:11.938852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241281868009870:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40305
2025-12-24T02:11:11.940455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241282926736369:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:11.939024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241281868009876:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:11.939078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009860:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.939552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241281868009798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.940541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241282926736307:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.940555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:11.940562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241282926736307:2048] [node 2] ICP22 created new session: [1:7587241282926736390:2048]
2025-12-24T02:11:11.940569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241282926736390:2048] [node 2] ICS09 handshake done sender: [1:7587241282926736369:2107] self: [1:7587241282926736368:0] peer: [2:7587241281868009880:0] socket: 459 qp: -1
2025-12-24T02:11:11.940576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241282926736390:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:11.940586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.971285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285007864811:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.971472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285007864816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:11.971905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285007864739:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.975097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241281868009881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40305
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.995226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009868:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.995242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241281868009868:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.995705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241281868009860:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.995031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241282926736393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34914
2025-12-24T02:11:11.996001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241281868009798:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:11.996023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:11.996030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241281868009798:2048] [node 1] ICP22 created new session: [2:7587241281868009883:2048]
2025-12-24T02:11:11.996037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241281868009883:2048] [node 1] ICS09 handshake done sender: [2:7587241281868009860:2109] self: [2:7587241281868009880:0] peer: [1:7587241282926736368:0] socket: 468 qp: -1
2025-12-24T02:11:11.996045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241281868009883:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:11.996055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:11.996967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736388:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.996999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736380:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.997103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241282926736307:2048] [node 2] ICP18 handshake (actor [1:7587241282926736388:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:11.996201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283743320103:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:11.996256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283743320104:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:11.996292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283743320105:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:11.996304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241283743320106:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:11.998317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285007864818:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:11.998606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285007864816:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.998631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285007864738:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:11.998715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736393:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.998746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736382:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:11.998750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241282926736382:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:11.999230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241282926736382:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:11.999300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241282926736379:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:11.999313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:11.999329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241282926736379:2111] [node 5] ICP22 created new session: [1:7587241282926736397:2111]
2025-12-24T02:11:11.999336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241282926736397:2111] [node 5] ICS09 handshake done sender: [1:7587241282926736382:2113] self: [1:7587241282926736381:0] peer: [5:7587241285007864819:0] socket: 493 qp: -1
2025-12-24T02:11:11.999343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241282926736397:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:11.999352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.008767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288424176976:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:12.009488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241284129209677:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.012560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284129209671:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.012575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241284129209671:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.012963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284129209669:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.012986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284129209663:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.013092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284129209605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.011685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283743320104:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.011729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241283743320071:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.011805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:12.012132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285007864810:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.012190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241285007864738:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241285007864816:2116] held: no
2025-12-24T02:11:12.014922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009859:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.018834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284129209671:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.018947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241284129209668:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.018965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.018973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284129209668:2107] [node 6] ICP22 created new session: [4:7587241288424176985:2107]
2025-12-24T02:11:12.018979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288424176985:2107] [node 6] ICS09 handshake done sender: [4:7587241284129209671:2111] self: [4:7587241284129209670:0] peer: [6:7587241288038287404:0] socket: 473 qp: -1
2025-12-24T02:11:12.018987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288424176985:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:12.018998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.022669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736384:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.018778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241288038287403:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.018813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283743320106:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.022852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241281868009868:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.018855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241289302832116:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:12.019398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285007864816:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.019469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285007864738:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.019480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.019488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285007864738:2048] [node 1] ICP22 created new session: [5:7587241289302832119:2048]
2025-12-24T02:11:12.019494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241289302832119:2048] [node 1] ICS09 handshake done sender: [5:7587241285007864816:2116] self: [5:7587241285007864819:0] peer: [1:7587241282926736381:0] socket: 497 qp: -1
2025-12-24T02:11:12.019499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241289302832119:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.019505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.026896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283743320103:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.026953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241283743320025:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.026971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241283743320068:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.026983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241283743320026:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.027117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287405:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.031030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009863:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.031043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241281868009863:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.031404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241281868009866:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.031420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.031428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241281868009866:2111] [node 5] ICP22 created new session: [2:7587241286162977184:2111]
2025-12-24T02:11:12.031433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286162977184:2111] [node 5] ICS09 handshake done sender: [2:7587241281868009868:2112] self: [2:7587241281868009867:0] peer: [5:7587241285007864817:0] socket: 477 qp: -1
2025-12-24T02:11:12.031441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286162977184:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:12.031452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.032800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286162977183:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40305
2025-12-24T02:11:12.026716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736371:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.026734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241282926736371:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.033054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241281868009863:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.033130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241281868009861:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.033147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.033152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241281868009861:2107] [node 6] ICP22 created new session: [2:7587241286162977188:2107]
2025-12-24T02:11:12.033157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286162977188:2107] [node 6] ICS09 handshake done sender: [2:7587241281868009863:2110] self: [2:7587241281868009862:0] peer: [6:7587241288038287408:0] socket: 471 qp: -1
2025-12-24T02:11:12.033161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286162977188:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:12.033169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.035157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736387:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.035168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241282926736387:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.036305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241282926736371:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.036462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286162977183:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.039470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288424176988:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:12.039520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288424176989:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:12.042967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241283919801319:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.046780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009870:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.046950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241282926736387:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.047052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241282926736367:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.047072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.047079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241282926736367:2106] [node 4] ICP22 created new session: [1:7587241287221703699:2106]
2025-12-24T02:11:12.047086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241287221703699:2106] [node 4] ICS09 handshake done sender: [1:7587241282926736371:2108] self: [1:7587241282926736370:0] peer: [4:7587241288424176983:0] socket: 462 qp: -1
2025-12-24T02:11:12.047093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241287221703699:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:12.047104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.047280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241282926736385:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.047291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.047296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241282926736385:2114] [node 6] ICP22 created new session: [1:7587241287221703701:2114]
2025-12-24T02:11:12.047300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241287221703701:2114] [node 6] ICS09 handshake done sender: [1:7587241282926736387:2116] self: [1:7587241282926736386:0] peer: [6:7587241288038287406:0] socket: 498 qp: -1
2025-12-24T02:11:12.047305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241287221703701:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:12.047310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.050784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285007864805:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.050799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241285007864805:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.051287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285007864811:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.051470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285007864739:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.051482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.051488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285007864739:2049] [node 2] ICP22 created new session: [5:7587241289302832122:2049]
2025-12-24T02:11:12.051495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241289302832122:2049] [node 2] ICS09 handshake done sender: [5:7587241285007864811:2114] self: [5:7587241285007864817:0] peer: [2:7587241281868009867:0] socket: 485 qp: -1
2025-12-24T02:11:12.051500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241289302832122:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.051507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.051741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285007864805:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.051804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241285007864801:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.051815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.051820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285007864801:2101] [node 6] ICP22 created new session: [5:7587241289302832124:2101]
2025-12-24T02:11:12.051824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241289302832124:2101] [node 6] ICS09 handshake done sender: [5:7587241285007864805:2108] self: [5:7587241285007864802:0] peer: [6:7587241288038287407:0] socket: 496 qp: -1
2025-12-24T02:11:12.051828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241289302832124:2101] [node 6] ICS10 traffic start
2025-12-24T02:11:12.051834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.058905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241283743320026:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241283743320103:2115] held: no
2025-12-24T02:11:12.066861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288424176988:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.095107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241283919801253:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.095162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
2025-12-24T02:11:12.095212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768630:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
2025-12-24T02:11:12.095254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768631:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
2025-12-24T02:11:12.095595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283743320104:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.098810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241282926736373:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.098826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241282926736373:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.099663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241283743320068:2081] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14074) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241283743320106:2118] held: no
2025-12-24T02:11:12.099719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241283743320105:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.099727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241283743320071:2086] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.099741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.099751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241283743320071:2086] [node 4] ICP22 created new session: [6:7587241288038287412:2086]
2025-12-24T02:11:12.099757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241288038287412:2086] [node 4] ICS09 handshake done sender: [6:7587241283743320104:2116] self: [6:7587241288038287404:0] peer: [4:7587241284129209670:0] socket: 483 qp: -1
2025-12-24T02:11:12.099764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241288038287412:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:12.099774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.100055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241283743320025:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241288038287403:2119] held: no
2025-12-24T02:11:12.100118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287409:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:12.100828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287410:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:12.102015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287411:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:12.099477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284129209669:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.100230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241284129209605:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241284129209669:2110] held: no
2025-12-24T02:11:12.100247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284129209605:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.100256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.100261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284129209605:2048] [node 1] ICP22 created new session: [4:7587241288424176990:2048]
2025-12-24T02:11:12.100266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288424176990:2048] [node 1] ICS09 handshake done sender: [4:7587241284129209669:2110] self: [4:7587241288424176983:0] peer: [1:7587241282926736370:0] socket: 465 qp: -1
2025-12-24T02:11:12.100271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288424176990:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.100277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.100399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241284129209677:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.100542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284129209606:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.100595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241284129209607:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.102915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241289302832126:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:12.102198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283743320103:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.102295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241283743320027:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.102338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241288038287403:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.102385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241283743320026:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.102396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.102402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241283743320026:2049] [node 2] ICP22 created new session: [6:7587241288038287417:2049]
2025-12-24T02:11:12.102408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241288038287417:2049] [node 2] ICS09 handshake done sender: [6:7587241283743320103:2115] self: [6:7587241288038287408:0] peer: [2:7587241281868009862:0] socket: 476 qp: -1
2025-12-24T02:11:12.102413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241288038287417:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.102418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.102499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283743320106:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.102532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241283743320025:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.102538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.102542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241283743320025:2048] [node 1] ICP22 created new session: [6:7587241288038287419:2048]
2025-12-24T02:11:12.102545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241288038287419:2048] [node 1] ICS09 handshake done sender: [6:7587241288038287403:2119] self: [6:7587241288038287406:0] peer: [1:7587241282926736386:0] socket: 503 qp: -1
2025-12-24T02:11:12.102549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241288038287419:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.102554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.102637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241283743320068:2081] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.102643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.102646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241283743320068:2081] [node 5] ICP22 created new session: [6:7587241288038287421:2081]
2025-12-24T02:11:12.102650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241288038287421:2081] [node 5] ICS09 handshake done sender: [6:7587241283743320106:2118] self: [6:7587241288038287407:0] peer: [5:7587241285007864802:0] socket: 499 qp: -1
2025-12-24T02:11:12.102653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241288038287421:2081] [node 5] ICS10 traffic start
2025-12-24T02:11:12.102657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.103677Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241282926736373:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.103284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009878:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.103300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241281868009878:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.106747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241288214768631:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.106779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241288214768630:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.106824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241289302832126:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.106890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285007864789:2097] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.110867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241281868009878:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.111426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288424176980:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.111447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241288424176980:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.112816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288424176995:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:12.116252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288424176988:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.118931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241281868009876:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.118962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.118972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241281868009876:2114] [node 4] ICP22 created new session: [2:7587241286162977193:2114]
2025-12-24T02:11:12.118980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286162977193:2114] [node 4] ICS09 handshake done sender: [2:7587241281868009878:2117] self: [2:7587241281868009877:0] peer: [4:7587241288424176992:0] socket: 510 qp: -1
2025-12-24T02:11:12.118987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286162977193:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:12.118999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.123120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241282926736308:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.123146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.123157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241282926736308:2049] [node 3] ICP22 created new session: [1:7587241287221703704:2049]
2025-12-24T02:11:12.123177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241287221703704:2049] [node 3] ICS09 handshake done sender: [1:7587241282926736373:2109] self: [1:7587241282926736372:0] peer: [3:7587241288214768632:0] socket: 463 qp: -1
2025-12-24T02:11:12.123187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241287221703704:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.123197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.131063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284129209606:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.131094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.131107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284129209606:2049] [node 2] ICP22 created new session: [4:7587241288424176996:2049]
2025-12-24T02:11:12.131116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288424176996:2049] [node 2] ICS09 handshake done sender: [4:7587241288424176988:2118] self: [4:7587241288424176992:0] peer: [2:7587241281868009877:0] socket: 467 qp: -1
2025-12-24T02:11:12.131125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288424176996:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.131142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.138511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241288038287423:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:12.134861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241288214768629:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.134911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241283919801324:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.134915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241283919801324:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.135539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241283919801326:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.135542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241283919801326:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.136125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241283919801254:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.136164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768633:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.139013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768634:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
2025-12-24T02:11:12.139140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241283919801324:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.139271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241283919801326:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.139406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241283919801320:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.139425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.139433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241283919801320:2107] [node 4] ICP22 created new session: [3:7587241288214768640:2107]
2025-12-24T02:11:12.139440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241288214768640:2107] [node 4] ICS09 handshake done sender: [3:7587241283919801324:2112] self: [3:7587241283919801323:0] peer: [4:7587241288424176994:0] socket: 480 qp: -1
2025-12-24T02:11:12.139447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241288214768640:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:12.139459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.139591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241283919801321:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.139955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241283743320105:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.134830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241284129209606:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.143135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288424176998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9766
2025-12-24T02:11:12.139598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.139604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241283919801321:2108] [node 6] ICP22 created new session: [3:7587241288214768642:2108]
2025-12-24T02:11:12.139608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241288214768642:2108] [node 6] ICS09 handshake done sender: [3:7587241283919801326:2113] self: [3:7587241283919801325:0] peer: [6:7587241288038287416:0] socket: 481 qp: -1
2025-12-24T02:11:12.139612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241288214768642:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:12.139616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.139692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241283919801319:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.142865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241281868009875:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.142882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241281868009875:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.140046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241283743320027:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.140061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.140068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241283743320027:2050] [node 3] ICP22 created new session: [6:7587241288038287424:2050]
2025-12-24T02:11:12.140076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241288038287424:2050] [node 3] ICS09 handshake done sender: [6:7587241283743320105:2117] self: [6:7587241288038287416:0] peer: [3:7587241283919801325:0] socket: 484 qp: -1
2025-12-24T02:11:12.140083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241288038287424:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.140093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.143818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241281868009875:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.143894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241281868009799:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.143916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.143923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241281868009799:2049] [node 3] ICP22 created new session: [2:7587241286162977196:2049]
2025-12-24T02:11:12.143930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286162977196:2049] [node 3] ICS09 handshake done sender: [2:7587241281868009875:2116] self: [2:7587241281868009874:0] peer: [3:7587241288214768638:0] socket: 504 qp: -1
2025-12-24T02:11:12.143936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286162977196:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.143946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.143746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288424176980:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.146845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241289302832128:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:12.147028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241289302832126:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.146936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241288424176976:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.146962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.146972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288424176976:2113] [node 5] ICP22 created new session: [4:7587241288424177000:2113]
2025-12-24T02:11:12.146979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288424177000:2113] [node 5] ICS09 handshake done sender: [4:7587241288424176980:2117] self: [4:7587241288424176979:0] peer: [5:7587241289302832127:0] socket: 515 qp: -1
2025-12-24T02:11:12.146989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288424177000:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:12.147000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.147091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285007864789:2097] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.147105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.147111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285007864789:2097] [node 4] ICP22 created new session: [5:7587241289302832129:2097]
2025-12-24T02:11:12.147116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241289302832129:2097] [node 4] ICS09 handshake done sender: [5:7587241289302832126:2119] self: [5:7587241289302832127:0] peer: [4:7587241288424176979:0] socket: 517 qp: -1
2025-12-24T02:11:12.147121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241289302832129:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:12.147128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.146873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241283919801253:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.146898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.146905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241283919801253:2048] [node 1] ICP22 created new session: [3:7587241288214768644:2048]
2025-12-24T02:11:12.146912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241288214768644:2048] [node 1] ICS09 handshake done sender: [3:7587241283919801319:2111] self: [3:7587241288214768632:0] peer: [1:7587241282926736372:0] socket: 479 qp: -1
2025-12-24T02:11:12.146919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241288214768644:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.146928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.147129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241288214768634:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.159507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241284129209677:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.159625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241284129209607:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9453) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241284129209677:2114] held: no
2025-12-24T02:11:12.159649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241284129209607:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.159661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.159669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241284129209607:2050] [node 3] ICP22 created new session: [4:7587241288424177002:2050]
2025-12-24T02:11:12.159676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288424177002:2050] [node 3] ICS09 handshake done sender: [4:7587241284129209677:2114] self: [4:7587241288424176994:0] peer: [3:7587241283919801323:0] socket: 488 qp: -1
2025-12-24T02:11:12.159686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288424177002:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.159697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.163434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241289302832131:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:12.165825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241289302832131:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.165862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285007864740:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.166591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241289302832133:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64054
2025-12-24T02:11:12.163098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241283919801321:2108] [node 6] ICP18 handshake (actor [3:7587241288214768634:2120]) is too old
2025-12-24T02:11:12.163148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241288214768646:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60708
2025-12-24T02:11:12.165313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241283919801254:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241288214768630:2117] held: no
2025-12-24T02:11:12.165358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241288214768630:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.165569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241283919801254:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.165583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.165590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241283919801254:2049] [node 2] ICP22 created new session: [3:7587241288214768649:2049]
2025-12-24T02:11:12.165597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241288214768649:2049] [node 2] ICS09 handshake done sender: [3:7587241288214768630:2117] self: [3:7587241288214768638:0] peer: [2:7587241281868009874:0] socket: 466 qp: -1
2025-12-24T02:11:12.165603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241288214768649:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.165611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.165988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241283919801330:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.165991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241283919801330:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.166716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241283919801330:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.166895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241289302832131:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.167062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285007864740:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.167080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.167087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285007864740:2050] [node 3] ICP22 created new session: [5:7587241289302832134:2050]
2025-12-24T02:11:12.167093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241289302832134:2050] [node 3] ICS09 handshake done sender: [5:7587241289302832131:2121] self: [5:7587241289302832132:0] peer: [3:7587241283919801329:0] socket: 457 qp: -1
2025-12-24T02:11:12.167100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241289302832134:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.167107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.170833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241283919801322:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.170866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.170879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241283919801322:2109] [node 5] ICP22 created new session: [3:7587241288214768652:2109]
2025-12-24T02:11:12.170890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241288214768652:2109] [node 5] ICS09 handshake done sender: [3:7587241283919801330:2115] self: [3:7587241283919801329:0] peer: [5:7587241289302832132:0] socket: 456 qp: -1
2025-12-24T02:11:12.170900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241288214768652:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:12.170914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.183666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:12.183779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241281868009883:2048] [node 1] ICS07 socket disconnect 468 reason# EndOfStream
2025-12-24T02:11:12.183788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241281868009883:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.183855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.183881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241281868009883:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:12.184290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241281868009798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.184302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241281868009883:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.184307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241281868009798:2048] [node 1] ICP30 unregister session Session# [2:7587241281868009883:2048] VirtualId# [2:7587241281868009880:0]
2025-12-24T02:11:12.184322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.184329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241281868009798:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.184331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.183462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:12.183565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241288214768644:2048] [node 1] ICS01 socket: 479 reason# EndOfStream
2025-12-24T02:11:12.183574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241283919801253:2048] [node 1] ICP30 unregister session Session# [3:7587241288214768644:2048] VirtualId# [3:7587241288214768632:0]
2025-12-24T02:11:12.183581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241288214768644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.183623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.183648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.189633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:12.189734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241288424176990:2048] [node 1] ICS01 socket: 465 reason# EndOfStream
2025-12-24T02:11:12.189745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241284129209605:2048] [node 1] ICP30 unregister session Session# [4:7587241288424176990:2048] VirtualId# [4:7587241288424176983:0]
2025-12-24T02:11:12.189752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241288424176990:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.189820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.189855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.189900Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:12.189982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241288038287419:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-12-24T02:11:12.189990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241283743320025:2048] [node 1] ICP30 unregister session Session# [6:7587241288038287419:2048] VirtualId# [6:7587241288038287406:0]
2025-12-24T02:11:12.189998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241288038287419:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.190035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.190061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.193481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:12.193589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241289302832119:2048] [node 1] ICS01 socket: 497 reason# EndOfStream
2025-12-24T02:11:12.193598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285007864738:2048] [node 1] ICP30 unregister session Session# [5:7587241289302832119:2048] VirtualId# [5:7587241285007864819:0]
2025-12-24T02:11:12.193607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241289302832119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.193685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.193714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.198821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:12.198945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241288214768649:2049] [node 2] ICS01 socket: 466 reason# EndOfStream
2025-12-24T02:11:12.198956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241283919801254:2049] [node 2] ICP30 unregister session Session# [3:7587241288214768649:2049] VirtualId# [3:7587241288214768638:0]
2025-12-24T02:11:12.198965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241288214768649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.199060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.199093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.206436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:12.206546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241289302832122:2049] [node 2] ICS07 socket disconnect 485 reason# EndOfStream
2025-12-24T02:11:12.206553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241289302832122:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.206611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.206634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241289302832122:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:12.207029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241285007864739:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.207042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241289302832122:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.207048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285007864739:2049] [node 2] ICP30 unregister session Session# [5:7587241289302832122:2049] VirtualId# [5:7587241285007864817:0]
2025-12-24T02:11:12.207062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.207069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241285007864739:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.207072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.209550Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:12.209656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241288038287417:2049] [node 2] ICS07 socket disconnect 476 reason# EndOfStream
2025-12-24T02:11:12.209664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241288038287417:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.209742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.209770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241288038287417:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:12.210016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:12.210091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241288424176996:2049] [node 2] ICS07 socket disconnect 467 reason# EndOfStream
2025-12-24T02:11:12.210096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241288424176996:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.210136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.210163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241288424176996:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:12.210582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241284129209606:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.210597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241288424176996:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.210602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241284129209606:2049] [node 2] ICP30 unregister session Session# [4:7587241288424176996:2049] VirtualId# [4:7587241288424176992:0]
2025-12-24T02:11:12.210619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.210625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241284129209606:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.210627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.210971Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241283743320026:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.210983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241288038287417:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.210988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241283743320026:2049] [node 2] ICP30 unregister session Session# [6:7587241288038287417:2049] VirtualId# [6:7587241288038287408:0]
2025-12-24T02:11:12.210998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.211002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241283743320026:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.211004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29085) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.218768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241285007864739:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:12.218817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:12.218922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241289302832134:2050] [node 3] ICS01 socket: 457 reason# EndOfStream
2025-12-24T02:11:12.218933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285007864740:2050] [node 3] ICP30 unregister session Session# [5:7587241289302832134:2050] VirtualId# [5:7587241289302832132:0]
2025-12-24T02:11:12.218942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241289302832134:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.219014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:12.219045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:12.221369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:12.221477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241288038287424:2050] [node 3] ICS01 socket: 484 reason# EndOfStream
2025-12-24T02:11:12.221486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241283743320027:2050] [node 3] ICP30 unregister session Session# [6:7587241288038287424:2050] VirtualId# [6:7587241288038287416:0]
2025-12-24T02:11:12.221494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241288038287424:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.221566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:12.221595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:12.227059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241283743320026:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:12.243977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:12.244102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241289302832129:2097] [node 4] ICS01 socket: 517 reason# EndOfStream
2025-12-24T02:11:12.244114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285007864789:2097] [node 4] ICP30 unregister session Session# [5:7587241289302832129:2097] VirtualId# [5:7587241289302832127:0]
2025-12-24T02:11:12.244126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241289302832129:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.244195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:12.244225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:12.245268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:12.245366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241288038287412:2086] [node 4] ICS07 socket disconnect 483 reason# EndOfStream
2025-12-24T02:11:12.245374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241288038287412:2086] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.245450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:12.245474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241288038287412:2086] [node 4] ICS15 start handshake
2025-12-24T02:11:12.245891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241283743320071:2086] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.245905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241288038287412:2086] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.245911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241283743320071:2086] [node 4] ICP30 unregister session Session# [6:7587241288038287412:2086] VirtualId# [6:7587241288038287404:0]
2025-12-24T02:11:12.245927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:12.245934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241283743320071:2086] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.245936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.254787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241283743320071:2086] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:12.306808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241289368624133:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.306830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241289368624134:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:12.314707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288581313204:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:12.318901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288581313205:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:12.302729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241285999679903:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.302752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241285999679904:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.306854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241287445956870:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.306866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241287445956871:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.302758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241285999679905:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.322827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285431468673:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.322842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285431468674:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.322847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285431468675:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.330950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241286934698799:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.330960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241286934698800:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.330966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241286934698801:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.366994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288581313258:2100] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.366989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241289368624192:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:12.367107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241289368624193:2104] [node 5] ICP01 ready to work
2025-12-24T02:11:12.368693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285431468699:2071] [node 4] ICP01 ready to work
2025-12-24T02:11:12.368821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624202:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.367087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241285999679950:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:12.371266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288581313263:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:12.374833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241287445956932:2099] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.376612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468706:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.376643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468707:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.376669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468708:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.379051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288581313273:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:12.382921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285431468720:2084] [node 6] ICP01 ready to work
2025-12-24T02:11:12.383077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241287445956934:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:12.383163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241287445956935:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:12.391171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241286934698825:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:12.391404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241286934698863:2066] [node 4] ICP01 ready to work
2025-12-24T02:11:12.419639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241289368624213:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.427065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468754:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.431246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241285999679976:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.436162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956949:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.436183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956950:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.436845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956947:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.436859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956948:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.434976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241286934698799:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43489586ns) {Configure:104(43488647ns) {ProcessConfigured:135(43007607ns) {StartInitialHandshake:157(43006604ns) {DropHandshakes:470(1369ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(2064ns) {SwitchToState:179(1100ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.458818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956948:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.458868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241287445956871:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.455375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624210:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.455453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624211:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.456235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624202:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.456424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624212:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.456469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624216:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.459035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624197:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.459043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241289368624197:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.464512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241289368624197:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.466782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241289368624134:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.466816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.466827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241289368624134:2049] [node 3] ICP22 created new session: [2:7587241289368624219:2049]
2025-12-24T02:11:12.466834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241289368624219:2049] [node 3] ICS09 handshake done sender: [2:7587241289368624197:2107] self: [2:7587241289368624196:0] peer: [3:7587241287445956953:0] socket: 470 qp: -1
2025-12-24T02:11:12.466844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241289368624219:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.466858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.475017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679984:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.475074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.475102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679981:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.475115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679982:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.475146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679983:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.478959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624210:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.482723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.482855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679982:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.482900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241285999679904:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.494728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624195:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.494755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241289368624195:2106] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.502831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624216:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.502890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241287445956871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241287445956948:2115] held: no
2025-12-24T02:11:12.502947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241287445956948:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.506774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679984:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.510793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241289368624195:2106] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.512337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288581313283:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48221
2025-12-24T02:11:12.511925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241287445956871:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.511951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.511960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241287445956871:2049] [node 2] ICP22 created new session: [3:7587241287445956955:2049]
2025-12-24T02:11:12.511969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241287445956955:2049] [node 2] ICS09 handshake done sender: [3:7587241287445956948:2115] self: [3:7587241287445956953:0] peer: [2:7587241289368624196:0] socket: 490 qp: -1
2025-12-24T02:11:12.511979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241287445956955:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.511992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.514366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313283:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.514378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241285999679905:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.514860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241289368624192:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.514881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.514891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241289368624192:2102] [node 4] ICP22 created new session: [2:7587241289368624225:2102]
2025-12-24T02:11:12.514899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241289368624225:2102] [node 4] ICS09 handshake done sender: [2:7587241289368624195:2106] self: [2:7587241289368624194:0] peer: [4:7587241285999679986:0] socket: 469 qp: -1
2025-12-24T02:11:12.514907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241289368624225:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:12.514916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.515338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679981:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.515369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679985:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.515396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241285999679903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.515573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468706:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.515734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285431468699:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.515534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313260:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.515543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288581313260:2101] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.516290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288581313260:2101] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.516367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468708:2080] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.516423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285431468673:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.515525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698877:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.519040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288581313258:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.519063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.519072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288581313258:2100] [node 4] ICP22 created new session: [1:7587241288581313286:2100]
2025-12-24T02:11:12.519080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288581313286:2100] [node 4] ICS09 handshake done sender: [1:7587241288581313260:2101] self: [1:7587241288581313259:0] peer: [4:7587241285999679988:0] socket: 463 qp: -1
2025-12-24T02:11:12.519087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288581313286:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:12.519126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.519308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313265:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.519313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288581313265:2103] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.520672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679983:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.521430Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241285999679904:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241285999679982:2116] held: no
2025-12-24T02:11:12.521479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.521616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999679991:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.521786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241285999679982:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.521847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241285999679981:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.521977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241285999679904:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.521994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.522000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241285999679904:2049] [node 2] ICP22 created new session: [4:7587241285999679994:2049]
2025-12-24T02:11:12.522005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241285999679994:2049] [node 2] ICS09 handshake done sender: [4:7587241285999679982:2116] self: [4:7587241285999679986:0] peer: [2:7587241289368624194:0] socket: 499 qp: -1
2025-12-24T02:11:12.522012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241285999679994:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.522020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.522135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241285999679903:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.522144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.522150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241285999679903:2048] [node 1] ICP22 created new session: [4:7587241285999679996:2048]
2025-12-24T02:11:12.522156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241285999679996:2048] [node 1] ICS09 handshake done sender: [4:7587241285999679981:2115] self: [4:7587241285999679988:0] peer: [1:7587241288581313259:0] socket: 497 qp: -1
2025-12-24T02:11:12.522160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241285999679996:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.522165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.522953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679954:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.522966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241285999679954:2095] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.523464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288581313265:2103] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.522871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956939:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.522890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241287445956939:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.531014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288581313263:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.531041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.531052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288581313263:2102] [node 5] ICP22 created new session: [1:7587241288581313290:2102]
2025-12-24T02:11:12.531061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288581313290:2102] [node 5] ICS09 handshake done sender: [1:7587241288581313265:2103] self: [1:7587241288581313264:0] peer: [5:7587241285431468756:0] socket: 476 qp: -1
2025-12-24T02:11:12.531069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288581313290:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:12.531080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.535137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624212:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.536184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624215:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.536190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241289368624215:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.536707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241289368624215:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.536794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241289368624213:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.536813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.536819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241289368624213:2111] [node 6] ICP22 created new session: [2:7587241289368624230:2111]
2025-12-24T02:11:12.536826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241289368624230:2111] [node 6] ICS09 handshake done sender: [2:7587241289368624215:2118] self: [2:7587241289368624214:0] peer: [6:7587241286934698880:0] socket: 508 qp: -1
2025-12-24T02:11:12.536832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241289368624230:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:12.536842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.535942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698879:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.536078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241286934698877:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.536101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241286934698800:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.537843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241286934698800:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241286934698877:2115] held: no
2025-12-24T02:11:12.537926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.538148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241286934698877:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.538207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241286934698800:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.538217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.538222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241286934698800:2049] [node 2] ICP22 created new session: [6:7587241286934698883:2049]
2025-12-24T02:11:12.538227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241286934698883:2049] [node 2] ICS09 handshake done sender: [6:7587241286934698877:2115] self: [6:7587241286934698880:0] peer: [2:7587241289368624214:0] socket: 509 qp: -1
2025-12-24T02:11:12.538231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241286934698883:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.538237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.539008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698878:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.542873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624211:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.542997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241289368624133:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.542004Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241287445956871:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(30050139ns) {HandleHandshakeStatus:329(30049005ns) {DropIncomingHandshake:442(470ns) DropOutgoingHandshake:457(334ns) SwitchToState:179(328ns) ProcessPendingSessionEvents:495(25901229ns) {ForwardSessionEventToSession:612(25899210ns)}}}
2025-12-24T02:11:12.544027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956947:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.546821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241287445956939:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.547137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241285999679954:2095] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.547235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313267:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.547248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288581313267:2104] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.547939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241289368624233:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2918
2025-12-24T02:11:12.542719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468757:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.550878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241285999679950:2092] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.550905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.550915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241285999679950:2092] [node 5] ICP22 created new session: [4:7587241285999679999:2092]
2025-12-24T02:11:12.550923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241285999679999:2092] [node 5] ICS09 handshake done sender: [4:7587241285999679954:2095] self: [4:7587241285999679953:0] peer: [5:7587241285431468755:0] socket: 467 qp: -1
2025-12-24T02:11:12.550931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241285999679999:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:12.550944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.551668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288581313267:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.551801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241286934698878:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.551840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241286934698863:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.550920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468707:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.551037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468754:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.551067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285431468708:2080] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.551421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468759:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.554752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241289368624211:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.559696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241285999679978:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.559711Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241285999679978:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.560411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.560434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698887:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.560602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241285999679978:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.560683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241285999679976:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.560700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.560706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241285999679976:2105] [node 6] ICP22 created new session: [4:7587241285999680002:2105]
2025-12-24T02:11:12.560712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241285999680002:2105] [node 6] ICS09 handshake done sender: [4:7587241285999679978:2113] self: [4:7587241285999679977:0] peer: [6:7587241286934698885:0] socket: 468 qp: -1
2025-12-24T02:11:12.560718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241285999680002:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:12.560726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.558879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285431468673:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.558915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.558928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285431468673:2048] [node 1] ICP22 created new session: [5:7587241285431468760:2048]
2025-12-24T02:11:12.558937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285431468760:2048] [node 1] ICS09 handshake done sender: [5:7587241285431468708:2080] self: [5:7587241285431468756:0] peer: [1:7587241288581313264:0] socket: 481 qp: -1
2025-12-24T02:11:12.558946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285431468760:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.558974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.559184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285431468674:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.559203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285431468675:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.561044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241285431468699:2071] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241285431468706:2078] held: no
2025-12-24T02:11:12.561230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285431468706:2078] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.561398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285431468699:2071] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.561407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.561413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285431468699:2071] [node 4] ICP22 created new session: [5:7587241285431468764:2071]
2025-12-24T02:11:12.561418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285431468764:2071] [node 4] ICS09 handshake done sender: [5:7587241285431468706:2078] self: [5:7587241285431468755:0] peer: [4:7587241285999679953:0] socket: 479 qp: -1
2025-12-24T02:11:12.561423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285431468764:2071] [node 4] ICS10 traffic start
2025-12-24T02:11:12.561428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.561523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.562942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468758:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.566801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956950:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.566871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241287445956934:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.566895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.566903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241287445956934:2104] [node 4] ICP22 created new session: [3:7587241287445956960:2104]
2025-12-24T02:11:12.566909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241287445956960:2104] [node 4] ICS09 handshake done sender: [3:7587241287445956939:2109] self: [3:7587241287445956938:0] peer: [4:7587241285999679987:0] socket: 487 qp: -1
2025-12-24T02:11:12.566917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241287445956960:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:12.566926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.567394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288581313204:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.567416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.567422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288581313204:2048] [node 2] ICP22 created new session: [1:7587241288581313294:2048]
2025-12-24T02:11:12.567428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288581313294:2048] [node 2] ICS09 handshake done sender: [1:7587241288581313267:2104] self: [1:7587241288581313266:0] peer: [2:7587241289368624232:0] socket: 477 qp: -1
2025-12-24T02:11:12.567432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288581313294:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:12.567438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.567607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956937:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.567614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241287445956937:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.570866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241286934698863:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11494) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241286934698878:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.578856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241285999680004:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58924
2025-12-24T02:11:12.578959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241286934698878:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.579163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241285999679984:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.579245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241285999679905:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.579264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.579270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241285999679905:2050] [node 3] ICP22 created new session: [4:7587241285999680005:2050]
2025-12-24T02:11:12.579276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241285999680005:2050] [node 3] ICS09 handshake done sender: [4:7587241285999679984:2118] self: [4:7587241285999679987:0] peer: [3:7587241287445956938:0] socket: 504 qp: -1
2025-12-24T02:11:12.579283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241285999680005:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.579291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.582813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241289368624199:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.582830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241289368624199:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.583190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241289368624133:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.583206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.583215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241289368624133:2048] [node 1] ICP22 created new session: [2:7587241289368624234:2048]
2025-12-24T02:11:12.583221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241289368624234:2048] [node 1] ICS09 handshake done sender: [2:7587241289368624211:2116] self: [2:7587241289368624232:0] peer: [1:7587241288581313266:0] socket: 489 qp: -1
2025-12-24T02:11:12.583229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241289368624234:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.583237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.587099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241288581313204:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19654914ns) {HandleHandshakeStatus:329(19654549ns) {DropIncomingHandshake:442(3559ns) DropOutgoingHandshake:457(114ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(19583918ns) {ForwardSessionEventToSession:612(19569628ns) ForwardSessionEventToSession:612(2319ns) ForwardSessionEventToSession:612(852ns) ForwardSessionEventToSession:612(2314ns) ForwardSessionEventToSession:612(746ns) ForwardSessionEventToSession:612(782ns) ForwardSessionEventToSession:612(732ns) ForwardSessionEventToSession:612(835ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(1236ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.593333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.593450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956949:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.593994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468768:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
2025-12-24T02:11:12.594042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285431468769:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26997
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.594775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241286934698863:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.594798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.594808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241286934698863:2066] [node 4] ICP22 created new session: [6:7587241286934698889:2066]
2025-12-24T02:11:12.594815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241286934698889:2066] [node 4] ICS09 handshake done sender: [6:7587241286934698878:2116] self: [6:7587241286934698885:0] peer: [4:7587241285999679977:0] socket: 514 qp: -1
2025-12-24T02:11:12.594823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241286934698889:2066] [node 4] ICS10 traffic start
2025-12-24T02:11:12.594832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.595075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698888:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.595123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241286934698879:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.595142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241286934698799:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.595357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241289368624199:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.596341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241289368624193:2104] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.596355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.596361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241289368624193:2104] [node 5] ICP22 created new session: [2:7587241289368624237:2104]
2025-12-24T02:11:12.596367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241289368624237:2104] [node 5] ICS09 handshake done sender: [2:7587241289368624199:2108] self: [2:7587241289368624198:0] peer: [5:7587241285431468762:0] socket: 471 qp: -1
2025-12-24T02:11:12.596373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241289368624237:2104] [node 5] ICS10 traffic start
2025-12-24T02:11:12.596380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.595569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313275:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.595579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288581313275:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.596100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288581313297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48221
2025-12-24T02:11:12.596114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288581313296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48221
2025-12-24T02:11:12.596721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288581313275:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.596776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288581313273:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.596783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.596788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288581313273:2106] [node 6] ICP22 created new session: [1:7587241288581313299:2106]
2025-12-24T02:11:12.596792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288581313299:2106] [node 6] ICS09 handshake done sender: [1:7587241288581313275:2107] self: [1:7587241288581313274:0] peer: [6:7587241286934698892:0] socket: 515 qp: -1
2025-12-24T02:11:12.596797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288581313299:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:12.596803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.598785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241287445956937:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.598960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956962:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.599909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241287445956870:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.600892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241285431468674:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241285431468707:2079] held: no
2025-12-24T02:11:12.600945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285431468707:2079] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.601013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285431468754:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.601063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285431468674:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.601077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.601083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285431468674:2049] [node 2] ICP22 created new session: [5:7587241285431468770:2049]
2025-12-24T02:11:12.601088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285431468770:2049] [node 2] ICS09 handshake done sender: [5:7587241285431468707:2079] self: [5:7587241285431468762:0] peer: [2:7587241289368624198:0] socket: 480 qp: -1
2025-12-24T02:11:12.601095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285431468770:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.601102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.601189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285431468675:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.601197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.601201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285431468675:2050] [node 3] ICP22 created new session: [5:7587241285431468772:2050]
2025-12-24T02:11:12.601204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285431468772:2050] [node 3] ICS09 handshake done sender: [5:7587241285431468754:2118] self: [5:7587241285431468763:0] peer: [3:7587241287445956936:0] socket: 488 qp: -1
2025-12-24T02:11:12.601208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285431468772:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.601212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.602789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241286934698886:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.602807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241286934698888:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.602842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698893:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.603832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241286934698825:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.603844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241286934698801:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.603957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313269:2105] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.603965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288581313269:2105] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.605170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288581313269:2105] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.605258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313296:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.605265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288581313205:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.605277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.605284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288581313205:2049] [node 3] ICP22 created new session: [1:7587241288581313302:2049]
2025-12-24T02:11:12.605289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288581313302:2049] [node 3] ICS09 handshake done sender: [1:7587241288581313269:2105] self: [1:7587241288581313268:0] peer: [3:7587241287445956967:0] socket: 511 qp: -1
2025-12-24T02:11:12.605294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288581313302:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.605301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.605412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241288581313205:2049] [node 3] ICP18 handshake (actor [1:7587241288581313296:2116]) is too old
2025-12-24T02:11:12.604497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241287445956941:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.604505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241287445956941:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.605066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241287445956968:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:314
2025-12-24T02:11:12.605770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241287445956941:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.606354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241287445956870:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23996) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241287445956962:2120] held: no
2025-12-24T02:11:12.606376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241287445956935:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.606387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.606391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241287445956935:2106] [node 6] ICP22 created new session: [3:7587241287445956970:2106]
2025-12-24T02:11:12.606396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241287445956970:2106] [node 6] ICS09 handshake done sender: [3:7587241287445956941:2110] self: [3:7587241287445956940:0] peer: [6:7587241286934698895:0] socket: 519 qp: -1
2025-12-24T02:11:12.606400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241287445956970:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:12.606406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.606503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241287445956962:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.606737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241287445956870:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.606743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.606748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241287445956870:2048] [node 1] ICP22 created new session: [3:7587241287445956972:2048]
2025-12-24T02:11:12.606752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241287445956972:2048] [node 1] ICS09 handshake done sender: [3:7587241287445956962:2120] self: [3:7587241287445956967:0] peer: [1:7587241288581313268:0] socket: 518 qp: -1
2025-12-24T02:11:12.606757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241287445956972:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.606762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.607159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288581313297:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.607183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241288581313273:2106] [node 6] ICP18 handshake (actor [1:7587241288581313297:2117]) is too old
2025-12-24T02:11:12.606945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285431468722:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.606955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241285431468722:2088] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.607903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285431468722:2088] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.607591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241287445956932:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.607606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.607613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241287445956932:2099] [node 5] ICP22 created new session: [3:7587241287445956974:2099]
2025-12-24T02:11:12.607618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241287445956974:2099] [node 5] ICS09 handshake done sender: [3:7587241287445956937:2108] self: [3:7587241287445956936:0] peer: [5:7587241285431468763:0] socket: 486 qp: -1
2025-12-24T02:11:12.607623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241287445956974:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:12.607630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.608100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241285431468720:2084] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.608116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.608121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285431468720:2084] [node 6] ICP22 created new session: [5:7587241285431468775:2084]
2025-12-24T02:11:12.608127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285431468775:2084] [node 6] ICS09 handshake done sender: [5:7587241285431468722:2088] self: [5:7587241285431468721:0] peer: [6:7587241286934698894:0] socket: 483 qp: -1
2025-12-24T02:11:12.608131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285431468775:2084] [node 6] ICS10 traffic start
2025-12-24T02:11:12.608137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.609254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241286934698801:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14849) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241286934698886:2119] held: no
2025-12-24T02:11:12.609925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241286934698825:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241286934698888:2121] held: no
2025-12-24T02:11:12.609979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698896:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.610111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241286934698897:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38667
2025-12-24T02:11:12.610236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241286934698886:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.610280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241286934698888:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.610316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241286934698801:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.610327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.610332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241286934698801:2050] [node 3] ICP22 created new session: [6:7587241286934698898:2050]
2025-12-24T02:11:12.610337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241286934698898:2050] [node 3] ICS09 handshake done sender: [6:7587241286934698886:2119] self: [6:7587241286934698895:0] peer: [3:7587241287445956940:0] socket: 522 qp: -1
2025-12-24T02:11:12.610343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241286934698898:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.610350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.610438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241286934698825:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.610443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.610447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241286934698825:2059] [node 5] ICP22 created new session: [6:7587241286934698900:2059]
2025-12-24T02:11:12.610451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241286934698900:2059] [node 5] ICS09 handshake done sender: [6:7587241286934698888:2121] self: [6:7587241286934698894:0] peer: [5:7587241285431468721:0] socket: 505 qp: -1
2025-12-24T02:11:12.610456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241286934698900:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:12.610461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.610741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241286934698879:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.610802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241286934698799:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.610814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.610820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241286934698799:2048] [node 1] ICP22 created new session: [6:7587241286934698902:2048]
2025-12-24T02:11:12.610825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241286934698902:2048] [node 1] ICS09 handshake done sender: [6:7587241286934698879:2117] self: [6:7587241286934698892:0] peer: [1:7587241288581313274:0] socket: 516 qp: -1
2025-12-24T02:11:12.610830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241286934698902:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.610835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.618772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:12.618865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241289368624234:2048] [node 1] ICS01 socket: 489 reason# EndOfStream
2025-12-24T02:11:12.618873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241289368624133:2048] [node 1] ICP30 unregister session Session# [2:7587241289368624234:2048] VirtualId# [2:7587241289368624232:0]
2025-12-24T02:11:12.618880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241289368624234:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.618925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.618949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.619035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:12.619098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241287445956972:2048] [node 1] ICS01 socket: 518 reason# EndOfStream
2025-12-24T02:11:12.619102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241287445956870:2048] [node 1] ICP30 unregister session Session# [3:7587241287445956972:2048] VirtualId# [3:7587241287445956967:0]
2025-12-24T02:11:12.619107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241287445956972:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.619134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.619152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.627641Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:12.627713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241286934698902:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:12.627719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241286934698799:2048] [node 1] ICP30 unregister session Session# [6:7587241286934698902:2048] VirtualId# [6:7587241286934698892:0]
2025-12-24T02:11:12.627724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241286934698902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.627768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.627798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.625438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:12.625503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285431468760:2048] [node 1] ICS01 socket: 481 reason# EndOfStream
2025-12-24T02:11:12.625508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285431468673:2048] [node 1] ICP30 unregister session Session# [5:7587241285431468760:2048] VirtualId# [5:7587241285431468756:0]
2025-12-24T02:11:12.625512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285431468760:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.625535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.625554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.625222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:12.625315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241285999679996:2048] [node 1] ICS01 socket: 497 reason# EndOfStream
2025-12-24T02:11:12.625324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241285999679903:2048] [node 1] ICP30 unregister session Session# [4:7587241285999679996:2048] VirtualId# [4:7587241285999679988:0]
2025-12-24T02:11:12.625330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241285999679996:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.625379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:12.625405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:12.634107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:12.634197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241287445956955:2049] [node 2] ICS07 socket disconnect 490 reason# EndOfStream
2025-12-24T02:11:12.634204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241287445956955:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.634234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.634260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241287445956955:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:12.636320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241287445956871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:12.636332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241287445956955:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:12.636603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241287445956871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:12.636614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241287445956955:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:12.636620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241287445956871:2049] [node 2] ICP30 unregister session Session# [3:7587241287445956955:2049] VirtualId# [3:7587241287445956953:0]
2025-12-24T02:11:12.636636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.636643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241287445956871:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.636646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26123) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:12.642756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:12.642855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241286934698883:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-12-24T02:11:12.642863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241286934698800:2049] [node 2] ICP30 unregister session Session# [6:7587241286934698883:2049] VirtualId# [6:7587241286934698880:0]
2025-12-24T02:11:12.642871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241286934698883:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.642925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.642948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.642802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:12.642887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285431468770:2049] [node 2] ICS01 socket: 480 reason# EndOfStream
2025-12-24T02:11:12.642894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285431468674:2049] [node 2] ICP30 unregister session Session# [5:7587241285431468770:2049] VirtualId# [5:7587241285431468762:0]
2025-12-24T02:11:12.642900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285431468770:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.642955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.642986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.642756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:12.643089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241285999679994:2049] [node 2] ICS01 socket: 499 reason# EndOfStream
2025-12-24T02:11:12.643094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241285999679904:2049] [node 2] ICP30 unregister session Session# [4:7587241285999679994:2049] VirtualId# [4:7587241285999679986:0]
2025-12-24T02:11:12.643099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241285999679994:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.643126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:12.643146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:12.650778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:12.650858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241286934698898:2050] [node 3] ICS01 socket: 522 reason# EndOfStream
2025-12-24T02:11:12.650866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241286934698801:2050] [node 3] ICP30 unregister session Session# [6:7587241286934698898:2050] VirtualId# [6:7587241286934698895:0]
2025-12-24T02:11:12.650872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241286934698898:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.650914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:12.650938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:12.652821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:12.652898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285431468772:2050] [node 3] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:12.652905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285431468675:2050] [node 3] ICP30 unregister session Session# [5:7587241285431468772:2050] VirtualId# [5:7587241285431468763:0]
2025-12-24T02:11:12.652910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285431468772:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.652946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:12.652968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:12.658878Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:12.658968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241286934698889:2066] [node 4] ICS01 socket: 514 reason# EndOfStream
2025-12-24T02:11:12.658978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241286934698863:2066] [node 4] ICP30 unregister session Session# [6:7587241286934698889:2066] VirtualId# [6:7587241286934698885:0]
2025-12-24T02:11:12.658987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241286934698889:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:12.659048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:12.659075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:12.695106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288843004714:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.695117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288843004715:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.695122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288843004716:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.706776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241287937467289:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.706791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241287937467290:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.706800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241287937467291:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.706733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288428064072:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:12.706763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288428064071:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:12.706716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241286014490903:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.726728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241286958128454:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.726790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241286958128455:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.718723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285564875601:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:12.718779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285564875603:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:12.718808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285564875602:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:12.726725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241286014490904:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:12.767074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288428064134:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:12.770798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288843004762:2087] [node 6] ICP01 ready to work
2025-12-24T02:11:12.773928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241287937467305:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:12.778789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285564875654:2069] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.781934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286014490965:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24374
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.784144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288428064140:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27952
2025-12-24T02:11:12.784197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288428064141:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27952
2025-12-24T02:11:12.784830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288428064133:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:12.787038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241285564875672:2074] [node 4] ICP01 ready to work
2025-12-24T02:11:12.788066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241288428064148:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:12.793179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241286014490968:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:12.793808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241287937467289:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19822429ns) {Configure:104(19822024ns) {ProcessConfigured:135(19463424ns) {StartInitialHandshake:157(19463120ns) {DropHandshakes:470(669ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(111ns) PrepareNewSessionHandshake:477(572ns) {SwitchToState:179(287ns)}}}}}
2025-12-24T02:11:12.793956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241287937467305:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19586220ns) {Configure:104(19585182ns) {ProcessConfigured:135(19381794ns) {StartInitialHandshake:157(19381679ns) {DropHandshakes:470(297ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(328ns) {SwitchToState:179(133ns)}}}}}
2025-12-24T02:11:12.794875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241288843004765:2090] [node 5] ICP01 ready to work
2025-12-24T02:11:12.794910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241286958128517:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:12.795019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241286958128518:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.805245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875677:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.805398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241288428064133:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20507263ns) {Configure:104(20506975ns) {ProcessConfigured:135(20251407ns) {StartInitialHandshake:157(20251275ns) {DropHandshakes:470(455ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(1232ns) {SwitchToState:179(50ns)}}}}}
2025-12-24T02:11:12.805449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064140:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.805617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288428064151:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27952
2025-12-24T02:11:12.803694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241286958128526:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:12.804072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128520:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.805881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241286014490975:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:12.806322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286014490974:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24374
2025-12-24T02:11:12.806818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286014490965:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.807498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241288428064148:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19362587ns) {Configure:104(19361748ns) {ProcessConfigured:135(19063805ns) {StartInitialHandshake:157(19062818ns) {DropHandshakes:470(579ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:477(651ns) {SwitchToState:179(410ns)}}}}}
2025-12-24T02:11:12.808068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064141:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.810717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241287937467351:2065] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.818792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241285564875603:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47232999ns) {Configure:104(47231605ns) {ProcessConfigured:135(46913999ns) {StartInitialHandshake:157(46913517ns) {DropHandshakes:470(333ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(770ns) {SwitchToState:179(341ns)}}}}}
2025-12-24T02:11:12.818929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241288843004714:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23978258ns) {Configure:104(23977682ns) {ProcessConfigured:135(23662324ns) {StartInitialHandshake:157(23662047ns) {DropHandshakes:470(394ns) {DropIncomingHandshake:442(38ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(22ns)}}}}}
2025-12-24T02:11:12.819606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004786:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.819619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004787:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.822284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004792:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.820275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467354:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.820435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467362:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.820447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467363:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.820458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467364:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.820604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241287937467291:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26605167ns) {Configure:104(26604604ns) {ProcessConfigured:135(26350271ns) {StartInitialHandshake:157(26350116ns) {DropHandshakes:470(341ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(600ns) {SwitchToState:179(130ns)}}}}}
2025-12-24T02:11:12.819775Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241286958128517:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24710262ns) {Configure:104(24708921ns) {ProcessConfigured:135(24390073ns) {StartInitialHandshake:157(24389823ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(83ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(867ns) {SwitchToState:179(388ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.823254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241286014490980:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:12.823680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241286014490980:2114] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.824131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241286014490904:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30409959ns) {Configure:104(30408976ns) {ProcessConfigured:135(30209883ns) {StartInitialHandshake:157(30209587ns) {DropHandshakes:470(314ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(106ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(707ns) {SwitchToState:179(100ns)}}}}}
2025-12-24T02:11:12.824025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875681:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
2025-12-24T02:11:12.824049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
2025-12-24T02:11:12.826786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241287937467354:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.826823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241287937467351:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.838170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241287937467290:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27351542ns) {Configure:104(27350161ns) {ProcessConfigured:135(26898063ns) {StartInitialHandshake:157(26897447ns) {DropHandshakes:470(407ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(849ns) {SwitchToState:179(621ns)}}}}}
2025-12-24T02:11:12.834913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241288843004716:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(60018022ns) {Configure:104(60016557ns) {ProcessConfigured:135(59711343ns) {StartInitialHandshake:157(59710717ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1244ns) {SwitchToState:179(766ns)}}}}}
2025-12-24T02:11:12.851690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875657:2100] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.851706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241285564875657:2100] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.851795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875661:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.851800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241285564875661:2102] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.854925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004787:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.854968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241288843004715:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.859890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286014490977:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.859903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241286014490977:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.860408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286014490983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24374
2025-12-24T02:11:12.860571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285564875657:2100] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.860670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241286014490977:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.860847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285564875661:2102] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.861057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241286014490975:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.861075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.861081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241286014490975:2111] [node 4] ICP22 created new session: [2:7587241286014490985:2111]
2025-12-24T02:11:12.861087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286014490985:2111] [node 4] ICS09 handshake done sender: [2:7587241286014490977:2113] self: [2:7587241286014490976:0] peer: [4:7587241288843004796:0] socket: 493 qp: -1
2025-12-24T02:11:12.861094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286014490985:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:12.861120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.861266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241286014490965:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.861354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241285564875602:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.861368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.861373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285564875602:2049] [node 2] ICP22 created new session: [5:7587241285564875685:2049]
2025-12-24T02:11:12.861378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285564875685:2049] [node 2] ICS09 handshake done sender: [5:7587241285564875657:2100] self: [5:7587241285564875613:0] peer: [2:7587241286014490981:0] socket: 470 qp: -1
2025-12-24T02:11:12.861382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285564875685:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.861387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.861473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241285564875654:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.861479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.861484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285564875654:2069] [node 6] ICP22 created new session: [5:7587241285564875687:2069]
2025-12-24T02:11:12.861489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285564875687:2069] [node 6] ICS09 handshake done sender: [5:7587241285564875661:2102] self: [5:7587241285564875660:0] peer: [6:7587241287937467371:0] socket: 472 qp: -1
2025-12-24T02:11:12.861492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285564875687:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:12.861496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.862074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004797:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.862039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241287937467363:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.862170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241287937467290:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.862261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467373:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.862323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288843004787:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.862443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241287937467354:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.862506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241287937467351:2065] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.862387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241288843004715:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.862402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.862407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288843004715:2049] [node 2] ICP22 created new session: [4:7587241288843004799:2049]
2025-12-24T02:11:12.862412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288843004799:2049] [node 2] ICS09 handshake done sender: [4:7587241288843004787:2114] self: [4:7587241288843004796:0] peer: [2:7587241286014490976:0] socket: 496 qp: -1
2025-12-24T02:11:12.862417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288843004799:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.862428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.862519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.862525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241287937467351:2065] [node 5] ICP22 created new session: [6:7587241287937467375:2065]
2025-12-24T02:11:12.862531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241287937467375:2065] [node 5] ICS09 handshake done sender: [6:7587241287937467354:2108] self: [6:7587241287937467371:0] peer: [5:7587241285564875660:0] socket: 485 qp: -1
2025-12-24T02:11:12.862536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241287937467375:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:12.862551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.862976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288428064155:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27952
2025-12-24T02:11:12.862828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241287937467362:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.863018Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241286958128455:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(60761370ns) {Configure:104(60760475ns) {ProcessConfigured:135(60355994ns) {StartInitialHandshake:157(60355654ns) {DropHandshakes:470(521ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(409ns) PrepareNewSessionHandshake:477(782ns) {SwitchToState:179(138ns)}}}}}
2025-12-24T02:11:12.866820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875682:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.868015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064151:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.867740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128534:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.867757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128535:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.870950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241288428064156:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27952
2025-12-24T02:11:12.873880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128520:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.873929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241286958128454:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.874729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286014490970:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.874753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241286014490970:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.874895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241286014490980:2114] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.874908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.874916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241286014490980:2114] [node 5] ICP22 created new session: [2:7587241286014490988:2114]
2025-12-24T02:11:12.874929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286014490988:2114] [node 5] ICS09 handshake done sender: [2:7587241286014490965:2107] self: [2:7587241286014490981:0] peer: [5:7587241285564875613:0] socket: 473 qp: -1
2025-12-24T02:11:12.874938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286014490988:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:12.874948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.875414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241286014490970:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.875553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241286014490968:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.875567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.875572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241286014490968:2109] [node 6] ICP22 created new session: [2:7587241286014490991:2109]
2025-12-24T02:11:12.875577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286014490991:2109] [node 6] ICS09 handshake done sender: [2:7587241286014490970:2110] self: [2:7587241286014490969:0] peer: [6:7587241287937467374:0] socket: 487 qp: -1
2025-12-24T02:11:12.875583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286014490991:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:12.875589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.878646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241287937467305:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.874767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241285564875654:2069] [node 6] ICP18 handshake (actor [5:7587241285564875682:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.878862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064156:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.878946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467377:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.882855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004786:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.883077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286014490974:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.883111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241286014490903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.883251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004778:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.883257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241288843004778:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.887662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128537:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.888193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241286958128520:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.888259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241286958128454:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.888268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.888273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241286958128454:2048] [node 1] ICP22 created new session: [3:7587241286958128538:2048]
2025-12-24T02:11:12.888278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241286958128538:2048] [node 1] ICS09 handshake done sender: [3:7587241286958128520:2109] self: [3:7587241286958128536:0] peer: [1:7587241288428064137:0] socket: 481 qp: -1
2025-12-24T02:11:12.888283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241286958128538:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.888288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.886797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064139:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.886815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288428064139:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.887534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288428064139:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.887780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288428064072:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.887801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.887807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288428064072:2049] [node 3] ICP22 created new session: [1:7587241288428064160:2049]
2025-12-24T02:11:12.887815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288428064160:2049] [node 3] ICS09 handshake done sender: [1:7587241288428064139:2111] self: [1:7587241288428064137:0] peer: [3:7587241286958128536:0] socket: 479 qp: -1
2025-12-24T02:11:12.887821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288428064160:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.887830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.888725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064143:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.888729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288428064143:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.889892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288428064143:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.889972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288428064071:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.890009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.890014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288428064071:2048] [node 2] ICP22 created new session: [1:7587241288428064164:2048]
2025-12-24T02:11:12.890022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288428064164:2048] [node 2] ICS09 handshake done sender: [1:7587241288428064143:2114] self: [1:7587241288428064142:0] peer: [2:7587241286014490993:0] socket: 478 qp: -1
2025-12-24T02:11:12.890027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288428064164:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:12.890035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.909550Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241286014490903:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241286014490974:2112] held: no
2025-12-24T02:11:12.907489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004798:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.907650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241288843004716:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.907759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004802:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.909696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288843004778:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.909796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241288843004762:2087] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.909810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.909820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288843004762:2087] [node 6] ICP22 created new session: [4:7587241288843004805:2087]
2025-12-24T02:11:12.909826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288843004805:2087] [node 6] ICS09 handshake done sender: [4:7587241288843004778:2107] self: [4:7587241288843004763:0] peer: [6:7587241287937467378:0] socket: 480 qp: -1
2025-12-24T02:11:12.909832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288843004805:2087] [node 6] ICS10 traffic start
2025-12-24T02:11:12.909841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.906983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128535:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.907032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241286958128455:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.909983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241287937467363:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.910093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.910211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467380:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.910219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241287937467290:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.910229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.910233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241287937467290:2049] [node 2] ICP22 created new session: [6:7587241287937467382:2049]
2025-12-24T02:11:12.910239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241287937467382:2049] [node 2] ICS09 handshake done sender: [6:7587241287937467363:2115] self: [6:7587241287937467374:0] peer: [2:7587241286014490969:0] socket: 499 qp: -1
2025-12-24T02:11:12.910244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241287937467382:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.910249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.911077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286014490994:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24374
2025-12-24T02:11:12.911451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241287937467362:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.911702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241286014490974:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.911874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241286014490903:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.911888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.911895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241286014490903:2048] [node 1] ICP22 created new session: [2:7587241286014490995:2048]
2025-12-24T02:11:12.911900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286014490995:2048] [node 1] ICS09 handshake done sender: [2:7587241286014490974:2112] self: [2:7587241286014490993:0] peer: [1:7587241288428064142:0] socket: 482 qp: -1
2025-12-24T02:11:12.911908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286014490995:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.911913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.912119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004798:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.912125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241287937467305:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.912138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.912143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241287937467305:2060] [node 4] ICP22 created new session: [6:7587241287937467384:2060]
2025-12-24T02:11:12.912148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241287937467384:2060] [node 4] ICS09 handshake done sender: [6:7587241287937467362:2114] self: [6:7587241287937467378:0] peer: [4:7587241288843004763:0] socket: 486 qp: -1
2025-12-24T02:11:12.912152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241287937467384:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:12.912157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.912572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128523:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.912583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241286958128523:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.914853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875691:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.915892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004808:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.916096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875681:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.916167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285564875672:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.916712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875677:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.916763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285564875601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.916865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064138:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.916877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288428064138:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.917538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
2025-12-24T02:11:12.918815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288428064138:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.915975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241286958128523:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.916436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241286958128518:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.916446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.916452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241286958128518:2108] [node 4] ICP22 created new session: [3:7587241286958128543:2108]
2025-12-24T02:11:12.916458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241286958128543:2108] [node 4] ICS09 handshake done sender: [3:7587241286958128523:2111] self: [3:7587241286958128522:0] peer: [4:7587241288843004804:0] socket: 488 qp: -1
2025-12-24T02:11:12.916464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241286958128543:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:12.916471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.918199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128534:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.918969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288428064134:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.918991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.918997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288428064134:2107] [node 5] ICP22 created new session: [1:7587241288428064168:2107]
2025-12-24T02:11:12.919003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288428064168:2107] [node 5] ICS09 handshake done sender: [1:7587241288428064138:2110] self: [1:7587241288428064135:0] peer: [5:7587241285564875693:0] socket: 477 qp: -1
2025-12-24T02:11:12.919009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288428064168:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:12.919030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.919212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285564875677:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.917711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004789:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.917720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241288843004789:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.918124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241288843004802:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.918145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288843004786:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.919069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241288843004714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.919212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288843004789:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.922727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241288843004765:2090] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.922742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.922749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288843004765:2090] [node 5] ICP22 created new session: [4:7587241288843004811:2090]
2025-12-24T02:11:12.922755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288843004811:2090] [node 5] ICS09 handshake done sender: [4:7587241288843004789:2115] self: [4:7587241288843004788:0] peer: [5:7587241285564875692:0] socket: 498 qp: -1
2025-12-24T02:11:12.922760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288843004811:2090] [node 5] ICS10 traffic start
2025-12-24T02:11:12.922769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:12.922858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241287937467364:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.923088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241287937467291:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.929093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285564875601:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.929115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.929125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285564875601:2048] [node 1] ICP22 created new session: [5:7587241285564875695:2048]
2025-12-24T02:11:12.929131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285564875695:2048] [node 1] ICS09 handshake done sender: [5:7587241285564875677:2114] self: [5:7587241285564875693:0] peer: [1:7587241288428064135:0] socket: 491 qp: -1
2025-12-24T02:11:12.929140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285564875695:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.929148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.930197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128528:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.926717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241286014490982:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.926736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241286014490982:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.930209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241286958128528:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.930666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128546:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.930715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.930902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241285564875601:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:12.930956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875697:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
2025-12-24T02:11:12.931354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285564875681:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.931437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285564875672:2074] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.931469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.931475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285564875672:2074] [node 4] ICP22 created new session: [5:7587241285564875698:2074]
2025-12-24T02:11:12.931480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285564875698:2074] [node 4] ICS09 handshake done sender: [5:7587241285564875681:2117] self: [5:7587241285564875692:0] peer: [4:7587241288843004788:0] socket: 507 qp: -1
2025-12-24T02:11:12.931487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285564875698:2074] [node 4] ICS10 traffic start
2025-12-24T02:11:12.931493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.931058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241288843004716:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.931082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.931091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288843004716:2050] [node 3] ICP22 created new session: [4:7587241288843004813:2050]
2025-12-24T02:11:12.931099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288843004813:2050] [node 3] ICS09 handshake done sender: [4:7587241288843004786:2113] self: [4:7587241288843004804:0] peer: [3:7587241286958128522:0] socket: 495 qp: -1
2025-12-24T02:11:12.931107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288843004813:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.931130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.934856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241287937467380:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.934897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241287937467289:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.935001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241286958128528:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.935233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128547:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.935017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064153:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.935026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288428064153:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.935452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241286958128526:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.935465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.935471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241286958128526:2110] [node 6] ICP22 created new session: [3:7587241286958128551:2110]
2025-12-24T02:11:12.935477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241286958128551:2110] [node 6] ICS09 handshake done sender: [3:7587241286958128528:2113] self: [3:7587241286958128527:0] peer: [6:7587241287937467386:0] socket: 490 qp: -1
2025-12-24T02:11:12.935483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241286958128551:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:12.935492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.935690Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064150:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.935697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241288428064150:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.935802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288428064153:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:12.940873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288428064148:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.940895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:12.940904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288428064148:2109] [node 6] ICP22 created new session: [1:7587241288428064171:2109]
2025-12-24T02:11:12.940911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288428064171:2109] [node 6] ICS09 handshake done sender: [1:7587241288428064153:2117] self: [1:7587241288428064149:0] peer: [6:7587241287937467387:0] socket: 475 qp: -1
2025-12-24T02:11:12.940917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288428064171:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:12.940924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:12.946814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241286014490982:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.946974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241286014490904:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.946990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.946997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241286014490904:2049] [node 3] ICP22 created new session: [2:7587241286014490998:2049]
2025-12-24T02:11:12.947003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241286014490998:2049] [node 3] ICS09 handshake done sender: [2:7587241286014490982:2115] self: [2:7587241286014490971:0] peer: [3:7587241286958128541:0] socket: 516 qp: -1
2025-12-24T02:11:12.947009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241286014490998:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:12.947017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.950985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241286958128553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56121
2025-12-24T02:11:12.951166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241286958128535:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:12.951211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241286958128455:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.951225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:12.951232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241286958128455:2049] [node 2] ICP22 created new session: [3:7587241286958128554:2049]
2025-12-24T02:11:12.951238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241286958128554:2049] [node 2] ICS09 handshake done sender: [3:7587241286958128535:2117] self: [3:7587241286958128541:0] peer: [2:7587241286014490971:0] socket: 521 qp: -1
2025-12-24T02:11:12.951244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241286958128554:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:12.951252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:12.956796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241285564875691:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.956849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241285564875603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:12.956470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241288843004765:2090] [node 5] ICP35 event processing took too much time PendingConnection:279(33707994ns) {HandleHandshakeStatus:329(33706935ns) {DropIncomingHandshake:442(3060ns) DropOutgoingHandshake:457(117ns) SwitchToState:179(451ns) ProcessPendingSessionEvents:495(33617708ns) {ForwardSessionEventToSession:612(33607527ns) ForwardSessionEventToSession:612(1312ns) ForwardSessionEventToSession:612(656ns) ForwardSessionEventToSession:612(616ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(784ns) ForwardSessionEventToSession:612(939ns) ForwardSessionEventToSession:612(808ns) ForwardSessionEventToSession:612(678ns) ForwardSessionEventToSession:612(614ns)}}}
2025-12-24T02:11:12.958853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128546:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.959069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241286958128518:2108] [node 4] ICP18 handshake (actor [3:7587241286958128546:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.962919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241288843004815:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5954
2025-12-24T02:11:12.963423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241287937467291:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241287937467364:2116] held: no
2025-12-24T02:11:12.964059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241287937467289:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241287937467380:2123] held: no
2025-12-24T02:11:12.959419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241288428064155:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.962825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241286014491000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24374
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.966808Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241288843004716:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(35715788ns) {HandleHandshakeStatus:329(35714560ns) {DropIncomingHandshake:442(210ns) DropOutgoingHandshake:457(1128ns) SwitchToState:179(265ns) ProcessPendingSessionEvents:495(27812090ns) {ForwardSessionEventToSession:612(27796282ns) ForwardSessionEventToSession:612(3195ns) ForwardSessionEventToSession:612(1200ns) ForwardSessionEventToSession:612(796ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(1148ns) ForwardSessionEventToSession:612(861ns) ForwardSessionEventToSession:612(682ns) ForwardSessionEventToSession:612(696ns)}}}
2025-12-24T02:11:12.966837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241288843004762:2087] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.967155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467388:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.969823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241287937467389:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23819
2025-12-24T02:11:12.969972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241286958128532:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:12.969992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241286958128532:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:12.970073Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241286958128526:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(34584624ns) {HandleHandshakeStatus:329(34583975ns) {DropIncomingHandshake:442(1640ns) DropOutgoingHandshake:457(109ns) SwitchToState:179(67ns) ProcessPendingSessionEvents:495(34515300ns) {ForwardSessionEventToSession:612(34500058ns) ForwardSessionEventToSession:612(2627ns) ForwardSessionEventToSession:612(643ns) ForwardSessionEventToSession:612(1995ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(695ns) ForwardSessionEventToSession:612(600ns) ForwardSessionEventToSession:612(764ns) ForwardSessionEventToSession:612(732ns)}}}
2025-12-24T02:11:12.970147Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241286014490904:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(23144729ns) {HandleHandshakeStatus:329(23144041ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(320ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(23061144ns) {ForwardSessionEventToSession:612(23048345ns) ForwardSessionEventToSession:612(2902ns) ForwardSessionEventToSession:612(1482ns) ForwardSessionEventToSession:612(789ns) ForwardSessionEventToSession:612(555ns) ForwardSessionEventToSession:612(850ns) ForwardSessionEventToSession:612(735ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(955ns) ForwardSessionEventToSession:612(652ns)}}}
2025-12-24T02:11:12.970237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241286014490904:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:12.970301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241288843004802:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.966789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241288428064150:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:12.975863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241288843004714:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.975893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.975904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241288843004714:2048] [node 1] ICP22 created new session: [4:7587241288843004816:2048]
2025-12-24T02:11:12.975912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241288843004816:2048] [node 1] ICS09 handshake done sender: [4:7587241288843004802:2121] self: [4:7587241288843004810:0] peer: [1:7587241288428064147:0] socket: 529 qp: -1
2025-12-24T02:11:12.975921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241288843004816:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.975929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.979664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241286958128532:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:12.979787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241286958128517:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.979809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:12.979817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241286958128517:2107] [node 5] ICP22 created new session: [3:7587241286958128557:2107]
2025-12-24T02:11:12.979824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241286958128557:2107] [node 5] ICS09 handshake done sender: [3:7587241286958128532:2114] self: [3:7587241286958128519:0] peer: [5:7587241285564875700:0] socket: 527 qp: -1
2025-12-24T02:11:12.979831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241286958128557:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:12.979840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:12.981311Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241285564875603:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241285564875691:2119] held: no
2025-12-24T02:11:12.981395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241285564875701:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58360
2025-12-24T02:11:12.981565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241285564875691:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.981641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241285564875603:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.981655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.981662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241285564875603:2050] [node 3] ICP22 created new session: [5:7587241285564875702:2050]
2025-12-24T02:11:12.981667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241285564875702:2050] [node 3] ICS09 handshake done sender: [5:7587241285564875691:2119] self: [5:7587241285564875700:0] peer: [3:7587241286958128519:0] socket: 528 qp: -1
2025-12-24T02:11:12.981672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241285564875702:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.981677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.978933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241287937467364:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:12.983125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241287937467291:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.983141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:12.983148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241287937467291:2050] [node 3] ICP22 created new session: [6:7587241287937467390:2050]
2025-12-24T02:11:12.983155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241287937467390:2050] [node 3] ICS09 handshake done sender: [6:7587241287937467364:2116] self: [6:7587241287937467386:0] peer: [3:7587241286958128527:0] socket: 500 qp: -1
2025-12-24T02:11:12.983161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241287937467390:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:12.983167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:12.983259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241287937467380:2123] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:12.983388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241287937467289:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:12.983398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:12.983402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241287937467289:2048] [node 1] ICP22 created new session: [6:7587241287937467392:2048]
2025-12-24T02:11:12.983406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241287937467392:2048] [node 1] ICS09 handshake done sender: [6:7587241287937467380:2123] self: [6:7587241287937467387:0] peer: [1:7587241288428064149:0] socket: 503 qp: -1
2025-12-24T02:11:12.983410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241287937467392:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:12.983416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:12.982774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241288428064133:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:12.982796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:12.982804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241288428064133:2108] [node 4] ICP22 created new session: [1:7587241288428064175:2108]
2025-12-24T02:11:12.982809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241288428064175:2108] [node 4] ICS09 handshake done sender: [1:7587241288428064150:2115] self: [1:7587241288428064147:0] peer: [4:7587241288843004810:0] socket: 492 qp: -1
2025-12-24T02:11:12.982814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241288428064175:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:12.982822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:12.998876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241288428064133:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(16058093ns) {HandleHandshakeStatus:329(16057292ns) {DropIncomingHandshake:442(3674ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(225ns) ProcessPendingSessionEvents:495(15973267ns) {ForwardSessionEventToSession:612(15953767ns) ForwardSessionEventToSession:612(3082ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(2168ns) ForwardSessionEventToSession:612(625ns) ForwardSessionEventToSession:612(1148ns) ForwardSessionEventToSession:612(962ns) ForwardSessionEventToSession:612(750ns) ForwardSessionEventToSession:612(833ns) ForwardSessionEventToSession:612(2007ns)}}}
2025-12-24T02:11:13.006779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:13.006883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241286014490995:2048] [node 1] ICS07 socket disconnect 482 reason# EndOfStream
2025-12-24T02:11:13.006890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241286014490995:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.006950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.006978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241286014490995:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.007389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241286014490903:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.007401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241286014490995:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.007407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241286014490903:2048] [node 1] ICP30 unregister session Session# [2:7587241286014490995:2048] VirtualId# [2:7587241286014490993:0]
2025-12-24T02:11:13.007421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.007429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241286014490903:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.007431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.007356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:13.007437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241286958128538:2048] [node 1] ICS07 socket disconnect 481 reason# EndOfStream
2025-12-24T02:11:13.007443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241286958128538:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.007482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.007501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241286958128538:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.014944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241286958128454:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.014968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241286958128538:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.014976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241286958128454:2048] [node 1] ICP30 unregister session Session# [3:7587241286958128538:2048] VirtualId# [3:7587241286958128536:0]
2025-12-24T02:11:13.014995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.015002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241286958128454:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.015006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.020439Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:13.020534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285564875695:2048] [node 1] ICS01 socket: 491 reason# EndOfStream
2025-12-24T02:11:13.020543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285564875601:2048] [node 1] ICP30 unregister session Session# [5:7587241285564875695:2048] VirtualId# [5:7587241285564875693:0]
2025-12-24T02:11:13.020551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285564875695:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.020597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.020621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.020766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:13.020836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241287937467392:2048] [node 1] ICS01 socket: 503 reason# EndOfStream
2025-12-24T02:11:13.020845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241287937467289:2048] [node 1] ICP30 unregister session Session# [6:7587241287937467392:2048] VirtualId# [6:7587241287937467387:0]
2025-12-24T02:11:13.020850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241287937467392:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.020875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.020896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.020853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:13.020929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241288843004816:2048] [node 1] ICS01 socket: 529 reason# EndOfStream
2025-12-24T02:11:13.020935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241288843004714:2048] [node 1] ICP30 unregister session Session# [4:7587241288843004816:2048] VirtualId# [4:7587241288843004810:0]
2025-12-24T02:11:13.020941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241288843004816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.020983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.021005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.034070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:13.034179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:13.034256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241288843004799:2049] [node 2] ICS07 socket disconnect 496 reason# EndOfStream
2025-12-24T02:11:13.034262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241288843004799:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.034308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.034329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241288843004799:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:13.034156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241287937467382:2049] [node 2] ICS01 socket: 499 reason# EndOfStream
2025-12-24T02:11:13.034168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241287937467290:2049] [node 2] ICP30 unregister session Session# [6:7587241287937467382:2049] VirtualId# [6:7587241287937467374:0]
2025-12-24T02:11:13.034175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241287937467382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.034219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.034241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.034807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241288843004715:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13919) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.034819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241288843004799:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.034823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241288843004715:2049] [node 2] ICP30 unregister session Session# [4:7587241288843004799:2049] VirtualId# [4:7587241288843004796:0]
2025-12-24T02:11:13.034837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.034843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241288843004715:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.034845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13919) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.034773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:13.034845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285564875685:2049] [node 2] ICS01 socket: 470 reason# EndOfStream
2025-12-24T02:11:13.034851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285564875602:2049] [node 2] ICP30 unregister session Session# [5:7587241285564875685:2049] VirtualId# [5:7587241285564875613:0]
2025-12-24T02:11:13.034857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285564875685:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.034896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.034919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.046784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241288843004715:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:13.050743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:13.050823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241285564875702:2050] [node 3] ICS01 socket: 528 reason# EndOfStream
2025-12-24T02:11:13.050831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241285564875603:2050] [node 3] ICP30 unregister session Session# [5:7587241285564875702:2050] VirtualId# [5:7587241285564875700:0]
2025-12-24T02:11:13.050838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241285564875702:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.050877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:13.050899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:13.054870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:13.054964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241287937467390:2050] [node 3] ICS01 socket: 500 reason# EndOfStream
2025-12-24T02:11:13.054973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241287937467291:2050] [node 3] ICP30 unregister session Session# [6:7587241287937467390:2050] VirtualId# [6:7587241287937467386:0]
2025-12-24T02:11:13.054980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241287937467390:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.055034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:13.055062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:13.094912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:13.095036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241287937467384:2060] [node 4] ICS01 socket: 486 reason# EndOfStream
2025-12-24T02:11:13.095049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241287937467305:2060] [node 4] ICP30 unregister session Session# [6:7587241287937467384:2060] VirtualId# [6:7587241287937467378:0]
2025-12-24T02:11:13.095057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241287937467384:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.095127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:13.095157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:13.143971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290678871737:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:13.143981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290678871738:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:13.142826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241289746359880:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.142835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241289746359881:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.142825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291264671051:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.146962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291264671052:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:13.174811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292790072816:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.174825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292790072817:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.174838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292790072818:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.178720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241291392636622:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.178738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241291392636623:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.178743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241291392636624:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.194868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241291957322554:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.194881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241291957322555:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.194886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241291957322556:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.260145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290678871799:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:13.260271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290678871800:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.261815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636683:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.260986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241291957322596:2082] [node 5] ICP01 ready to work
2025-12-24T02:11:13.261089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241291957322597:2085] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.262236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359941:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.262257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359942:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.262267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359940:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.262477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636684:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.271152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072877:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.275368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072878:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.274757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290678871811:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:13.279210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241289746359945:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:13.280027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291264671118:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:13.286918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291264671123:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:13.287109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241291264671051:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22507893ns) {Configure:104(22506926ns) {ProcessConfigured:135(22123806ns) {StartInitialHandshake:157(22123376ns) {DropHandshakes:470(688ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(709ns) {SwitchToState:179(181ns)}}}}}
2025-12-24T02:11:13.289515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291264671117:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.291500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636685:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.291374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322628:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.311598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241291264671129:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-12-24T02:11:13.311636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241291264671117:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22042565ns) {Configure:104(22041311ns) {ProcessConfigured:135(21722954ns) {StartInitialHandshake:157(21722283ns) {DropHandshakes:470(462ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(1003ns) {SwitchToState:179(958ns)}}}}}
2025-12-24T02:11:13.314752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241289746359952:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:13.322926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241289746359954:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:13.323323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292790072880:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.330934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359960:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.342926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241289746359954:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19912288ns) {Configure:104(19911339ns) {ProcessConfigured:135(19587138ns) {StartInitialHandshake:157(19586343ns) {DropHandshakes:470(991ns) {DropIncomingHandshake:442(171ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(2057ns) {SwitchToState:179(660ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.351499Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241289746359952:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36654732ns) {Configure:104(36653216ns) {ProcessConfigured:135(36313224ns) {StartInitialHandshake:157(36312875ns) {DropHandshakes:470(1684ns) {DropIncomingHandshake:442(373ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(1226ns) {SwitchToState:179(601ns)}}}}}
2025-12-24T02:11:13.350972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241292790072817:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27165357ns) {Configure:104(8156003ns) {ProcessConfigured:135(7743929ns) {StartInitialHandshake:157(7741982ns) {DropHandshakes:470(1433ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(235ns)} GenerateSessionVirtualId:988(487ns) PrepareNewSessionHandshake:477(1370ns) {SwitchToState:179(1023ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.354890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241292790072880:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31467570ns) {Configure:104(31466183ns) {ProcessConfigured:135(31140516ns) {StartInitialHandshake:157(31139875ns) {DropHandshakes:470(369ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(1190ns) {SwitchToState:179(402ns)}}}}}
2025-12-24T02:11:13.354735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241290678871816:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4459
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.357302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636689:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.357885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241290678871818:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4459
2025-12-24T02:11:13.366957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241290678871817:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4459
2025-12-24T02:11:13.367019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241291392636683:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.370950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241291264671131:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-12-24T02:11:13.375173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871802:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.375191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290678871802:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.386733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241291392636622:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.386803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241291392636622:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19683967ns) {IncomingHandshake:261(19682670ns) {DropIncomingHandshake:442(319ns) PrepareNewSessionHandshake:477(1115ns) {SwitchToState:179(413ns)} GenerateSessionVirtualId:988(321ns) IssueIncomingHandshakeReply:193(19672819ns) {SwitchToState:179(1029ns)}}}
2025-12-24T02:11:13.387067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671129:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.390743Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241292790072816:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28300578ns) {Configure:104(28299142ns) {ProcessConfigured:135(27972118ns) {StartInitialHandshake:157(27970714ns) {DropHandshakes:470(488ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(1484ns) {SwitchToState:179(564ns)}}}}}
2025-12-24T02:11:13.391156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290678871802:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:13.394909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292790072893:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:13.395718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072877:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.395756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241292790072816:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.399563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359941:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.399597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359942:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.399628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241289746359880:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.399722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241291392636684:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.400483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241291392636692:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:13.400869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241291392636692:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.406799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871804:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.406822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871806:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.406825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290678871804:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.406828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290678871806:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.406817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241291957322599:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.406836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241291957322599:2089] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.407535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241291957322599:2089] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:13.407642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241291957322596:2082] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.407660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:13.407667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241291957322596:2082] [node 5] ICP22 created new session: [6:7587241291957322630:2082]
2025-12-24T02:11:13.407674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241291957322630:2082] [node 5] ICS09 handshake done sender: [6:7587241291957322599:2089] self: [6:7587241291957322598:0] peer: [5:7587241291392636693:0] socket: 479 qp: -1
2025-12-24T02:11:13.407683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241291957322630:2082] [node 5] ICS10 traffic start
2025-12-24T02:11:13.407707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:13.414731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290678871799:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.414770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:13.414787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290678871799:2107] [node 5] ICP22 created new session: [1:7587241290678871821:2107]
2025-12-24T02:11:13.414795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241290678871821:2107] [node 5] ICS09 handshake done sender: [1:7587241290678871802:2109] self: [1:7587241290678871801:0] peer: [5:7587241291392636690:0] socket: 475 qp: -1
2025-12-24T02:11:13.414805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241290678871821:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:13.414825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.418824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359940:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.418891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241289746359881:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.419682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359966:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.420457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241289746359967:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11643
2025-12-24T02:11:13.420771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241289746359940:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:13.420852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241289746359881:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.420885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:13.420894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241289746359881:2049] [node 2] ICP22 created new session: [3:7587241289746359968:2049]
2025-12-24T02:11:13.420901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241289746359968:2049] [node 2] ICS09 handshake done sender: [3:7587241289746359940:2107] self: [3:7587241289746359965:0] peer: [2:7587241291264671111:0] socket: 483 qp: -1
2025-12-24T02:11:13.420908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241289746359968:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:13.420917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:13.422802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290678871806:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.419742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671112:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.419761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241291264671112:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.420577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291264671112:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:13.422967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291264671052:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.422989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.422998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291264671052:2049] [node 3] ICP22 created new session: [2:7587241291264671134:2049]
2025-12-24T02:11:13.423004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241291264671134:2049] [node 3] ICS09 handshake done sender: [2:7587241291264671112:2107] self: [2:7587241291264671111:0] peer: [3:7587241289746359965:0] socket: 474 qp: -1
2025-12-24T02:11:13.423010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241291264671134:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:13.423031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:13.422893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072878:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.426785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241291957322596:2082] [node 5] ICP35 event processing took too much time PendingConnection:279(19109276ns) {HandleHandshakeStatus:329(19108354ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(376ns) SwitchToState:179(278ns) ProcessPendingSessionEvents:495(7982175ns) {ForwardSessionEventToSession:612(7967956ns) ForwardSessionEventToSession:612(1844ns) ForwardSessionEventToSession:612(1119ns) ForwardSessionEventToSession:612(766ns) ForwardSessionEventToSession:612(812ns) ForwardSessionEventToSession:612(1678ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(857ns) ForwardSessionEventToSession:612(743ns) ForwardSessionEventToSession:612(1172ns)}}}
2025-12-24T02:11:13.426937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290678871738:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.426959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.426968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290678871738:2049] [node 3] ICP22 created new session: [1:7587241290678871824:2049]
2025-12-24T02:11:13.426975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241290678871824:2049] [node 3] ICS09 handshake done sender: [1:7587241290678871806:2111] self: [1:7587241290678871805:0] peer: [3:7587241289746359963:0] socket: 477 qp: -1
2025-12-24T02:11:13.426982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241290678871824:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:13.427006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.430961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636694:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.431019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636695:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.431209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241291392636684:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:13.431283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241291392636692:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.431298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:13.431304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241291392636692:2113] [node 6] ICP22 created new session: [5:7587241291392636696:2113]
2025-12-24T02:11:13.431312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241291392636696:2113] [node 6] ICS09 handshake done sender: [5:7587241291392636684:2109] self: [5:7587241291392636693:0] peer: [6:7587241291957322598:0] socket: 487 qp: -1
2025-12-24T02:11:13.431320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241291392636696:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:13.431355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:13.431661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241291392636683:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:13.431729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241291392636622:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.431739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:13.431744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241291392636622:2048] [node 1] ICP22 created new session: [5:7587241291392636698:2048]
2025-12-24T02:11:13.431750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241291392636698:2048] [node 1] ICS09 handshake done sender: [5:7587241291392636683:2108] self: [5:7587241291392636690:0] peer: [1:7587241290678871801:0] socket: 478 qp: -1
2025-12-24T02:11:13.431755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241291392636698:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:13.431784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.435549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241291264671136:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-12-24T02:11:13.438882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241291392636689:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.439086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241291392636701:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:13.439488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241291392636701:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.439200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871817:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.439240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871815:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.439243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290678871815:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.439825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290678871815:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:13.439910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290678871737:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.439930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:13.439936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290678871737:2048] [node 2] ICP22 created new session: [1:7587241290678871829:2048]
2025-12-24T02:11:13.439941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241290678871829:2048] [node 2] ICS09 handshake done sender: [1:7587241290678871815:2114] self: [1:7587241290678871812:0] peer: [2:7587241291264671138:0] socket: 499 qp: -1
2025-12-24T02:11:13.439947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241290678871829:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:13.439954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:13.439041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671136:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.439085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241291264671051:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.439746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671131:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.443109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241289746359941:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.443723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241291264671051:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241291264671136:2117] held: no
2025-12-24T02:11:13.443744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241291957322628:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.444164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241291957322555:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.446745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072889:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.446762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241292790072889:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.451120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359960:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.451146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241289746359880:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.451165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:13.451174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241289746359880:2048] [node 1] ICP22 created new session: [3:7587241289746359971:2048]
2025-12-24T02:11:13.451181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241289746359971:2048] [node 1] ICS09 handshake done sender: [3:7587241289746359941:2108] self: [3:7587241289746359963:0] peer: [1:7587241290678871805:0] socket: 484 qp: -1
2025-12-24T02:11:13.451189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241289746359971:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:13.451199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:13.454215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671120:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.454236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241291264671120:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.450727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072897:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.454679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871816:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.454751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290678871804:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:13.454795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291264671120:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.454954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241291392636685:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.454911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291264671118:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.454926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:13.454934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291264671118:2109] [node 6] ICP22 created new session: [2:7587241291264671142:2109]
2025-12-24T02:11:13.454940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241291264671142:2109] [node 6] ICS09 handshake done sender: [2:7587241291264671120:2110] self: [2:7587241291264671119:0] peer: [6:7587241291957322632:0] socket: 492 qp: -1
2025-12-24T02:11:13.454946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241291264671142:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:13.454967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.455366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241291392636624:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.454863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241290678871738:2049] [node 3] ICP18 handshake (actor [1:7587241290678871816:2115]) is too old
2025-12-24T02:11:13.455024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072897:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.455194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322633:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.455074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072899:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.455235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292790072877:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:13.455275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241292790072818:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.456420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241292790072816:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.456435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:13.456443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292790072816:2048] [node 1] ICP22 created new session: [4:7587241292790072902:2048]
2025-12-24T02:11:13.456449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241292790072902:2048] [node 1] ICS09 handshake done sender: [4:7587241292790072877:2108] self: [4:7587241292790072896:0] peer: [1:7587241290678871803:0] socket: 488 qp: -1
2025-12-24T02:11:13.456456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241292790072902:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:13.456475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:13.456779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072901:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.458847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241290678871799:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(59890499ns) {HandleHandshakeStatus:329(59885442ns) {DropIncomingHandshake:442(462ns) DropOutgoingHandshake:457(693ns) SwitchToState:179(216ns) ProcessPendingSessionEvents:495(32011160ns) {ForwardSessionEventToSession:612(31998534ns) ForwardSessionEventToSession:612(1972ns) ForwardSessionEventToSession:612(993ns) ForwardSessionEventToSession:612(794ns) ForwardSessionEventToSession:612(928ns) ForwardSessionEventToSession:612(1043ns) ForwardSessionEventToSession:612(804ns) ForwardSessionEventToSession:612(858ns) ForwardSessionEventToSession:612(665ns)}}}
2025-12-24T02:11:13.455413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359961:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.455423Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241289746359961:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.459194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241289746359961:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:13.459040Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241291957322633:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.459395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241291957322554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.459170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241291264671140:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22324
2025-12-24T02:11:13.462955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291264671136:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:13.463049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241291264671051:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.463066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:13.463093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291264671051:2048] [node 1] ICP22 created new session: [2:7587241291264671145:2048]
2025-12-24T02:11:13.463099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241291264671145:2048] [node 1] ICS09 handshake done sender: [2:7587241291264671136:2117] self: [2:7587241291264671138:0] peer: [1:7587241290678871812:0] socket: 524 qp: -1
2025-12-24T02:11:13.463105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241291264671145:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:13.463113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.462823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322634:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.462878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.466989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241289746359954:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.467017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:13.467024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241289746359954:2114] [node 4] ICP22 created new session: [3:7587241289746359976:2114]
2025-12-24T02:11:13.467031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241289746359976:2114] [node 4] ICS09 handshake done sender: [3:7587241289746359961:2117] self: [3:7587241289746359959:0] peer: [4:7587241292790072900:0] socket: 521 qp: -1
2025-12-24T02:11:13.467037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241289746359976:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:13.467057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:13.466744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290678871800:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.466773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:13.466781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072901:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.467934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241292790072817:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.470867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241291957322628:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:13.470973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241291957322555:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.470987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:13.470993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241291957322555:2049] [node 2] ICP22 created new session: [6:7587241291957322637:2049]
2025-12-24T02:11:13.471000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241291957322637:2049] [node 2] ICS09 handshake done sender: [6:7587241291957322628:2113] self: [6:7587241291957322632:0] peer: [2:7587241291264671119:0] socket: 494 qp: -1
2025-12-24T02:11:13.471005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241291957322637:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:13.471012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:13.470850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671130:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.470861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241291264671130:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.466783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290678871800:2108] [node 4] ICP22 created new session: [1:7587241290678871832:2108]
2025-12-24T02:11:13.466795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241290678871832:2108] [node 4] ICS09 handshake done sender: [1:7587241290678871804:2110] self: [1:7587241290678871803:0] peer: [4:7587241292790072896:0] socket: 476 qp: -1
2025-12-24T02:11:13.466804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241290678871832:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:13.466814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:13.470944Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241290678871799:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:13.471893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291264671130:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:13.472264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291264671117:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.472279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:13.472283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291264671117:2108] [node 4] ICP22 created new session: [2:7587241291264671148:2108]
2025-12-24T02:11:13.472288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241291264671148:2108] [node 4] ICS09 handshake done sender: [2:7587241291264671130:2115] self: [2:7587241291264671127:0] peer: [4:7587241292790072906:0] socket: 532 qp: -1
2025-12-24T02:11:13.472292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241291264671148:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:13.472298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:13.470793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292790072889:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:13.471989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241292790072880:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.471998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:13.472003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292790072880:2110] [node 5] ICP22 created new session: [4:7587241292790072907:2110]
2025-12-24T02:11:13.472009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241292790072907:2110] [node 5] ICS09 handshake done sender: [4:7587241292790072889:2113] self: [4:7587241292790072884:0] peer: [5:7587241291392636702:0] socket: 508 qp: -1
2025-12-24T02:11:13.472014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241292790072907:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:13.472020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:13.472134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292790072818:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241292790072897:2117] held: no
2025-12-24T02:11:13.473051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292790072817:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241292790072901:2119] held: no
2025-12-24T02:11:13.475564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636705:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.475504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241291264671052:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(52501786ns) {HandleHandshakeStatus:329(52500958ns) {DropIncomingHandshake:442(5552ns) DropOutgoingHandshake:457(127ns) SwitchToState:179(222ns) ProcessPendingSessionEvents:495(28669605ns) {ForwardSessionEventToSession:612(28657426ns) ForwardSessionEventToSession:612(1888ns) ForwardSessionEventToSession:612(1263ns) ForwardSessionEventToSession:612(721ns) ForwardSessionEventToSession:612(1180ns) ForwardSessionEventToSession:612(1026ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(681ns) ForwardSessionEventToSession:612(719ns)}}}
2025-12-24T02:11:13.478725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290678871814:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.478747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290678871814:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.478729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359947:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.478751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241289746359947:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.478800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.479060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241291392636689:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:13.479229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241291392636704:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.479630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241291392636623:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.480338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636707:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.480445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241291392636701:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.480454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:13.480461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241291392636701:2116] [node 4] ICP22 created new session: [5:7587241291392636708:2116]
2025-12-24T02:11:13.480467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241291392636708:2116] [node 4] ICS09 handshake done sender: [5:7587241291392636689:2112] self: [5:7587241291392636702:0] peer: [4:7587241292790072884:0] socket: 509 qp: -1
2025-12-24T02:11:13.480473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241291392636708:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:13.480498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:13.481000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241291392636704:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:13.481040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241291392636623:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.481048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:13.481052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241291392636623:2049] [node 2] ICP22 created new session: [5:7587241291392636710:2049]
2025-12-24T02:11:13.481057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241291392636710:2049] [node 2] ICS09 handshake done sender: [5:7587241291392636704:2117] self: [5:7587241291392636706:0] peer: [2:7587241291264671126:0] socket: 525 qp: -1
2025-12-24T02:11:13.481062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241291392636710:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:13.481068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:13.482539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241291957322635:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.482596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241291957322556:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:13.479755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291264671128:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.479765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241291264671128:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.480863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291264671128:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:13.481251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291264671123:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.481258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:13.481263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291264671123:2111] [node 5] ICP22 created new session: [2:7587241291264671151:2111]
2025-12-24T02:11:13.481267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241291264671151:2111] [node 5] ICS09 handshake done sender: [2:7587241291264671128:2113] self: [2:7587241291264671126:0] peer: [5:7587241291392636706:0] socket: 501 qp: -1
2025-12-24T02:11:13.481270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241291264671151:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:13.481276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.487401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241289746359962:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.487414Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241289746359962:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.489140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241289746359962:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:13.489242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241289746359952:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.489263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:13.489270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241289746359952:2113] [node 6] ICP22 created new session: [3:7587241289746359979:2113]
2025-12-24T02:11:13.489277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241289746359979:2113] [node 6] ICS09 handshake done sender: [3:7587241289746359962:2118] self: [3:7587241289746359958:0] peer: [6:7587241291957322639:0] socket: 530 qp: -1
2025-12-24T02:11:13.489284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241289746359979:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:13.489290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:13.488025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322640:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.488056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322641:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.490855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241291957322556:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241291957322635:2116] held: no
2025-12-24T02:11:13.491155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241291957322635:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:13.491257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241291957322556:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.491271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.491277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241291957322556:2050] [node 3] ICP22 created new session: [6:7587241291957322642:2050]
2025-12-24T02:11:13.491284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241291957322642:2050] [node 3] ICS09 handshake done sender: [6:7587241291957322635:2116] self: [6:7587241291957322639:0] peer: [3:7587241289746359958:0] socket: 533 qp: -1
2025-12-24T02:11:13.491291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241291957322642:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:13.491300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:13.503756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241289746359947:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:13.503897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241289746359945:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.503913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:13.503949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241289746359945:2110] [node 5] ICP22 created new session: [3:7587241289746359982:2110]
2025-12-24T02:11:13.503956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241289746359982:2110] [node 5] ICS09 handshake done sender: [3:7587241289746359947:2111] self: [3:7587241289746359946:0] peer: [5:7587241291392636703:0] socket: 491 qp: -1
2025-12-24T02:11:13.503963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241289746359982:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:13.503971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:13.502954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290678871814:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:13.503215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290678871811:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.503231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:13.503242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290678871811:2112] [node 6] ICP22 created new session: [1:7587241290678871835:2112]
2025-12-24T02:11:13.503247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241290678871835:2112] [node 6] ICS09 handshake done sender: [1:7587241290678871814:2113] self: [1:7587241290678871813:0] peer: [6:7587241291957322636:0] socket: 514 qp: -1
2025-12-24T02:11:13.503254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241290678871835:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:13.503268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:13.514948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241291957322640:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.515026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241291957322597:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.518805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292790072895:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.518823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241292790072895:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.519148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072910:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.523227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241291392636712:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4966
2025-12-24T02:11:13.522985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241290678871811:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(19724900ns) {HandleHandshakeStatus:329(19723811ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(264ns) SwitchToState:179(278ns) ProcessPendingSessionEvents:495(19629915ns) {ForwardSessionEventToSession:612(19615007ns) ForwardSessionEventToSession:612(2560ns) ForwardSessionEventToSession:612(1239ns) ForwardSessionEventToSession:612(691ns) ForwardSessionEventToSession:612(1755ns) ForwardSessionEventToSession:612(856ns) ForwardSessionEventToSession:612(898ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(580ns) ForwardSessionEventToSession:612(632ns)}}}
2025-12-24T02:11:13.523024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292790072895:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:13.523112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241292790072893:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.523132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:13.523140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292790072893:2115] [node 6] ICP22 created new session: [4:7587241292790072912:2115]
2025-12-24T02:11:13.523147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241292790072912:2115] [node 6] ICS09 handshake done sender: [4:7587241292790072895:2116] self: [4:7587241292790072894:0] peer: [6:7587241291957322644:0] socket: 535 qp: -1
2025-12-24T02:11:13.523154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241292790072912:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:13.523162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:13.526730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241292790072909:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2804
2025-12-24T02:11:13.526867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292790072901:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:13.527085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241292790072817:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.527102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:13.527133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292790072817:2049] [node 2] ICP22 created new session: [4:7587241292790072914:2049]
2025-12-24T02:11:13.527139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241292790072914:2049] [node 2] ICS09 handshake done sender: [4:7587241292790072901:2119] self: [4:7587241292790072906:0] peer: [2:7587241291264671127:0] socket: 534 qp: -1
2025-12-24T02:11:13.527146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241292790072914:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:13.527153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:13.527595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292790072897:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:13.527667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241292790072818:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.527679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.527685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292790072818:2050] [node 3] ICP22 created new session: [4:7587241292790072916:2050]
2025-12-24T02:11:13.527690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241292790072916:2050] [node 3] ICS09 handshake done sender: [4:7587241292790072897:2117] self: [4:7587241292790072900:0] peer: [3:7587241289746359959:0] socket: 522 qp: -1
2025-12-24T02:11:13.527695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241292790072916:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:13.527702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:13.526830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241291392636685:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.530877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322645:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.530952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241291957322646:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31819
2025-12-24T02:11:13.531101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241291957322633:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:13.534848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241291392636624:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.534872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.534879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241291392636624:2050] [node 3] ICP22 created new session: [5:7587241291392636713:2050]
2025-12-24T02:11:13.534885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241291392636713:2050] [node 3] ICS09 handshake done sender: [5:7587241291392636685:2110] self: [5:7587241291392636703:0] peer: [3:7587241289746359946:0] socket: 495 qp: -1
2025-12-24T02:11:13.534893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241291392636713:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:13.534907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:13.542885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241291957322597:2085] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62474) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241291957322640:2117] held: no
2025-12-24T02:11:13.542965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241291957322597:2085] [node 4] ICP35 event processing took too much time PendingConnection:279(19437833ns) {HandleHandshakeStatus:395(19437028ns) {LogHandshakeFail:472(3489ns) {UpdateErrorStateLog:545(726ns)} DropOutgoingHandshake:457(1222ns) ProcessPendingSessionEvents:495(22407ns) {DropSessionEvent:512(8740ns) DropSessionEvent:512(2166ns) DropSessionEvent:512(810ns) DropSessionEvent:512(1424ns) DropSessionEvent:512(1369ns) DropSessionEvent:512(2216ns) DropSessionEvent:512(826ns) DropSessionEvent:512(1276ns) DropSessionEvent:512(1093ns) DropSessionEvent:512(817ns)}}}
2025-12-24T02:11:13.542988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241291957322640:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:13.543008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241291957322554:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.543026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:13.543037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241291957322554:2048] [node 1] ICP22 created new session: [6:7587241291957322647:2048]
2025-12-24T02:11:13.543044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241291957322647:2048] [node 1] ICS09 handshake done sender: [6:7587241291957322633:2114] self: [6:7587241291957322636:0] peer: [1:7587241290678871813:0] socket: 515 qp: -1
2025-12-24T02:11:13.543052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241291957322647:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:13.543062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:13.546717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241291957322597:2085] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:13.546735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:13.546741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241291957322597:2085] [node 4] ICP22 created new session: [6:7587241291957322649:2085]
2025-12-24T02:11:13.546748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241291957322649:2085] [node 4] ICS09 handshake done sender: [6:7587241291957322640:2117] self: [6:7587241291957322644:0] peer: [4:7587241292790072894:0] socket: 496 qp: -1
2025-12-24T02:11:13.546753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241291957322649:2085] [node 4] ICS10 traffic start
2025-12-24T02:11:13.546761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:13.558823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241291957322554:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27637874ns) {HandleHandshakeStatus:329(27637202ns) {DropIncomingHandshake:442(1139ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(412ns) ProcessPendingSessionEvents:495(168ns)}}
2025-12-24T02:11:13.690285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:13.690385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241289746359971:2048] [node 1] ICS07 socket disconnect 484 reason# EndOfStream
2025-12-24T02:11:13.690391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241289746359971:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.690428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.690444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241289746359971:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.690740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:13.690828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241291264671145:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-12-24T02:11:13.690836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241291264671051:2048] [node 1] ICP30 unregister session Session# [2:7587241291264671145:2048] VirtualId# [2:7587241291264671138:0]
2025-12-24T02:11:13.690844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241291264671145:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.690906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.690940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.694820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241289746359880:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:13.694835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241289746359971:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.695173Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241289746359880:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.695184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241289746359971:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.695190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241289746359880:2048] [node 1] ICP30 unregister session Session# [3:7587241289746359971:2048] VirtualId# [3:7587241289746359963:0]
2025-12-24T02:11:13.695206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.695213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241289746359880:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.695217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.702799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:13.702910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241292790072902:2048] [node 1] ICS07 socket disconnect 488 reason# EndOfStream
2025-12-24T02:11:13.702920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241292790072902:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.702978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.703001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241292790072902:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.702894Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:13.702977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241291957322647:2048] [node 1] ICS07 socket disconnect 515 reason# EndOfStream
2025-12-24T02:11:13.702982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241291957322647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.703041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.703063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241291957322647:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.703436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241291957322554:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.703449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241291957322647:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.703455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241291957322554:2048] [node 1] ICP30 unregister session Session# [6:7587241291957322647:2048] VirtualId# [6:7587241291957322636:0]
2025-12-24T02:11:13.703479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.703486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241291957322554:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.703488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.703485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292790072816:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.703496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241292790072902:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.703500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241292790072816:2048] [node 1] ICP30 unregister session Session# [4:7587241292790072902:2048] VirtualId# [4:7587241292790072896:0]
2025-12-24T02:11:13.703514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.703519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241292790072816:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.703522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.704204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:13.704283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241291392636698:2048] [node 1] ICS07 socket disconnect 478 reason# EndOfStream
2025-12-24T02:11:13.704288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241291392636698:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.704329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:13.704350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241291392636698:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:13.704725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241291392636622:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.704734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241291392636698:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.704739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241291392636622:2048] [node 1] ICP30 unregister session Session# [5:7587241291392636698:2048] VirtualId# [5:7587241291392636690:0]
2025-12-24T02:11:13.704752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:13.704757Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241291392636622:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.704759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27409) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.706765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241289746359880:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:13.714750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241292790072816:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:13.714759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:13.714860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241289746359968:2049] [node 2] ICS07 socket disconnect 483 reason# EndOfStream
2025-12-24T02:11:13.714867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241289746359968:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.714915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.714937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241289746359968:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:13.715291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241289746359881:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.715304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241289746359968:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.715309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241289746359881:2049] [node 2] ICP30 unregister session Session# [3:7587241289746359968:2049] VirtualId# [3:7587241289746359965:0]
2025-12-24T02:11:13.715323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.715329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241289746359881:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.715332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.718084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:13.718160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241292790072914:2049] [node 2] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:13.718167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241292790072817:2049] [node 2] ICP30 unregister session Session# [4:7587241292790072914:2049] VirtualId# [4:7587241292790072906:0]
2025-12-24T02:11:13.718173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241292790072914:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.718223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.718246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.714749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241291392636622:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:13.718098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:13.718167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241291392636710:2049] [node 2] ICS07 socket disconnect 525 reason# EndOfStream
2025-12-24T02:11:13.718173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241291392636710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.718217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.718238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241291392636710:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:13.718678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241291392636623:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.718685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241291392636710:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.718690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241291392636623:2049] [node 2] ICP30 unregister session Session# [5:7587241291392636710:2049] VirtualId# [5:7587241291392636706:0]
2025-12-24T02:11:13.718718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.718723Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241291392636623:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.718726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.718760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241291957322554:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:13.718793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:13.718866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241291957322637:2049] [node 2] ICS07 socket disconnect 494 reason# EndOfStream
2025-12-24T02:11:13.718871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241291957322637:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.718917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:13.718936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241291957322637:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:13.719304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241291957322555:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.719315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241291957322637:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.719320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241291957322555:2049] [node 2] ICP30 unregister session Session# [6:7587241291957322637:2049] VirtualId# [6:7587241291957322632:0]
2025-12-24T02:11:13.719336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:13.719349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241291957322555:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.719351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13399) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.730767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241291957322555:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:13.734740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241291392636623:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:13.752486Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:13.752622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241291957322642:2050] [node 3] ICS07 socket disconnect 533 reason# EndOfStream
2025-12-24T02:11:13.752631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241291957322642:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.752689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:13.752712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241291957322642:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:13.753310Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241291957322556:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.753325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241291957322642:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.753331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241291957322556:2050] [node 3] ICP30 unregister session Session# [6:7587241291957322642:2050] VirtualId# [6:7587241291957322639:0]
2025-12-24T02:11:13.753347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:13.753354Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241291957322556:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.753357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.753960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:13.754056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241291392636713:2050] [node 3] ICS07 socket disconnect 495 reason# EndOfStream
2025-12-24T02:11:13.754061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241291392636713:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.754115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:13.754139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241291392636713:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:13.754533Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241291392636624:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:13.754543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241291392636713:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:13.754548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241291392636624:2050] [node 3] ICP30 unregister session Session# [5:7587241291392636713:2050] VirtualId# [5:7587241291392636703:0]
2025-12-24T02:11:13.754563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:13.754567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241291392636624:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.754570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31533) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:13.762041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241291392636624:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:13.766757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241291957322556:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:13.778798Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:13.778913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241291957322649:2085] [node 4] ICS01 socket: 496 reason# EndOfStream
2025-12-24T02:11:13.778924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241291957322597:2085] [node 4] ICP30 unregister session Session# [6:7587241291957322649:2085] VirtualId# [6:7587241291957322644:0]
2025-12-24T02:11:13.778934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241291957322649:2085] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:13.778991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:13.779017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:13.830730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290840733153:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:13.834849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290840733154:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:13.838901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241290327942975:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.838935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241290327942976:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.846768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291283135717:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.846790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291283135718:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:13.855270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292262131564:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.855392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292262131565:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.855398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241292262131566:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.870764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241290845551862:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.870784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241290845551863:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.882717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241290845551861:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:13.914850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241293766251121:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:13.918873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241293766251120:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:13.934731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241293766251119:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.967401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241290327943035:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:13.966951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241291283135779:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61467
2025-12-24T02:11:13.969698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241290845551931:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:13.970962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241293766251180:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:13.971177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290840733219:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.978925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241290840733224:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44364
2025-12-24T02:11:13.984758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241290845551930:2101] [node 4] ICP01 ready to work
2025-12-24T02:11:13.984132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291283135782:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:13.984469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291283135783:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:13.984712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290840733217:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:13.985152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241290840733223:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:13.990807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241290840733153:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29497839ns) {Configure:104(29496186ns) {ProcessConfigured:135(29036650ns) {StartInitialHandshake:157(29035588ns) {DropHandshakes:470(1651ns) {DropIncomingHandshake:442(183ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:477(1598ns) {SwitchToState:179(830ns)}}}}}
2025-12-24T02:11:13.991168Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241293766251121:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20276272ns) {Configure:104(20275246ns) {ProcessConfigured:135(18209919ns) {StartInitialHandshake:157(18209117ns) {DropHandshakes:470(307ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(1366ns) {SwitchToState:179(848ns)}}}}}
2025-12-24T02:11:13.991399Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241293766251180:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20367288ns) {Configure:104(20366820ns) {ProcessConfigured:135(18334870ns) {StartInitialHandshake:157(18334487ns) {DropHandshakes:470(146ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:477(601ns) {SwitchToState:179(391ns)}}}}}
2025-12-24T02:11:13.991431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241290327943035:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.991655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241290327943039:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:13.992012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241290327943039:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.992345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241290845551927:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:13.992360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241290845551927:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:13.992950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241290845551936:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29685
2025-12-24T02:11:13.993241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241290845551927:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:13.993353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241290845551863:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:13.993373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:13.993381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241290845551863:2050] [node 3] ICP22 created new session: [5:7587241290845551938:2050]
2025-12-24T02:11:13.993388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241290845551938:2050] [node 3] ICS09 handshake done sender: [5:7587241290845551927:2111] self: [5:7587241290845551926:0] peer: [3:7587241290327943040:0] socket: 481 qp: -1
2025-12-24T02:11:13.993398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241290845551938:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:13.993407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:13.994808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291283135779:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.998919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241293766251189:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:13.998956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241293766251188:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:13.999386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241290327943044:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:13.999483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241290327943041:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:13.999495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241290327943042:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:13.999547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241290327943043:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:13.999930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241291283135784:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:14.002878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241290327943035:2107] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.004096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241295140519236:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29685
2025-12-24T02:11:14.003748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241290327943042:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.003879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241290327943039:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.003894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.003902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241290327943039:2108] [node 5] ICP22 created new session: [3:7587241294622910341:2108]
2025-12-24T02:11:14.003909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241294622910341:2108] [node 5] ICS09 handshake done sender: [3:7587241290327943035:2107] self: [3:7587241290327943040:0] peer: [5:7587241290845551926:0] socket: 484 qp: -1
2025-12-24T02:11:14.003917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241294622910341:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:14.003942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.004408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241295135700525:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44364
2025-12-24T02:11:14.004560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241290845551930:2101] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19738553ns) {Configure:104(19738170ns) {ProcessConfigured:135(19419954ns) {StartInitialHandshake:157(19419660ns) {DropHandshakes:470(265ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(780ns) {SwitchToState:179(434ns)}}}}}
2025-12-24T02:11:14.005373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241293766251188:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.005798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241293766251119:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.003021Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241291283135782:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18801633ns) {Configure:104(18800503ns) {ProcessConfigured:135(18530106ns) {StartInitialHandshake:157(18529496ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(2407ns) {SwitchToState:179(1568ns)}}}}}
2025-12-24T02:11:14.004033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241291283135783:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19502630ns) {Configure:104(19502269ns) {ProcessConfigured:135(19287465ns) {StartInitialHandshake:157(19286643ns) {DropHandshakes:470(94ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1438ns) {SwitchToState:179(578ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.009017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241295140519239:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29685
2025-12-24T02:11:14.008381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241293766251184:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.008392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241293766251184:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.008838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218487:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.009271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241295140519236:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.009295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241290845551862:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.009577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241293766251189:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.009609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241293766251180:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.009687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241293766251184:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.009383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241295578103089:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.009393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241295578103089:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.010260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241295578103089:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.009986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241293766251121:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.010003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.010009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241293766251121:2050] [node 3] ICP22 created new session: [6:7587241298061218490:2050]
2025-12-24T02:11:14.010014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298061218490:2050] [node 3] ICS09 handshake done sender: [6:7587241293766251184:2109] self: [6:7587241293766251181:0] peer: [3:7587241294622910348:0] socket: 499 qp: -1
2025-12-24T02:11:14.010019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298061218490:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:14.010036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.010089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241293766251188:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.010147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241293766251119:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.010154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.010158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241293766251119:2048] [node 1] ICP22 created new session: [6:7587241298061218492:2048]
2025-12-24T02:11:14.010161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298061218492:2048] [node 1] ICS09 handshake done sender: [6:7587241293766251188:2111] self: [6:7587241298061218486:0] peer: [1:7587241290840733220:0] socket: 488 qp: -1
2025-12-24T02:11:14.010166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298061218492:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.010172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.010425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241295140519241:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.004831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241290840733217:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19633379ns) {Configure:104(19632776ns) {ProcessConfigured:135(19295702ns) {StartInitialHandshake:157(19295403ns) {DropHandshakes:470(143ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1032ns) {SwitchToState:179(429ns)}}}}}
2025-12-24T02:11:14.006019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290840733221:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.006025Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290840733221:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.006771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241290840733223:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20021855ns) {Configure:104(20020390ns) {ProcessConfigured:135(19651447ns) {StartInitialHandshake:157(19651144ns) {DropHandshakes:470(747ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(592ns) PrepareNewSessionHandshake:477(1071ns) {SwitchToState:179(471ns)}}}}}
2025-12-24T02:11:14.008109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290840733221:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.008248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241295135700525:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.008257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290840733219:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.008271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.008278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290840733219:2108] [node 6] ICP22 created new session: [1:7587241295135700530:2108]
2025-12-24T02:11:14.008284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241295135700530:2108] [node 6] ICS09 handshake done sender: [1:7587241290840733221:2110] self: [1:7587241290840733220:0] peer: [6:7587241298061218486:0] socket: 487 qp: -1
2025-12-24T02:11:14.008292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241295135700530:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:14.008312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.010788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241295140519236:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.005068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241290327942976:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36786599ns) {Configure:104(36785795ns) {ProcessConfigured:135(36377865ns) {StartInitialHandshake:157(36377458ns) {DropHandshakes:470(1318ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(936ns) {SwitchToState:179(517ns)}}}}}
2025-12-24T02:11:14.005115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241290327942976:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.005566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241290327943043:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.005797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241294622910347:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:14.008153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241294622910347:2114] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.010634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241294622910349:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:14.010873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241290845551862:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.010889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.010894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241290845551862:2049] [node 2] ICP22 created new session: [5:7587241295140519242:2049]
2025-12-24T02:11:14.010899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241295140519242:2049] [node 2] ICS09 handshake done sender: [5:7587241295140519236:2114] self: [5:7587241295140519240:0] peer: [2:7587241291283135790:0] socket: 510 qp: -1
2025-12-24T02:11:14.010904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241295140519242:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.010910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.011655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291283135782:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.011673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.011678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291283135782:2107] [node 5] ICP22 created new session: [2:7587241295578103094:2107]
2025-12-24T02:11:14.011682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241295578103094:2107] [node 5] ICS09 handshake done sender: [2:7587241295578103089:2113] self: [2:7587241291283135790:0] peer: [5:7587241295140519240:0] socket: 509 qp: -1
2025-12-24T02:11:14.011687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241295578103094:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:14.011694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.013659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241295578103097:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61467
2025-12-24T02:11:14.013722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241295578103098:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61467
2025-12-24T02:11:14.013909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241290327943043:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.014030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241294622910347:2114] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.014043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.014049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241294622910347:2114] [node 6] ICP22 created new session: [3:7587241294622910350:2114]
2025-12-24T02:11:14.014054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241294622910350:2114] [node 6] ICS09 handshake done sender: [3:7587241290327943043:2111] self: [3:7587241294622910348:0] peer: [6:7587241293766251181:0] socket: 507 qp: -1
2025-12-24T02:11:14.014060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241294622910350:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:14.014076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.014351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241295578103097:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.014385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241291283135717:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.014782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290840733227:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.014796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241290840733227:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.016268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241290840733154:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(46493258ns) {Configure:104(46491997ns) {ProcessConfigured:135(45920132ns) {StartInitialHandshake:157(45918625ns) {DropHandshakes:470(972ns) {DropIncomingHandshake:442(101ns) DropOutgoingHandshake:457(365ns)} GenerateSessionVirtualId:988(335ns) PrepareNewSessionHandshake:477(985ns) {SwitchToState:179(156ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.016964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241290845551936:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.015503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218494:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.016466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098927:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.019361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241290840733227:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.018813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241290845551933:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.018832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241290845551933:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.019779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241290327943041:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.019978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241294622910353:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:14.020062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241290840733224:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.020102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290840733153:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.020117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.020123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290840733153:2048] [node 2] ICP22 created new session: [1:7587241295135700537:2048]
2025-12-24T02:11:14.020129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241295135700537:2048] [node 2] ICS09 handshake done sender: [1:7587241290840733227:2112] self: [1:7587241290840733213:0] peer: [2:7587241295578103099:0] socket: 498 qp: -1
2025-12-24T02:11:14.020134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241295135700537:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:14.020142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.018874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241291283135787:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.018887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241291283135787:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.020372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241294622910353:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.019519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298061218494:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.019949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241293766251120:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.022829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241290327942975:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35955716ns) {Configure:104(35954602ns) {ProcessConfigured:135(35615055ns) {StartInitialHandshake:157(35614512ns) {DropHandshakes:470(1113ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(176ns)} GenerateSessionVirtualId:988(382ns) PrepareNewSessionHandshake:477(844ns) {SwitchToState:179(104ns)}}}}}
2025-12-24T02:11:14.026852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241292262131627:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.026867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241292262131627:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: 2025-12-24T02:11:14.027356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241295578103092:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:14.027368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241295578103092:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.032915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241295140519239:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.033382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241290845551861:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.034786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241295135700529:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.034802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241295135700529:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.035442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241295135700529:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.035554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290840733223:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.035572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.035582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290840733223:2109] [node 5] ICP22 created new session: [1:7587241295135700542:2109]
2025-12-24T02:11:14.035588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241295135700542:2109] [node 5] ICS09 handshake done sender: [1:7587241295135700529:2115] self: [1:7587241290840733226:0] peer: [5:7587241295140519245:0] socket: 519 qp: -1
2025-12-24T02:11:14.035595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241295135700542:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:14.035607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.040045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218496:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.040110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241290840733153:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19968700ns) {HandleHandshakeStatus:329(19967972ns) {DropIncomingHandshake:442(3539ns) DropOutgoingHandshake:457(429ns) SwitchToState:179(46ns) ProcessPendingSessionEvents:495(19894523ns) {ForwardSessionEventToSession:612(19873868ns) ForwardSessionEventToSession:612(2711ns) ForwardSessionEventToSession:612(783ns) ForwardSessionEventToSession:612(1609ns) ForwardSessionEventToSession:612(711ns) ForwardSessionEventToSession:612(831ns) ForwardSessionEventToSession:612(852ns) ForwardSessionEventToSession:612(618ns)}}}
2025-12-24T02:11:14.039649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241291283135717:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241295578103097:2115] held: no
2025-12-24T02:11:14.040219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241291283135787:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.041507Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241291283135784:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41502969ns) {Configure:104(41502096ns) {ProcessConfigured:135(41102954ns) {StartInitialHandshake:157(41102330ns) {DropHandshakes:470(1076ns) {DropIncomingHandshake:442(229ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(1179ns) {SwitchToState:179(826ns)}}}}}
2025-12-24T02:11:14.041757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291283135718:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.041767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.041773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291283135718:2049] [node 3] ICP22 created new session: [2:7587241295578103103:2049]
2025-12-24T02:11:14.041779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241295578103103:2049] [node 3] ICS09 handshake done sender: [2:7587241291283135787:2112] self: [2:7587241291283135786:0] peer: [3:7587241294622910344:0] socket: 494 qp: -1
2025-12-24T02:11:14.040528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241296557098927:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.041787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241295578103103:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:14.041821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.040848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241296557098931:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:14.041271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241296557098931:2112] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.041973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241292262131627:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.042728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241295135700540:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44364
2025-12-24T02:11:14.042822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241290845551933:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.042774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241295578103101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61467
2025-12-24T02:11:14.043009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241293766251189:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.043074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241293766251180:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.043087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.043093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241293766251180:2107] [node 5] ICP22 created new session: [6:7587241298061218497:2107]
2025-12-24T02:11:14.043099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298061218497:2107] [node 5] ICS09 handshake done sender: [6:7587241293766251189:2112] self: [6:7587241298061218489:0] peer: [5:7587241290845551932:0] socket: 505 qp: -1
2025-12-24T02:11:14.043105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298061218497:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:14.043116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.048649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241295578103097:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.048932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241294622910357:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:14.049007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241294622910358:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:14.049663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241295135700535:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.049674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241295135700535:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.050392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241295135700535:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.050490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290840733154:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.049048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241294622910359:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:14.050501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.050513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290840733154:2049] [node 3] ICP22 created new session: [1:7587241295135700545:2049]
2025-12-24T02:11:14.049152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241290327943041:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.050519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241295135700545:2049] [node 3] ICS09 handshake done sender: [1:7587241295135700535:2116] self: [1:7587241290840733218:0] peer: [3:7587241294622910362:0] socket: 496 qp: -1
2025-12-24T02:11:14.050526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241295135700545:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:14.050535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.051830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241295140519237:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.051851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241295140519237:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.049334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241294622910357:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.049343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241294622910353:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.049354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.049364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241294622910353:2116] [node 4] ICP22 created new session: [3:7587241294622910360:2116]
2025-12-24T02:11:14.049370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241294622910360:2116] [node 4] ICS09 handshake done sender: [3:7587241290327943041:2109] self: [3:7587241294622910354:0] peer: [4:7587241292262131626:0] socket: 489 qp: -1
2025-12-24T02:11:14.049376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241294622910360:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:14.049387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.049554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241290327942975:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.051786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241290327943042:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.054932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241291283135717:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.054956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.054964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291283135717:2048] [node 1] ICP22 created new session: [2:7587241295578103107:2048]
2025-12-24T02:11:14.054971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241295578103107:2048] [node 1] ICS09 handshake done sender: [2:7587241295578103097:2115] self: [2:7587241295578103099:0] peer: [1:7587241290840733213:0] socket: 504 qp: -1
2025-12-24T02:11:14.054977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241295578103107:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.054984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.055437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241290327942976:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.055461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.055468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241290327942976:2049] [node 2] ICP22 created new session: [3:7587241294622910363:2049]
2025-12-24T02:11:14.055478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241294622910363:2049] [node 2] ICS09 handshake done sender: [3:7587241290327943042:2110] self: [3:7587241294622910344:0] peer: [2:7587241291283135786:0] socket: 506 qp: -1
2025-12-24T02:11:14.055485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241294622910363:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.055492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.050768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098928:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.052053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241296557098928:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.057301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218499:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.058939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241292262131566:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.058962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.058970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292262131566:2050] [node 3] ICP22 created new session: [4:7587241296557098934:2050]
2025-12-24T02:11:14.058979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241296557098934:2050] [node 3] ICS09 handshake done sender: [4:7587241292262131627:2108] self: [4:7587241292262131626:0] peer: [3:7587241294622910354:0] socket: 485 qp: -1
2025-12-24T02:11:14.058986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241296557098934:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:14.059016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.059034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241290845551931:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.059064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.059070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241290845551931:2103] [node 6] ICP22 created new session: [5:7587241295140519247:2103]
2025-12-24T02:11:14.059077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241295140519247:2103] [node 6] ICS09 handshake done sender: [5:7587241290845551933:2112] self: [5:7587241290845551932:0] peer: [6:7587241298061218489:0] socket: 495 qp: -1
2025-12-24T02:11:14.059081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241295140519247:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:14.059102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.059185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298061218494:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.059051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241295578103092:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.064309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291283135783:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.064329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.064358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291283135783:2108] [node 6] ICP22 created new session: [2:7587241295578103109:2108]
2025-12-24T02:11:14.064365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241295578103109:2108] [node 6] ICS09 handshake done sender: [2:7587241295578103092:2114] self: [2:7587241291283135791:0] peer: [6:7587241298061218495:0] socket: 525 qp: -1
2025-12-24T02:11:14.064372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241295578103109:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:14.064184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241294622910365:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43072
2025-12-24T02:11:14.064379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.064806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241294622910357:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.064911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241290327942975:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.064927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.064933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241290327942975:2048] [node 1] ICP22 created new session: [3:7587241294622910366:2048]
2025-12-24T02:11:14.064939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241294622910366:2048] [node 1] ICS09 handshake done sender: [3:7587241294622910357:2118] self: [3:7587241294622910362:0] peer: [1:7587241290840733218:0] socket: 530 qp: -1
2025-12-24T02:11:14.064947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241294622910366:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.065593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241295135700540:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.064953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.066794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241291283135718:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(25002071ns) {HandleHandshakeStatus:329(25000953ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(177ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(15858594ns) {ForwardSessionEventToSession:612(15844102ns) ForwardSessionEventToSession:612(1620ns) ForwardSessionEventToSession:612(1214ns) ForwardSessionEventToSession:612(699ns) ForwardSessionEventToSession:612(775ns) ForwardSessionEventToSession:612(1400ns) ForwardSessionEventToSession:612(1077ns) ForwardSessionEventToSession:612(584ns)}}}
2025-12-24T02:11:14.065742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241290840733154:2049] [node 3] ICP18 handshake (actor [1:7587241295135700540:2117]) is too old
2025-12-24T02:11:14.067038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241296557098937:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:14.068335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241292262131564:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.068358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241292262131564:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16245417ns) {IncomingHandshake:261(16244865ns) {DropIncomingHandshake:442(226ns) PrepareNewSessionHandshake:477(989ns) {SwitchToState:179(213ns)} GenerateSessionVirtualId:988(433ns) IssueIncomingHandshakeReply:193(16236027ns) {SwitchToState:179(346ns)}}}
2025-12-24T02:11:14.068061Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241293766251180:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(24964305ns) {HandleHandshakeStatus:329(24963904ns) {DropIncomingHandshake:442(166ns) DropOutgoingHandshake:457(5349ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(24889207ns) {ForwardSessionEventToSession:612(24886142ns) ForwardSessionEventToSession:612(2400ns)}}}
2025-12-24T02:11:14.068079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241293766251120:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.068089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.068114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241293766251120:2049] [node 2] ICP22 created new session: [6:7587241298061218500:2049]
2025-12-24T02:11:14.068120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298061218500:2049] [node 2] ICS09 handshake done sender: [6:7587241298061218494:2114] self: [6:7587241298061218495:0] peer: [2:7587241291283135791:0] socket: 480 qp: -1
2025-12-24T02:11:14.068127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298061218500:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.068134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.071258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098940:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.071280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098941:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.074817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241295140519237:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.074809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241295135700526:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.074825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241295135700526:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.076231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241290845551930:2101] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.076252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.076264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241290845551930:2101] [node 4] ICP22 created new session: [5:7587241295140519250:2101]
2025-12-24T02:11:14.076271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241295140519250:2101] [node 4] ICS09 handshake done sender: [5:7587241295140519237:2115] self: [5:7587241290845551935:0] peer: [4:7587241296557098932:0] socket: 511 qp: -1
2025-12-24T02:11:14.076280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241295140519250:2101] [node 4] ICS10 traffic start
2025-12-24T02:11:14.076288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.082935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241295140519252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29685
2025-12-24T02:11:14.086834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241296557098927:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.091051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241292262131566:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(48266885ns) {HandleHandshakeStatus:329(48266369ns) {DropIncomingHandshake:442(692ns) DropOutgoingHandshake:457(297ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(20020610ns) {ForwardSessionEventToSession:612(20018078ns)}}}
2025-12-24T02:11:14.091186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241296557098931:2112] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.091202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.091209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241296557098931:2112] [node 5] ICP22 created new session: [4:7587241296557098942:2112]
2025-12-24T02:11:14.091214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241296557098942:2112] [node 5] ICS09 handshake done sender: [4:7587241296557098927:2110] self: [4:7587241296557098932:0] peer: [5:7587241290845551935:0] socket: 516 qp: -1
2025-12-24T02:11:14.091221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241296557098942:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:14.091230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.091368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241296557098940:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.091491Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241290845551931:2103] [node 6] ICP35 event processing took too much time PendingConnection:279(48530432ns) {HandleHandshakeStatus:329(48530099ns) {DropIncomingHandshake:442(10120ns) DropOutgoingHandshake:457(226ns) SwitchToState:179(52ns) ProcessPendingSessionEvents:495(20433836ns) {ForwardSessionEventToSession:612(20428044ns) ForwardSessionEventToSession:612(1859ns) ForwardSessionEventToSession:612(1175ns) ForwardSessionEventToSession:612(877ns)}}}
2025-12-24T02:11:14.091515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241295140519239:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.091650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241290845551861:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.091661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.091666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241290845551861:2048] [node 1] ICP22 created new session: [5:7587241295140519253:2048]
2025-12-24T02:11:14.091670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241295140519253:2048] [node 1] ICS09 handshake done sender: [5:7587241295140519239:2116] self: [5:7587241295140519245:0] peer: [1:7587241290840733226:0] socket: 521 qp: -1
2025-12-24T02:11:14.091676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241295140519253:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.091681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.095068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241292262131565:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24256689ns) {Configure:104(24254486ns) {ProcessConfigured:135(23968244ns) {StartInitialHandshake:157(23967501ns) {DropHandshakes:470(385ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(284ns) PrepareNewSessionHandshake:477(837ns) {SwitchToState:179(98ns)}}}}}
2025-12-24T02:11:14.095098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241292262131565:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.095831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241295135700526:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.095947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241290840733217:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.095964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.095989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241290840733217:2107] [node 4] ICP22 created new session: [1:7587241295135700548:2107]
2025-12-24T02:11:14.095997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241295135700548:2107] [node 4] ICS09 handshake done sender: [1:7587241295135700526:2114] self: [1:7587241290840733225:0] peer: [4:7587241296557098933:0] socket: 512 qp: -1
2025-12-24T02:11:14.096004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241295135700548:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:14.096012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.099895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241296557098937:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32786598ns) {Configure:104(32785575ns) {ProcessConfigured:135(32357458ns) {StartInitialHandshake:157(32356727ns) {DropHandshakes:470(408ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(558ns) PrepareNewSessionHandshake:477(958ns) {SwitchToState:179(372ns)}}}}}
2025-12-24T02:11:14.096495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241290845551930:2101] [node 4] ICP35 event processing took too much time PendingConnection:279(20244659ns) {HandleHandshakeStatus:329(20243509ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(171ns) SwitchToState:179(82ns) ProcessPendingSessionEvents:495(12536777ns) {ForwardSessionEventToSession:612(12532897ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.102778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241295578103102:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.102799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241295578103102:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.103628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218502:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.103818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241295578103112:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61467
2025-12-24T02:11:14.104583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241295578103102:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.104689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241291283135784:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.104702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.104730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241291283135784:2110] [node 4] ICP22 created new session: [2:7587241295578103113:2110]
2025-12-24T02:11:14.104738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241295578103113:2110] [node 4] ICS09 handshake done sender: [2:7587241295578103102:2118] self: [2:7587241291283135792:0] peer: [4:7587241296557098946:0] socket: 542 qp: -1
2025-12-24T02:11:14.104744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241295578103113:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:14.104760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.110365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298061218504:2118] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.114742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298061218502:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.114782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298061218504:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.112572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098949:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.114417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.114541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241296557098951:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59705
2025-12-24T02:11:14.114633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241296557098940:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.114970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241296557098951:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.115028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241296557098947:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.115030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241296557098947:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.115322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241292262131565:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.115338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.115344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292262131565:2049] [node 2] ICP22 created new session: [4:7587241296557098952:2049]
2025-12-24T02:11:14.115362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241296557098952:2049] [node 2] ICS09 handshake done sender: [4:7587241296557098940:2114] self: [4:7587241296557098946:0] peer: [2:7587241291283135792:0] socket: 544 qp: -1
2025-12-24T02:11:14.115369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241296557098952:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.115379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.116030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241295578103112:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.116134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241291283135784:2110] [node 4] ICP18 handshake (actor [2:7587241295578103112:2119]) is too old
2025-12-24T02:11:14.116626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241296557098947:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.127161Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292262131564:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241296557098928:2111] held: no
2025-12-24T02:11:14.127242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241292262131564:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24286993ns) {HandleHandshakeStatus:395(24286033ns) {LogHandshakeFail:472(5615ns) {UpdateErrorStateLog:545(2482ns)} DropOutgoingHandshake:457(915ns) ProcessPendingSessionEvents:495(13852ns) {DropSessionEvent:512(8524ns) DropSessionEvent:512(2109ns) DropSessionEvent:512(1481ns)}}}
2025-12-24T02:11:14.127281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241296557098928:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.130982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298061218509:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62816
2025-12-24T02:11:14.131370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298061218502:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.131455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298061218504:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.131489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.131515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298061218504:2118] [node 4] ICP22 created new session: [6:7587241298061218510:2118]
2025-12-24T02:11:14.131524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298061218510:2118] [node 4] ICS09 handshake done sender: [6:7587241298061218502:2117] self: [6:7587241298061218508:0] peer: [4:7587241296557098938:0] socket: 545 qp: -1
2025-12-24T02:11:14.131535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298061218510:2118] [node 4] ICS10 traffic start
2025-12-24T02:11:14.131545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.131731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241292262131564:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.131752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.131760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241292262131564:2048] [node 1] ICP22 created new session: [4:7587241296557098956:2048]
2025-12-24T02:11:14.131766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241296557098956:2048] [node 1] ICS09 handshake done sender: [4:7587241296557098928:2111] self: [4:7587241296557098933:0] peer: [1:7587241290840733225:0] socket: 517 qp: -1
2025-12-24T02:11:14.131772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241296557098956:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.131801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.132356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241296557098937:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.132378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.132384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241296557098937:2113] [node 6] ICP22 created new session: [4:7587241296557098958:2113]
2025-12-24T02:11:14.132390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241296557098958:2113] [node 6] ICS09 handshake done sender: [4:7587241296557098947:2117] self: [4:7587241296557098938:0] peer: [6:7587241298061218508:0] socket: 529 qp: -1
2025-12-24T02:11:14.132395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241296557098958:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:14.132403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.140139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241296557098937:2113] [node 6] ICP35 event processing took too much time PendingConnection:279(23192634ns) {HandleHandshakeStatus:329(23192064ns) {DropIncomingHandshake:442(7407ns) DropOutgoingHandshake:457(118ns) SwitchToState:179(51ns) ProcessPendingSessionEvents:495(7667328ns) {ForwardSessionEventToSession:612(7665977ns)}}}
2025-12-24T02:11:14.318824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:14.318928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241295578103107:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-12-24T02:11:14.318937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241291283135717:2048] [node 1] ICP30 unregister session Session# [2:7587241295578103107:2048] VirtualId# [2:7587241295578103099:0]
2025-12-24T02:11:14.318946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241295578103107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.318997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.319030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.322794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:14.322920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241294622910366:2048] [node 1] ICS07 socket disconnect 530 reason# EndOfStream
2025-12-24T02:11:14.322930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241294622910366:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.322997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.323021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241294622910366:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:14.323525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241290327942975:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.323539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241294622910366:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.323545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241290327942975:2048] [node 1] ICP30 unregister session Session# [3:7587241294622910366:2048] VirtualId# [3:7587241294622910362:0]
2025-12-24T02:11:14.323561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.323569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241290327942975:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.323572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.330783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:14.330902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241298061218492:2048] [node 1] ICS07 socket disconnect 488 reason# EndOfStream
2025-12-24T02:11:14.330911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298061218492:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.330986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.331012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241298061218492:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:14.331482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241293766251119:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.331498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298061218492:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.331504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241293766251119:2048] [node 1] ICP30 unregister session Session# [6:7587241298061218492:2048] VirtualId# [6:7587241298061218486:0]
2025-12-24T02:11:14.331524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.331532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241293766251119:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.331535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.330848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:14.330944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241296557098956:2048] [node 1] ICS07 socket disconnect 517 reason# EndOfStream
2025-12-24T02:11:14.330951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241296557098956:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.331009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.331034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241296557098956:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:14.334790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241290327942975:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:14.334855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:14.334956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241295140519253:2048] [node 1] ICS07 socket disconnect 521 reason# EndOfStream
2025-12-24T02:11:14.334963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241295140519253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.335035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.335062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241295140519253:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:14.335537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292262131564:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.335564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241296557098956:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.335571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241292262131564:2048] [node 1] ICP30 unregister session Session# [4:7587241296557098956:2048] VirtualId# [4:7587241296557098933:0]
2025-12-24T02:11:14.335597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.335605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241292262131564:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.335608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.335775Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241290845551861:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.335789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241295140519253:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.335795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241290845551861:2048] [node 1] ICP30 unregister session Session# [5:7587241295140519253:2048] VirtualId# [5:7587241295140519245:0]
2025-12-24T02:11:14.335810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.335816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241290845551861:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.335818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19629) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.346931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241293766251119:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:14.347476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241292262131564:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:14.347411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241290845551861:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:14.350799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:14.350924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241294622910363:2049] [node 2] ICS07 socket disconnect 506 reason# EndOfStream
2025-12-24T02:11:14.350933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241294622910363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.350998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.351023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241294622910363:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:14.351528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241290327942976:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.351541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241294622910363:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.351547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241290327942976:2049] [node 2] ICP30 unregister session Session# [3:7587241294622910363:2049] VirtualId# [3:7587241294622910344:0]
2025-12-24T02:11:14.351565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.351574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241290327942976:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.351577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.358733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:14.358833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241296557098952:2049] [node 2] ICS07 socket disconnect 544 reason# EndOfStream
2025-12-24T02:11:14.358840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241296557098952:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.358908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.358932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241296557098952:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:14.359387Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241292262131565:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.359399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241296557098952:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.359404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241292262131565:2049] [node 2] ICP30 unregister session Session# [4:7587241296557098952:2049] VirtualId# [4:7587241296557098946:0]
2025-12-24T02:11:14.359419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.359426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241292262131565:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.359429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.360907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:14.360999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241298061218500:2049] [node 2] ICS07 socket disconnect 480 reason# EndOfStream
2025-12-24T02:11:14.361006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298061218500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.361069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.361094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241298061218500:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:14.361297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:14.361368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241295140519242:2049] [node 2] ICS07 socket disconnect 510 reason# EndOfStream
2025-12-24T02:11:14.361373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241295140519242:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.361422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.361454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241295140519242:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:14.361896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241290845551862:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.361912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241295140519242:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.361916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241290845551862:2049] [node 2] ICP30 unregister session Session# [5:7587241295140519242:2049] VirtualId# [5:7587241295140519240:0]
2025-12-24T02:11:14.361932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.361939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241290845551862:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.361942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.366089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241293766251120:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.366110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298061218500:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.366116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241293766251120:2049] [node 2] ICP30 unregister session Session# [6:7587241298061218500:2049] VirtualId# [6:7587241298061218495:0]
2025-12-24T02:11:14.366136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.366144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241293766251120:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.366149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7152) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.370854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241292262131565:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:14.370782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241290845551862:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:14.374773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241293766251120:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:14.374891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:14.374989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241290845551938:2050] [node 3] ICS07 socket disconnect 481 reason# EndOfStream
2025-12-24T02:11:14.374996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241290845551938:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.375038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:14.375066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241290845551938:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:14.375556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241290845551863:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.375581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241290845551938:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.375588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241290845551863:2050] [node 3] ICP30 unregister session Session# [5:7587241290845551938:2050] VirtualId# [5:7587241290845551926:0]
2025-12-24T02:11:14.375608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:14.375617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241290845551863:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.375620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.378749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:14.378864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241298061218490:2050] [node 3] ICS07 socket disconnect 499 reason# EndOfStream
2025-12-24T02:11:14.378872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298061218490:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.378944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:14.378978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241298061218490:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:14.381326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241293766251121:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.381349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298061218490:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.381356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241293766251121:2050] [node 3] ICP30 unregister session Session# [6:7587241298061218490:2050] VirtualId# [6:7587241293766251181:0]
2025-12-24T02:11:14.381371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:14.381377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241293766251121:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.381380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16552) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.390769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241290845551863:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:14.390844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241293766251121:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:14.390976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:14.391069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241295140519250:2101] [node 4] ICS01 socket: 511 reason# EndOfStream
2025-12-24T02:11:14.391077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241290845551930:2101] [node 4] ICP30 unregister session Session# [5:7587241295140519250:2101] VirtualId# [5:7587241290845551935:0]
2025-12-24T02:11:14.391091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241295140519250:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.391162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:14.391190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:14.394998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:14.395107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241298061218510:2118] [node 4] ICS07 socket disconnect 545 reason# EndOfStream
2025-12-24T02:11:14.395113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298061218510:2118] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.395185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:14.395212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241298061218510:2118] [node 4] ICS15 start handshake
2025-12-24T02:11:14.398955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241298061218504:2118] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14825) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:14.398976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298061218510:2118] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:14.398984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241298061218504:2118] [node 4] ICP30 unregister session Session# [6:7587241298061218510:2118] VirtualId# [6:7587241298061218508:0]
2025-12-24T02:11:14.399002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:14.399009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241298061218504:2118] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14825) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:14.399012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14825) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:14.435151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241296893961939:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.435174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241296893961940:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:14.440310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241297177781146:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.440327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241297177781147:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.440332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241297177781148:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:14.427219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241294655028466:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:14.427226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241294655028467:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:14.442801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241296986444979:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.442817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241296986444980:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.442704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297722442474:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.442713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297722442475:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.442716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297722442476:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:14.451933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295168481501:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.451953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295168481502:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.451962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295168481503:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.541713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893961999:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.542855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445039:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
2025-12-24T02:11:14.543381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295168481563:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.544395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442535:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.546862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445040:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
2025-12-24T02:11:14.551792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241294655028534:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:14.551887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241294655028535:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:14.576837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297722442535:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.577051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297722442537:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:14.577416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297722442537:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.580508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241296986445042:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.582089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781207:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.578566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241296893962006:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:14.578664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241296893962007:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:14.578758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241296893962008:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.583187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442539:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.583489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297722442535:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.583551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297722442537:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.583567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.583572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297722442537:2109] [node 6] ICP22 created new session: [5:7587241297722442540:2109]
2025-12-24T02:11:14.583580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297722442540:2109] [node 6] ICS09 handshake done sender: [5:7587241297722442535:2108] self: [5:7587241297722442538:0] peer: [6:7587241295168481564:0] socket: 493 qp: -1
2025-12-24T02:11:14.583587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297722442540:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:14.583605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.582485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295168481565:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.582501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241295168481565:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.583300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295168481565:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.583825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241295168481563:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.583833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.583837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295168481563:2106] [node 5] ICP22 created new session: [6:7587241295168481571:2106]
2025-12-24T02:11:14.583842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295168481571:2106] [node 5] ICS09 handshake done sender: [6:7587241295168481565:2109] self: [6:7587241295168481564:0] peer: [5:7587241297722442538:0] socket: 491 qp: -1
2025-12-24T02:11:14.583846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295168481571:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:14.583853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.597743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241294655028541:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.602765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445040:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.602940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241296986445054:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.604534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241296986445055:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.605759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442544:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.603825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241294655028542:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17672
2025-12-24T02:11:14.603874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241294655028543:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17672
2025-12-24T02:11:14.607997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297722442553:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:14.623620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241297177781217:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:14.623737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241297177781218:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.635067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241294655028547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17672
2025-12-24T02:11:14.638842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241295168481501:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16040255ns) {Configure:104(16039489ns) {ProcessConfigured:135(15687001ns) {StartInitialHandshake:157(15686017ns) {DropHandshakes:470(1032ns) {DropIncomingHandshake:442(132ns) DropOutgoingHandshake:457(157ns)} GenerateSessionVirtualId:988(386ns) PrepareNewSessionHandshake:477(1167ns) {SwitchToState:179(493ns)}}}}}
2025-12-24T02:11:14.651255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295168481578:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.659408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893961999:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.659944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893962011:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
2025-12-24T02:11:14.659957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893962012:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
2025-12-24T02:11:14.659998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893962013:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
2025-12-24T02:11:14.660009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893962014:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
2025-12-24T02:11:14.660046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241296893961939:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.662954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028542:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.662998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028527:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.663001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241294655028527:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.663475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241294655028548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17672
2025-12-24T02:11:14.665932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241294655028527:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.665858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241297177781218:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31167648ns) {Configure:104(31165580ns) {ProcessConfigured:135(26241261ns) {StartInitialHandshake:157(26239273ns) {DropHandshakes:470(701ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(1168ns) {SwitchToState:179(54ns)}}}}}
2025-12-24T02:11:14.665274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445039:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.665600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241296986444979:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.665618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445058:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
2025-12-24T02:11:14.674855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028543:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.674905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028529:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.674909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241294655028529:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.675310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241294655028466:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.675338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.675347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241294655028466:2048] [node 2] ICP22 created new session: [1:7587241294655028551:2048]
2025-12-24T02:11:14.675374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241294655028551:2048] [node 2] ICS09 handshake done sender: [1:7587241294655028527:2107] self: [1:7587241294655028526:0] peer: [2:7587241296893962023:0] socket: 486 qp: -1
2025-12-24T02:11:14.675390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241294655028551:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:14.675399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.675677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445063:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.680161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241294655028554:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17672
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.680371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781207:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.680284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
2025-12-24T02:11:14.679603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962012:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.679670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241296893962024:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61784
2025-12-24T02:11:14.683513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241294655028529:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.682971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962011:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.684610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241294655028467:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.684634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.684639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241294655028467:2049] [node 3] ICP22 created new session: [1:7587241294655028557:2049]
2025-12-24T02:11:14.684645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241294655028557:2049] [node 3] ICS09 handshake done sender: [1:7587241294655028529:2108] self: [1:7587241294655028528:0] peer: [3:7587241296986445062:0] socket: 487 qp: -1
2025-12-24T02:11:14.684649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241294655028557:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:14.684657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.685295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241296893961939:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241296893961999:2107] held: no
2025-12-24T02:11:14.685332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241296893961999:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.686733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241296893961939:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.686749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.686754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241296893961939:2048] [node 1] ICP22 created new session: [2:7587241296893962026:2048]
2025-12-24T02:11:14.686759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241296893962026:2048] [node 1] ICS09 handshake done sender: [2:7587241296893961999:2107] self: [2:7587241296893962023:0] peer: [1:7587241294655028526:0] socket: 488 qp: -1
2025-12-24T02:11:14.686764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241296893962026:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.686771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.683703Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241296986445039:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.683755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241296986444979:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.683772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.683781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241296986444979:2048] [node 1] ICP22 created new session: [3:7587241296986445065:2048]
2025-12-24T02:11:14.683787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241296986445065:2048] [node 1] ICS09 handshake done sender: [3:7587241296986445039:2107] self: [3:7587241296986445062:0] peer: [1:7587241294655028528:0] socket: 490 qp: -1
2025-12-24T02:11:14.683795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241296986445065:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.683804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.717820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445068:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.718938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445058:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.720262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445063:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.720296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445048:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.720299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241296986445048:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.720956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241296986445048:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.721070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445057:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.721077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241296986445057:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.721574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241296986445042:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.721588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.721596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241296986445042:2109] [node 4] ICP22 created new session: [3:7587241296986445072:2109]
2025-12-24T02:11:14.721602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241296986445072:2109] [node 4] ICS09 handshake done sender: [3:7587241296986445048:2112] self: [3:7587241296986445047:0] peer: [4:7587241297177781226:0] socket: 500 qp: -1
2025-12-24T02:11:14.721609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241296986445072:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:14.721617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.718821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442556:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.722404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442557:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.719341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241297177781148:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.719450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781225:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.722043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781227:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.719979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481583:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.720037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481584:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.720080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481585:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.720788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295168481584:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.720823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295168481503:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.722338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481587:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.723168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028547:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028548:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295168481583:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295168481502:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.723465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295168481585:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295168481501:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.723474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028538:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241294655028538:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.723870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241296986445057:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.723633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962013:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962016:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241296893962016:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.724874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962010:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.724879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241296893962010:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.725306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962014:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.725807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028545:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.725818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241294655028545:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.723202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297722442556:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297722442544:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.723255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297722442475:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.723266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297722442474:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.726833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445068:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.726850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241296986445054:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.726864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.726870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241296986445054:2113] [node 6] ICP22 created new session: [3:7587241296986445076:2113]
2025-12-24T02:11:14.726877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241296986445076:2113] [node 6] ICS09 handshake done sender: [3:7587241296986445057:2115] self: [3:7587241296986445056:0] peer: [6:7587241295168481586:0] socket: 498 qp: -1
2025-12-24T02:11:14.726882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241296986445076:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:14.726893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.727308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442560:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.727337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442561:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.727521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295168481584:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.727608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481590:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.727624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481591:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.727632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295168481503:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.727643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.727648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295168481503:2050] [node 3] ICP22 created new session: [6:7587241295168481592:2050]
2025-12-24T02:11:14.727653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295168481592:2050] [node 3] ICS09 handshake done sender: [6:7587241295168481584:2116] self: [6:7587241295168481586:0] peer: [3:7587241296986445056:0] socket: 538 qp: -1
2025-12-24T02:11:14.727658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295168481592:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:14.727664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.726414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241297177781148:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241297177781207:2108] held: no
2025-12-24T02:11:14.726511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781225:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.726524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241297177781207:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.730929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241294655028538:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.731109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241294655028545:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.731227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028554:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.731239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241294655028535:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.731255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.731262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241294655028535:2110] [node 5] ICP22 created new session: [1:7587241294655028564:2110]
2025-12-24T02:11:14.731268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241294655028564:2110] [node 5] ICS09 handshake done sender: [1:7587241294655028538:2112] self: [1:7587241294655028537:0] peer: [5:7587241297722442559:0] socket: 509 qp: -1
2025-12-24T02:11:14.731274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241294655028564:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:14.731284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.731491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241294655028541:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.731501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.731511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241294655028541:2111] [node 6] ICP22 created new session: [1:7587241294655028566:2111]
2025-12-24T02:11:14.731517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241294655028566:2111] [node 6] ICS09 handshake done sender: [1:7587241294655028545:2116] self: [1:7587241294655028544:0] peer: [6:7587241295168481589:0] socket: 496 qp: -1
2025-12-24T02:11:14.731521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241294655028566:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:14.731527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.732083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241294655028540:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.732087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241294655028540:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.731613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241297177781148:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.731629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.731633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241297177781148:2050] [node 3] ICP22 created new session: [4:7587241297177781229:2050]
2025-12-24T02:11:14.731638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241297177781229:2050] [node 3] ICS09 handshake done sender: [4:7587241297177781207:2108] self: [4:7587241297177781226:0] peer: [3:7587241296986445047:0] socket: 501 qp: -1
2025-12-24T02:11:14.731643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241297177781229:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:14.731648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.731790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241297177781146:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.733363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241294655028540:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.733898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241297722442474:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241297722442544:2112] held: no
2025-12-24T02:11:14.734109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297722442556:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.732857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241296893962016:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.733027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241296893962010:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.733095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241296893962007:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.733110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.733116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241296893962007:2110] [node 5] ICP22 created new session: [2:7587241296893962034:2110]
2025-12-24T02:11:14.733121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241296893962034:2110] [node 5] ICS09 handshake done sender: [2:7587241296893962016:2117] self: [2:7587241296893962015:0] peer: [5:7587241297722442558:0] socket: 524 qp: -1
2025-12-24T02:11:14.733127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241296893962034:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:14.733135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.733241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241296893962008:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.733250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.733253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241296893962008:2111] [node 6] ICP22 created new session: [2:7587241296893962036:2111]
2025-12-24T02:11:14.733258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241296893962036:2111] [node 6] ICS09 handshake done sender: [2:7587241296893962010:2112] self: [2:7587241296893962009:0] peer: [6:7587241295168481588:0] socket: 523 qp: -1
2025-12-24T02:11:14.733262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241296893962036:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:14.733267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.734183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297722442475:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.734196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.734202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297722442475:2049] [node 2] ICP22 created new session: [5:7587241297722442564:2049]
2025-12-24T02:11:14.734207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297722442564:2049] [node 2] ICS09 handshake done sender: [5:7587241297722442556:2117] self: [5:7587241297722442558:0] peer: [2:7587241296893962015:0] socket: 536 qp: -1
2025-12-24T02:11:14.734212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297722442564:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.734217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.734368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442562:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.734377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297722442544:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.735056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241296986444980:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.735643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781228:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.735772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781233:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.735938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297722442474:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.735953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.735958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297722442474:2048] [node 1] ICP22 created new session: [5:7587241297722442566:2048]
2025-12-24T02:11:14.735964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297722442566:2048] [node 1] ICS09 handshake done sender: [5:7587241297722442544:2112] self: [5:7587241297722442559:0] peer: [1:7587241294655028537:0] socket: 510 qp: -1
2025-12-24T02:11:14.735970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297722442566:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.735976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.736093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297722442557:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.736160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297722442476:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.736432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241296986445060:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.736441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241296986445060:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.737267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241297177781225:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.737389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241296986445060:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.737450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297722442562:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.737474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297722442553:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.738834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442570:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.739029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295168481585:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.739179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241294655028534:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.739202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.739209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241294655028534:2109] [node 4] ICP22 created new session: [1:7587241294655028570:2109]
2025-12-24T02:11:14.739216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241294655028570:2109] [node 4] ICS09 handshake done sender: [1:7587241294655028540:2113] self: [1:7587241294655028539:0] peer: [4:7587241297177781232:0] socket: 532 qp: -1
2025-12-24T02:11:14.739223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241294655028570:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:14.739231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.739169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781222:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.739179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241297177781222:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.739551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241297177781146:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.739562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.739567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241297177781146:2048] [node 1] ICP22 created new session: [4:7587241297177781234:2048]
2025-12-24T02:11:14.739572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241297177781234:2048] [node 1] ICS09 handshake done sender: [4:7587241297177781225:2116] self: [4:7587241297177781232:0] peer: [1:7587241294655028539:0] socket: 534 qp: -1
2025-12-24T02:11:14.739577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241297177781234:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.739584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.740463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241296986445055:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.740482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.740487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241296986445055:2114] [node 5] ICP22 created new session: [3:7587241296986445080:2114]
2025-12-24T02:11:14.740492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241296986445080:2114] [node 5] ICS09 handshake done sender: [3:7587241296986445060:2117] self: [3:7587241296986445059:0] peer: [5:7587241297722442568:0] socket: 543 qp: -1
2025-12-24T02:11:14.740497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241296986445080:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:14.740504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.740797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295168481501:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.740814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.740818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295168481501:2048] [node 1] ICP22 created new session: [6:7587241295168481595:2048]
2025-12-24T02:11:14.740824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295168481595:2048] [node 1] ICS09 handshake done sender: [6:7587241295168481585:2117] self: [6:7587241295168481589:0] peer: [1:7587241294655028544:0] socket: 539 qp: -1
2025-12-24T02:11:14.740829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295168481595:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.740837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.742006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241295168481501:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:14.742023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295168481583:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.742090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295168481502:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.742103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.742109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295168481502:2049] [node 2] ICP22 created new session: [6:7587241295168481597:2049]
2025-12-24T02:11:14.742113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295168481597:2049] [node 2] ICS09 handshake done sender: [6:7587241295168481583:2115] self: [6:7587241295168481588:0] peer: [2:7587241296893962009:0] socket: 537 qp: -1
2025-12-24T02:11:14.742119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295168481597:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.742124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.743172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781228:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.743376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781238:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.743827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297722442557:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.743939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241297177781222:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.744004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297722442476:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.744019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.744024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297722442476:2050] [node 3] ICP22 created new session: [5:7587241297722442571:2050]
2025-12-24T02:11:14.744030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297722442571:2050] [node 3] ICS09 handshake done sender: [5:7587241297722442557:2118] self: [5:7587241297722442568:0] peer: [3:7587241296986445059:0] socket: 544 qp: -1
2025-12-24T02:11:14.744036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297722442571:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:14.744043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.745010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241297177781218:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.745024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.745030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241297177781218:2113] [node 5] ICP22 created new session: [4:7587241297177781240:2113]
2025-12-24T02:11:14.745036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241297177781240:2113] [node 5] ICS09 handshake done sender: [4:7587241297177781222:2114] self: [4:7587241297177781221:0] peer: [5:7587241297722442569:0] socket: 516 qp: -1
2025-12-24T02:11:14.745040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241297177781240:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:14.745048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.746569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481599:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.746883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241297722442476:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:14.746938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297722442573:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4394
2025-12-24T02:11:14.747151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297722442562:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.747238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297722442553:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.747251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.747257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297722442553:2115] [node 4] ICP22 created new session: [5:7587241297722442574:2115]
2025-12-24T02:11:14.747262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297722442574:2115] [node 4] ICS09 handshake done sender: [5:7587241297722442562:2121] self: [5:7587241297722442569:0] peer: [4:7587241297177781221:0] socket: 511 qp: -1
2025-12-24T02:11:14.747268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297722442574:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:14.747277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.747992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781238:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.748095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241297177781218:2113] [node 5] ICP18 handshake (actor [4:7587241297177781238:2120]) is too old
2025-12-24T02:11:14.748443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295168481599:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.748473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295168481578:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.748576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781224:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.748582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241297177781224:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.749223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295168481601:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37428
2025-12-24T02:11:14.749460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241297177781224:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.749624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295168481599:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.749679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295168481578:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.749692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.749697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295168481578:2112] [node 4] ICP22 created new session: [6:7587241295168481602:2112]
2025-12-24T02:11:14.749702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295168481602:2112] [node 4] ICS09 handshake done sender: [6:7587241295168481599:2121] self: [6:7587241295168481600:0] peer: [4:7587241297177781223:0] socket: 519 qp: -1
2025-12-24T02:11:14.749707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295168481602:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:14.749714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.750571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241297177781217:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.750586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.750591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241297177781217:2112] [node 6] ICP22 created new session: [4:7587241297177781243:2112]
2025-12-24T02:11:14.750596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241297177781243:2112] [node 6] ICS09 handshake done sender: [4:7587241297177781224:2115] self: [4:7587241297177781223:0] peer: [6:7587241295168481600:0] socket: 526 qp: -1
2025-12-24T02:11:14.750601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241297177781243:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:14.750606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.750942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962018:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.750957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241296893962018:2118] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.752221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241296986445082:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32053
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.752399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781245:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.752505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241296893962018:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.752695Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241296986445068:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.752768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241296986444980:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.752781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.752786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241296986444980:2049] [node 2] ICP22 created new session: [3:7587241296986445083:2049]
2025-12-24T02:11:14.752792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241296986445083:2049] [node 2] ICS09 handshake done sender: [3:7587241296986445068:2120] self: [3:7587241296986445078:0] peer: [2:7587241296893962017:0] socket: 535 qp: -1
2025-12-24T02:11:14.752797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241296986445083:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.752804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.753590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241296893961940:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.753606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.753612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241296893961940:2049] [node 3] ICP22 created new session: [2:7587241296893962040:2049]
2025-12-24T02:11:14.753618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241296893962040:2049] [node 3] ICS09 handshake done sender: [2:7587241296893962018:2118] self: [2:7587241296893962017:0] peer: [3:7587241296986445078:0] socket: 508 qp: -1
2025-12-24T02:11:14.753623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241296893962040:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:14.753630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.755502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241297177781245:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.755532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241297177781147:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.755638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241296893962020:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.755646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241296893962020:2119] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.756214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241297177781247:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10274
2025-12-24T02:11:14.756306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241296893962020:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.756470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241297177781245:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.756524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241297177781147:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.756541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.756546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241297177781147:2049] [node 2] ICP22 created new session: [4:7587241297177781248:2049]
2025-12-24T02:11:14.756551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241297177781248:2049] [node 2] ICS09 handshake done sender: [4:7587241297177781245:2121] self: [4:7587241297177781246:0] peer: [2:7587241296893962019:0] socket: 522 qp: -1
2025-12-24T02:11:14.756557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241297177781248:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.756565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.757486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241296893962006:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.757502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.757506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241296893962006:2109] [node 4] ICP22 created new session: [2:7587241296893962043:2109]
2025-12-24T02:11:14.757512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241296893962043:2109] [node 4] ICS09 handshake done sender: [2:7587241296893962020:2119] self: [2:7587241296893962019:0] peer: [4:7587241297177781246:0] socket: 492 qp: -1
2025-12-24T02:11:14.757517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241296893962043:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:14.757524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.761585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:14.761662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241296893962026:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:14.761669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241296893961939:2048] [node 1] ICP30 unregister session Session# [2:7587241296893962026:2048] VirtualId# [2:7587241296893962023:0]
2025-12-24T02:11:14.761676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241296893962026:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.761710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.761728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.762054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:14.762111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241296986445065:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-12-24T02:11:14.762115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241296986444979:2048] [node 1] ICP30 unregister session Session# [3:7587241296986445065:2048] VirtualId# [3:7587241296986445062:0]
2025-12-24T02:11:14.762119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241296986445065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.762140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.762152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.766946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:14.767052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241297177781234:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:14.767062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241297177781146:2048] [node 1] ICP30 unregister session Session# [4:7587241297177781234:2048] VirtualId# [4:7587241297177781232:0]
2025-12-24T02:11:14.767068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241297177781234:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.767133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.767161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.769244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:14.769319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241296986445083:2049] [node 2] ICS01 socket: 535 reason# EndOfStream
2025-12-24T02:11:14.769324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241296986444980:2049] [node 2] ICP30 unregister session Session# [3:7587241296986445083:2049] VirtualId# [3:7587241296986445078:0]
2025-12-24T02:11:14.769327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241296986445083:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.769365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.769381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.768244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:14.768323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297722442566:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-12-24T02:11:14.768330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297722442474:2048] [node 1] ICP30 unregister session Session# [5:7587241297722442566:2048] VirtualId# [5:7587241297722442559:0]
2025-12-24T02:11:14.768336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297722442566:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.768385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.768407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.768442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:14.768497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241295168481595:2048] [node 1] ICS01 socket: 539 reason# EndOfStream
2025-12-24T02:11:14.768502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295168481501:2048] [node 1] ICP30 unregister session Session# [6:7587241295168481595:2048] VirtualId# [6:7587241295168481589:0]
2025-12-24T02:11:14.768506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241295168481595:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.768532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:14.768550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:14.770883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:14.770973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241297177781248:2049] [node 2] ICS01 socket: 522 reason# EndOfStream
2025-12-24T02:11:14.770980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241297177781147:2049] [node 2] ICP30 unregister session Session# [4:7587241297177781248:2049] VirtualId# [4:7587241297177781246:0]
2025-12-24T02:11:14.770986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241297177781248:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.771043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.771068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.775431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:14.775537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297722442564:2049] [node 2] ICS01 socket: 536 reason# EndOfStream
2025-12-24T02:11:14.775545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297722442475:2049] [node 2] ICP30 unregister session Session# [5:7587241297722442564:2049] VirtualId# [5:7587241297722442558:0]
2025-12-24T02:11:14.775551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297722442564:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.775600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.775628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.775669Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:14.775729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241295168481597:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:14.775734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295168481502:2049] [node 2] ICP30 unregister session Session# [6:7587241295168481597:2049] VirtualId# [6:7587241295168481588:0]
2025-12-24T02:11:14.775738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241295168481597:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.775766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:14.775788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:14.779263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:14.779341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241295168481592:2050] [node 3] ICS01 socket: 538 reason# EndOfStream
2025-12-24T02:11:14.779350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295168481503:2050] [node 3] ICP30 unregister session Session# [6:7587241295168481592:2050] VirtualId# [6:7587241295168481586:0]
2025-12-24T02:11:14.779372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241295168481592:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.779419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:14.779443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:14.779480Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:14.779549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297722442571:2050] [node 3] ICS01 socket: 544 reason# EndOfStream
2025-12-24T02:11:14.779552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297722442476:2050] [node 3] ICP30 unregister session Session# [5:7587241297722442571:2050] VirtualId# [5:7587241297722442568:0]
2025-12-24T02:11:14.779555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297722442571:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.779588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:14.779606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:14.797492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:14.797597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241295168481602:2112] [node 4] ICS01 socket: 519 reason# EndOfStream
2025-12-24T02:11:14.797610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295168481578:2112] [node 4] ICP30 unregister session Session# [6:7587241295168481602:2112] VirtualId# [6:7587241295168481600:0]
2025-12-24T02:11:14.797618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241295168481602:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.797678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:14.797714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:14.797758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:14.797829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297722442574:2115] [node 4] ICS01 socket: 511 reason# EndOfStream
2025-12-24T02:11:14.797835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297722442553:2115] [node 4] ICP30 unregister session Session# [5:7587241297722442574:2115] VirtualId# [5:7587241297722442569:0]
2025-12-24T02:11:14.797840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297722442574:2115] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:14.797870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:14.797891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:14.840656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241296920398730:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:14.840742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241296920398731:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:14.840477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241293966858958:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.840490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241293966858959:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:14.846887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241295274540870:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.846904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241295274540871:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.846909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241295274540872:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:14.846848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241295105597032:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.846858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241295105597033:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.851480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297710243238:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.851490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297710243239:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.851494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297710243240:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:14.854210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295880483955:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:14.854220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295880483956:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:14.854225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295880483957:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:14.936631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241293966859022:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.937104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241293966859023:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32804
2025-12-24T02:11:14.937717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241293966859029:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:14.937805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241293966859030:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.938638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243299:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.939787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241296920398795:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.940085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241295105597095:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27482
2025-12-24T02:11:14.940109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241295105597096:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27482
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.946977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241295880484016:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.947937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243300:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:14.948966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241295105597098:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:14.948994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241295274540931:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:14.951262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241296920398801:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:14.951391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241296920398802:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.959258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241293966859023:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.959776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241293966858958:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.956722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243301:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:14.960450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297710243300:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.960698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297710243301:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.961175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297710243238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.962991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241297710243299:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.963371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297710243239:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.963902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241293966859032:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.963916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241293966859032:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.964603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241295105597095:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.964997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241295105597033:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.965219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243306:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:14.965363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241297710243303:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:14.965409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297710243299:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.965113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241293966859032:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.965348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241293966859019:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.965353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241293966859019:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.965420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241296920398799:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:14.966135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398791:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.966148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241296920398791:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.967266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398804:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.967291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241296920398804:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.965271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241295274540932:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:14.974762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295880484018:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.971506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297710243303:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.974788Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241295880484018:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.971536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297710243239:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.971565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.971574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297710243239:2049] [node 2] ICP22 created new session: [5:7587241297710243308:2049]
2025-12-24T02:11:14.971581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297710243308:2049] [node 2] ICS09 handshake done sender: [5:7587241297710243299:2108] self: [5:7587241297710243305:0] peer: [2:7587241293966859031:0] socket: 501 qp: -1
2025-12-24T02:11:14.971592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297710243308:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:14.971618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.979194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241295105597096:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.979241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241295105597032:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.978877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241296920398791:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:14.979086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241296920398730:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.979104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:14.979114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241296920398730:2048] [node 2] ICP22 created new session: [1:7587241296920398811:2048]
2025-12-24T02:11:14.979122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241296920398811:2048] [node 2] ICS09 handshake done sender: [1:7587241296920398791:2107] self: [1:7587241296920398790:0] peer: [2:7587241293966859038:0] socket: 494 qp: -1
2025-12-24T02:11:14.979132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241296920398811:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:14.979142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:14.979801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398794:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.979805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241296920398794:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.980484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241293966859029:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.980504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.980510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241293966859029:2111] [node 5] ICP22 created new session: [2:7587241293966859042:2111]
2025-12-24T02:11:14.980520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241293966859042:2111] [node 5] ICS09 handshake done sender: [2:7587241293966859032:2113] self: [2:7587241293966859031:0] peer: [5:7587241297710243305:0] socket: 500 qp: -1
2025-12-24T02:11:14.980528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241293966859042:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:14.980535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.984388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241295274540931:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.984434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241295274540932:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.984942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241295274540871:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.985189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241295274540872:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:14.985439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241295105597100:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.985451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241295105597100:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.987305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295880484018:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.989106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241295105597100:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:14.987768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398795:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.990592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241293966859025:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:14.990613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241293966859025:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:14.990873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243310:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.991623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241295105597098:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.991645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:14.991653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241295105597098:2110] [node 4] ICP22 created new session: [3:7587241295105597107:2110]
2025-12-24T02:11:14.991661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241295105597107:2110] [node 4] ICS09 handshake done sender: [3:7587241295105597100:2111] self: [3:7587241295105597099:0] peer: [4:7587241295274540934:0] socket: 512 qp: -1
2025-12-24T02:11:14.991669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241295105597107:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:14.991678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:14.991850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241295105597105:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27482
2025-12-24T02:11:14.991864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241295105597106:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27482
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.992077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241297710243311:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:14.992264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297710243300:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:14.992337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297710243303:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.992369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:14.992377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297710243303:2111] [node 6] ICP22 created new session: [5:7587241297710243312:2111]
2025-12-24T02:11:14.992384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297710243312:2111] [node 6] ICS09 handshake done sender: [5:7587241297710243300:2109] self: [5:7587241297710243307:0] peer: [6:7587241295880484017:0] socket: 508 qp: -1
2025-12-24T02:11:14.992390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297710243312:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:14.992397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:14.992676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241295105597096:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.992745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241295105597032:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.992760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.992765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241295105597032:2048] [node 1] ICP22 created new session: [3:7587241295105597109:2048]
2025-12-24T02:11:14.992770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241295105597109:2048] [node 1] ICS09 handshake done sender: [3:7587241295105597096:2109] self: [3:7587241295105597103:0] peer: [1:7587241296920398793:0] socket: 506 qp: -1
2025-12-24T02:11:14.992775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241295105597109:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.992782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:14.993760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241297710243301:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:14.993826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297710243238:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:14.993840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:14.993846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297710243238:2048] [node 1] ICP22 created new session: [5:7587241297710243314:2048]
2025-12-24T02:11:14.993851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241297710243314:2048] [node 1] ICS09 handshake done sender: [5:7587241297710243301:2110] self: [5:7587241297710243304:0] peer: [1:7587241296920398803:0] socket: 511 qp: -1
2025-12-24T02:11:14.993870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241297710243314:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:14.993876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.992211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241296920398794:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:14.993595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241296920398804:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:14.994061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241296920398731:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.994074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:14.994079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241296920398731:2049] [node 3] ICP22 created new session: [1:7587241296920398817:2049]
2025-12-24T02:11:14.994085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241296920398817:2049] [node 3] ICS09 handshake done sender: [1:7587241296920398794:2108] self: [1:7587241296920398793:0] peer: [3:7587241295105597103:0] socket: 505 qp: -1
2025-12-24T02:11:14.994090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241296920398817:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:14.994096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:14.994231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241296920398801:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.994236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.994258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241296920398801:2111] [node 5] ICP22 created new session: [1:7587241296920398819:2111]
2025-12-24T02:11:14.994263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241296920398819:2111] [node 5] ICS09 handshake done sender: [1:7587241296920398804:2113] self: [1:7587241296920398803:0] peer: [5:7587241297710243304:0] socket: 510 qp: -1
2025-12-24T02:11:14.994267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241296920398819:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:14.994272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.994909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241295880484016:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:14.994945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:14.994950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295880484016:2101] [node 5] ICP22 created new session: [6:7587241295880484024:2101]
2025-12-24T02:11:14.994958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241295880484024:2101] [node 5] ICS09 handshake done sender: [6:7587241295880484018:2108] self: [6:7587241295880484017:0] peer: [5:7587241297710243307:0] socket: 507 qp: -1
2025-12-24T02:11:14.994965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241295880484024:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:14.994972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:14.995158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295880484022:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:14.995171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241295880484023:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:14.995436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241295274540935:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:15.003200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295880484022:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.003653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295880483955:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.003877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398806:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.003889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241296920398806:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.005184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241296920398806:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.005370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241296920398802:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.005386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.005414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241296920398802:2112] [node 6] ICP22 created new session: [1:7587241301215366119:2112]
2025-12-24T02:11:15.005421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241301215366119:2112] [node 6] ICS09 handshake done sender: [1:7587241296920398806:2114] self: [1:7587241296920398805:0] peer: [6:7587241300175451322:0] socket: 517 qp: -1
2025-12-24T02:11:15.005428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241301215366119:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.005436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.002988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241293966859019:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.003776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241293966858959:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.003785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.003791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241293966858959:2049] [node 3] ICP22 created new session: [2:7587241298261826341:2049]
2025-12-24T02:11:15.003796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241298261826341:2049] [node 3] ICS09 handshake done sender: [2:7587241293966859019:2107] self: [2:7587241293966859018:0] peer: [3:7587241295105597102:0] socket: 492 qp: -1
2025-12-24T02:11:15.003801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241298261826341:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:15.003807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.005442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241293966859044:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32804
2025-12-24T02:11:15.005914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241293966859023:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.006087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241293966858958:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.006098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.006104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241293966858958:2048] [node 1] ICP22 created new session: [2:7587241298261826344:2048]
2025-12-24T02:11:15.006109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241298261826344:2048] [node 1] ICS09 handshake done sender: [2:7587241293966859023:2109] self: [2:7587241293966859038:0] peer: [1:7587241296920398790:0] socket: 496 qp: -1
2025-12-24T02:11:15.006115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241298261826344:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.006120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.003328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241295105597095:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.003486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241295105597033:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.003498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.003507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241295105597033:2049] [node 2] ICP22 created new session: [3:7587241299400564407:2049]
2025-12-24T02:11:15.003514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299400564407:2049] [node 2] ICS09 handshake done sender: [3:7587241295105597095:2108] self: [3:7587241295105597102:0] peer: [2:7587241293966859018:0] socket: 499 qp: -1
2025-12-24T02:11:15.003522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299400564407:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.003531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.010779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241295274540936:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:15.010835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241295274540932:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.010881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241293966859025:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.011111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241293966859022:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.011126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.011134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241293966859022:2108] [node 4] ICP22 created new session: [2:7587241298261826347:2108]
2025-12-24T02:11:15.011141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241298261826347:2108] [node 4] ICS09 handshake done sender: [2:7587241293966859025:2110] self: [2:7587241293966859024:0] peer: [4:7587241295274540933:0] socket: 497 qp: -1
2025-12-24T02:11:15.011148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241298261826347:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:15.011157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.010841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241295880484023:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.011277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295880483956:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.011478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451323:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:15.011825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241301215366121:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-12-24T02:11:15.013147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295880484022:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.015115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302005210613:2115] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.027050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241295274540872:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.027075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.027083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241295274540872:2050] [node 3] ICP22 created new session: [4:7587241299569508235:2050]
2025-12-24T02:11:15.027090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241299569508235:2050] [node 3] ICS09 handshake done sender: [4:7587241295274540932:2109] self: [4:7587241295274540934:0] peer: [3:7587241295105597099:0] socket: 515 qp: -1
2025-12-24T02:11:15.027099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241299569508235:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.027125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.027468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241299569508237:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:15.028367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241299569508234:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:15.028513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241293966859034:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.028527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241293966859034:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.029403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241293966859022:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(18258127ns) {HandleHandshakeStatus:329(18257285ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(418ns) SwitchToState:179(228ns) ProcessPendingSessionEvents:495(18170329ns) {ForwardSessionEventToSession:612(18160507ns) ForwardSessionEventToSession:612(1224ns) ForwardSessionEventToSession:612(585ns) ForwardSessionEventToSession:612(489ns) ForwardSessionEventToSession:612(651ns) ForwardSessionEventToSession:612(1300ns) ForwardSessionEventToSession:612(534ns) ForwardSessionEventToSession:612(936ns) ForwardSessionEventToSession:612(724ns)}}}
2025-12-24T02:11:15.029472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299400564410:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:15.030451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241293966859034:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.030558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241293966859030:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.030573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.030580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241293966859030:2112] [node 6] ICP22 created new session: [2:7587241298261826350:2112]
2025-12-24T02:11:15.030586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241298261826350:2112] [node 6] ICS09 handshake done sender: [2:7587241293966859034:2114] self: [2:7587241293966859033:0] peer: [6:7587241300175451324:0] socket: 520 qp: -1
2025-12-24T02:11:15.030593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241298261826350:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.030601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.030970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299400564413:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:15.035045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241295274540936:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.035073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241295274540931:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.038999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295880483955:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.039029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.039042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295880483955:2048] [node 1] ICP22 created new session: [6:7587241300175451326:2048]
2025-12-24T02:11:15.039051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241300175451326:2048] [node 1] ICS09 handshake done sender: [6:7587241295880484022:2110] self: [6:7587241300175451322:0] peer: [1:7587241296920398805:0] socket: 522 qp: -1
2025-12-24T02:11:15.039062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241300175451326:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.039073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.039479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241300175451328:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:15.039717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451329:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:15.039730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451330:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:15.039823Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241302005210613:2115] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24641132ns) {Configure:104(24640347ns) {ProcessConfigured:135(24281088ns) {StartInitialHandshake:157(24280415ns) {DropHandshakes:470(947ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(149ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1241ns) {SwitchToState:179(145ns)}}}}}
2025-12-24T02:11:15.043002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241295880484023:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.043505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241299569508244:2116] [node 5] ICP01 ready to work
2025-12-24T02:11:15.043530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241295274540871:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.043539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.043545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241295274540871:2049] [node 2] ICP22 created new session: [4:7587241299569508245:2049]
2025-12-24T02:11:15.043553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241299569508245:2049] [node 2] ICS09 handshake done sender: [4:7587241295274540931:2108] self: [4:7587241295274540933:0] peer: [2:7587241293966859024:0] socket: 498 qp: -1
2025-12-24T02:11:15.043561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241299569508245:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.043567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.045061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295880483956:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.045080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.045088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295880483956:2049] [node 2] ICP22 created new session: [6:7587241300175451331:2049]
2025-12-24T02:11:15.045095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241300175451331:2049] [node 2] ICS09 handshake done sender: [6:7587241295880484023:2111] self: [6:7587241300175451324:0] peer: [2:7587241293966859033:0] socket: 502 qp: -1
2025-12-24T02:11:15.045103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241300175451331:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.045111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.054993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241299569508247:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:15.055049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241295274540870:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.055597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241299569508250:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.056595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299400564417:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27482
2025-12-24T02:11:15.057186Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241299400564413:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26147526ns) {Configure:104(26146810ns) {ProcessConfigured:135(25858289ns) {StartInitialHandshake:157(25857774ns) {DropHandshakes:470(458ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:477(1294ns) {SwitchToState:179(397ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.059762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302005210617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:15.059948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302005210618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:15.061507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451339:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:15.062990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302005210617:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.063384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241297710243240:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.063584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299400564415:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.063596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241299400564415:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.064239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302005210620:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:15.064343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299400564415:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.064214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241301215366122:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61993
2025-12-24T02:11:15.066857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241296920398810:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.066872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241296920398810:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.071137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241300175451339:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.071182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241295880483957:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.068433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241299400564410:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.068456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.068466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299400564410:2114] [node 5] ICP22 created new session: [3:7587241299400564421:2114]
2025-12-24T02:11:15.068473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299400564421:2114] [node 5] ICS09 handshake done sender: [3:7587241299400564415:2116] self: [3:7587241299400564411:0] peer: [5:7587241302005210619:0] socket: 541 qp: -1
2025-12-24T02:11:15.068481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299400564421:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:15.068492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.068960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302005210617:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.072150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241297710243240:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.072166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.072172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241297710243240:2050] [node 3] ICP22 created new session: [5:7587241302005210621:2050]
2025-12-24T02:11:15.072180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302005210621:2050] [node 3] ICS09 handshake done sender: [5:7587241302005210617:2117] self: [5:7587241302005210619:0] peer: [3:7587241299400564411:0] socket: 547 qp: -1
2025-12-24T02:11:15.072187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302005210621:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.072196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.070873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241296920398810:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.071446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241296920398799:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.071459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.071467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241296920398799:2110] [node 4] ICP22 created new session: [1:7587241301215366124:2110]
2025-12-24T02:11:15.071473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241301215366124:2110] [node 4] ICS09 handshake done sender: [1:7587241296920398810:2115] self: [1:7587241296920398809:0] peer: [4:7587241299569508249:0] socket: 504 qp: -1
2025-12-24T02:11:15.071481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241301215366124:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:15.071489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.068698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299400564417:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.071958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241299569508254:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3119
2025-12-24T02:11:15.078495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241300175451329:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.078617Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299400564418:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.078635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241299400564418:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.078591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241300175451328:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.078949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241299569508247:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.079042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241299569508250:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.079077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241299569508241:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.079080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241299569508241:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.079505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241295274540936:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.080979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241295274540870:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.081020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.081028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241295274540870:2048] [node 1] ICP22 created new session: [4:7587241299569508256:2048]
2025-12-24T02:11:15.081035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241299569508256:2048] [node 1] ICS09 handshake done sender: [4:7587241295274540936:2111] self: [4:7587241299569508249:0] peer: [1:7587241296920398809:0] socket: 531 qp: -1
2025-12-24T02:11:15.081044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241299569508256:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.081054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.081122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241295274540870:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:15.081519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241299569508241:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.081614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241299569508237:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.081629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.081634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241299569508237:2112] [node 6] ICP22 created new session: [4:7587241299569508260:2112]
2025-12-24T02:11:15.081639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241299569508260:2112] [node 6] ICS09 handshake done sender: [4:7587241299569508241:2115] self: [4:7587241299569508240:0] peer: [6:7587241300175451341:0] socket: 536 qp: -1
2025-12-24T02:11:15.081644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241299569508260:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.081650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.082818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299400564418:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.079783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241301215366122:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.079826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241296920398799:2110] [node 4] ICP18 handshake (actor [1:7587241301215366122:2117]) is too old
2025-12-24T02:11:15.083017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241299400564413:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.083033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.083041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299400564413:2115] [node 6] ICP22 created new session: [3:7587241299400564425:2115]
2025-12-24T02:11:15.083048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299400564425:2115] [node 6] ICS09 handshake done sender: [3:7587241299400564418:2118] self: [3:7587241299400564414:0] peer: [6:7587241300175451340:0] socket: 552 qp: -1
2025-12-24T02:11:15.083053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299400564425:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:15.083061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.083271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241299569508252:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.083280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241299569508252:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.082890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302005210618:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.083030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302005210613:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.095017Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241300175451328:2113] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12044) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241300175451329:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.096753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241295880483957:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241300175451339:2118] held: no
2025-12-24T02:11:15.096815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451342:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.097031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241300175451343:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59741
2025-12-24T02:11:15.097196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241300175451339:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.097260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241300175451329:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.097319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241295880483957:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.097335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.097343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241295880483957:2050] [node 3] ICP22 created new session: [6:7587241300175451344:2050]
2025-12-24T02:11:15.097351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241300175451344:2050] [node 3] ICS09 handshake done sender: [6:7587241300175451339:2118] self: [6:7587241300175451340:0] peer: [3:7587241299400564414:0] socket: 555 qp: -1
2025-12-24T02:11:15.097359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241300175451344:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.097367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.097507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241300175451328:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.097514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.097520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241300175451328:2113] [node 4] ICP22 created new session: [6:7587241300175451346:2113]
2025-12-24T02:11:15.097525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241300175451346:2113] [node 4] ICS09 handshake done sender: [6:7587241300175451329:2114] self: [6:7587241300175451341:0] peer: [4:7587241299569508240:0] socket: 538 qp: -1
2025-12-24T02:11:15.097530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241300175451346:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:15.097535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.099440Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241299569508252:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.099564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241299569508244:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.099581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.099589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241299569508244:2116] [node 5] ICP22 created new session: [4:7587241299569508263:2116]
2025-12-24T02:11:15.099596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241299569508263:2116] [node 5] ICS09 handshake done sender: [4:7587241299569508252:2119] self: [4:7587241299569508251:0] peer: [5:7587241302005210623:0] socket: 551 qp: -1
2025-12-24T02:11:15.099602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241299569508263:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:15.099610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.108427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302005210624:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16125
2025-12-24T02:11:15.109382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302005210618:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.110263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241302005210613:2115] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12044) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241302005210618:2118] held: no
2025-12-24T02:11:15.110311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302005210613:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.110330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.110343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302005210613:2115] [node 4] ICP22 created new session: [5:7587241302005210625:2115]
2025-12-24T02:11:15.110356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302005210625:2115] [node 4] ICS09 handshake done sender: [5:7587241302005210618:2118] self: [5:7587241302005210623:0] peer: [4:7587241299569508251:0] socket: 554 qp: -1
2025-12-24T02:11:15.110370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302005210625:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:15.110384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.166513Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:15.166612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241298261826344:2048] [node 1] ICS01 socket: 496 reason# EndOfStream
2025-12-24T02:11:15.166621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241293966858958:2048] [node 1] ICP30 unregister session Session# [2:7587241298261826344:2048] VirtualId# [2:7587241293966859038:0]
2025-12-24T02:11:15.166628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241298261826344:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.166680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.166827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.167255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:15.167333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241295105597109:2048] [node 1] ICS07 socket disconnect 506 reason# EndOfStream
2025-12-24T02:11:15.167339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241295105597109:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.167394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.167413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241295105597109:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:15.169248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241295105597032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:15.169265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241295105597109:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:15.169807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241295105597032:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.169821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241295105597109:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.169826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241295105597032:2048] [node 1] ICP30 unregister session Session# [3:7587241295105597109:2048] VirtualId# [3:7587241295105597103:0]
2025-12-24T02:11:15.169843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.169850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241295105597032:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.169852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.173099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:15.173170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241297710243314:2048] [node 1] ICS07 socket disconnect 511 reason# EndOfStream
2025-12-24T02:11:15.173175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297710243314:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.173222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.173241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241297710243314:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:15.174761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:15.174850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241300175451326:2048] [node 1] ICS07 socket disconnect 522 reason# EndOfStream
2025-12-24T02:11:15.174856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241300175451326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.174924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.174946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241300175451326:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:15.175345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241295880483955:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.175373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241300175451326:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.175379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295880483955:2048] [node 1] ICP30 unregister session Session# [6:7587241300175451326:2048] VirtualId# [6:7587241300175451322:0]
2025-12-24T02:11:15.175397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.175404Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241295880483955:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.175406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.175464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:15.175558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241299569508256:2048] [node 1] ICS07 socket disconnect 531 reason# EndOfStream
2025-12-24T02:11:15.175562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241299569508256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.175627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.175654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241299569508256:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:15.176154Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241295274540870:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.176167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241299569508256:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.176172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241295274540870:2048] [node 1] ICP30 unregister session Session# [4:7587241299569508256:2048] VirtualId# [4:7587241299569508249:0]
2025-12-24T02:11:15.176186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.176191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241295274540870:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.176194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.175975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241297710243238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.175996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297710243314:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.176002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297710243238:2048] [node 1] ICP30 unregister session Session# [5:7587241297710243314:2048] VirtualId# [5:7587241297710243304:0]
2025-12-24T02:11:15.176033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.176039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241297710243238:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.176041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.184037Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:15.184138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241299400564407:2049] [node 2] ICS07 socket disconnect 499 reason# EndOfStream
2025-12-24T02:11:15.184145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241299400564407:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.184193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.184216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241299400564407:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:15.186537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241297710243238:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:15.186589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241295274540870:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:15.186482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241295105597033:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:15.186502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241299400564407:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:15.187129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241295105597033:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.187142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241299400564407:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.187147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241295105597033:2049] [node 2] ICP30 unregister session Session# [3:7587241299400564407:2049] VirtualId# [3:7587241295105597102:0]
2025-12-24T02:11:15.187164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.187170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241295105597033:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.187172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.188145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241295105597032:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:15.188158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241295105597033:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:15.188525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:15.188614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241297710243308:2049] [node 2] ICS07 socket disconnect 501 reason# EndOfStream
2025-12-24T02:11:15.188620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241297710243308:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.188671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.188700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241297710243308:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:15.189110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241297710243239:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.189189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241297710243308:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.189194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297710243239:2049] [node 2] ICP30 unregister session Session# [5:7587241297710243308:2049] VirtualId# [5:7587241297710243305:0]
2025-12-24T02:11:15.189211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.189216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241297710243239:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.189219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.190342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:15.190424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241299569508245:2049] [node 2] ICS07 socket disconnect 498 reason# EndOfStream
2025-12-24T02:11:15.190435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241299569508245:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.190507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.190545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241299569508245:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:15.190741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241295880483955:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:15.190845Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:15.190911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241300175451331:2049] [node 2] ICS07 socket disconnect 502 reason# EndOfStream
2025-12-24T02:11:15.190917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241300175451331:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.190973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.190994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241300175451331:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:15.191337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241295880483956:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.191350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241300175451331:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.191355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295880483956:2049] [node 2] ICP30 unregister session Session# [6:7587241300175451331:2049] VirtualId# [6:7587241300175451324:0]
2025-12-24T02:11:15.191384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.191389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241295880483956:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.191391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.194114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241295274540871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.194132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241299569508245:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.194137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241295274540871:2049] [node 2] ICP30 unregister session Session# [4:7587241299569508245:2049] VirtualId# [4:7587241295274540933:0]
2025-12-24T02:11:15.194153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.194160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241295274540871:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.194162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9344) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.195896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:15.195984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241299569508235:2050] [node 3] ICS07 socket disconnect 515 reason# EndOfStream
2025-12-24T02:11:15.195994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241299569508235:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.196054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.196086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241299569508235:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:15.196333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:15.196411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241302005210621:2050] [node 3] ICS07 socket disconnect 547 reason# EndOfStream
2025-12-24T02:11:15.196415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241302005210621:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.196447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.196467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241302005210621:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:15.196877Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241297710243240:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.196891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241302005210621:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.196898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241297710243240:2050] [node 3] ICP30 unregister session Session# [5:7587241302005210621:2050] VirtualId# [5:7587241302005210619:0]
2025-12-24T02:11:15.196912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.196917Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241297710243240:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.196920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.197120Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241295274540872:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.197132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241299569508235:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.197136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241295274540872:2050] [node 3] ICP30 unregister session Session# [4:7587241299569508235:2050] VirtualId# [4:7587241295274540934:0]
2025-12-24T02:11:15.197145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.197149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241295274540872:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.197152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23147) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.198586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241297710243239:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:15.198953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:15.199037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241300175451344:2050] [node 3] ICS01 socket: 555 reason# EndOfStream
2025-12-24T02:11:15.199045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241295880483957:2050] [node 3] ICP30 unregister session Session# [6:7587241300175451344:2050] VirtualId# [6:7587241300175451340:0]
2025-12-24T02:11:15.199051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241300175451344:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.199115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.199141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.202750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241295880483956:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:15.208271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241297710243240:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:15.218189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:15.218296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241302005210625:2115] [node 4] ICS07 socket disconnect 554 reason# EndOfStream
2025-12-24T02:11:15.218302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241302005210625:2115] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.218370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:15.218397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241302005210625:2115] [node 4] ICS15 start handshake
2025-12-24T02:11:15.218871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:15.218947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241300175451346:2113] [node 4] ICS01 socket: 538 reason# EndOfStream
2025-12-24T02:11:15.218955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241300175451328:2113] [node 4] ICP30 unregister session Session# [6:7587241300175451346:2113] VirtualId# [6:7587241300175451341:0]
2025-12-24T02:11:15.218961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241300175451346:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.219029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:15.219007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241302005210613:2115] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12044) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:15.219058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:15.219034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241302005210625:2115] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:15.219040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241302005210613:2115] [node 4] ICP30 unregister session Session# [5:7587241302005210625:2115] VirtualId# [5:7587241302005210623:0]
2025-12-24T02:11:15.219054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:15.219061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241302005210613:2115] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12044) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:15.219064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12044) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:15.251595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241300321543392:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.254784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241300321543393:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.250063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241299059212999:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.250109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241299059212998:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:15.259754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299744749976:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.259765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299744749977:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.261708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301410118152:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.261723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301410118154:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.261756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301410118153:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.263655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302098280296:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.263664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302098280297:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.263673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302098280298:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.275443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302355028202:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.275467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302355028203:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.275472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302355028204:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.297741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241299059213065:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:15.296882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241300321543452:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55111
2025-12-24T02:11:15.299203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241300321543457:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:15.298884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302355028264:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.300144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750036:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.300171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750037:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.299966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241300321543463:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:15.300084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241300321543465:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.302024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118213:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.302043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118214:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.304395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280358:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.304410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280359:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.298393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241299059213069:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:15.298414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241299059213070:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:15.304873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302098280358:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.305033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302098280363:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:15.305640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302098280363:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.308425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241299059213077:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34384
2025-12-24T02:11:15.303751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299744750040:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:15.303856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299744750041:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:15.307579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302355028266:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.307595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241302355028266:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.308252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302355028266:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.308595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241302355028264:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.308608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.308615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302355028264:2108] [node 5] ICP22 created new session: [6:7587241302355028269:2108]
2025-12-24T02:11:15.308625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302355028269:2108] [node 5] ICS09 handshake done sender: [6:7587241302355028266:2109] self: [6:7587241302355028265:0] peer: [5:7587241302098280364:0] socket: 503 qp: -1
2025-12-24T02:11:15.308633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302355028269:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:15.308642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.308796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241299744750049:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.309493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028271:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.315412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543452:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.315457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241300321543392:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.315918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750036:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.315471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301410118223:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:15.315575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301410118224:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:15.316107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302355028271:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.316455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302355028204:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.316041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750037:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.318790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241299059213078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34384
2025-12-24T02:11:15.319064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213059:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.319077Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241299059213059:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.319606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213063:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.319616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241299059213063:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.319930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750051:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.319949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241299744750051:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.322879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241299744749977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.323119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543456:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.323131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241300321543456:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.325032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118214:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.327005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213077:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.327047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213078:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.327099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241299059213080:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34384
2025-12-24T02:11:15.327206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241299059213079:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34384
2025-12-24T02:11:15.328120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118213:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.328171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301410118152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.328273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213080:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.328391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213068:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.328398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241299059213068:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.329007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118232:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.329119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241299059213068:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.329321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301410118213:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.329393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301410118152:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.329412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.329419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301410118152:2048] [node 1] ICP22 created new session: [4:7587241301410118233:2048]
2025-12-24T02:11:15.329425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301410118233:2048] [node 1] ICS09 handshake done sender: [4:7587241301410118213:2108] self: [4:7587241301410118231:0] peer: [1:7587241299059213067:0] socket: 510 qp: -1
2025-12-24T02:11:15.329433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301410118233:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.329441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.330272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241299059213065:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.330291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.330297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241299059213065:2108] [node 4] ICP22 created new session: [1:7587241299059213085:2108]
2025-12-24T02:11:15.330302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241299059213085:2108] [node 4] ICS09 handshake done sender: [1:7587241299059213068:2112] self: [1:7587241299059213067:0] peer: [4:7587241301410118231:0] socket: 502 qp: -1
2025-12-24T02:11:15.330309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241299059213085:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:15.330317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.330798Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241301410118153:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24188414ns) {Configure:104(24184582ns) {ProcessConfigured:135(23868268ns) {StartInitialHandshake:157(23866999ns) {DropHandshakes:470(607ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(693ns) {SwitchToState:179(500ns)}}}}}
2025-12-24T02:11:15.330865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301410118153:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.331143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543459:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.331155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241300321543459:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.332258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241299744749976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.332304Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241299744749976:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16329289ns) {IncomingHandshake:261(16328747ns) {DropIncomingHandshake:442(64ns) PrepareNewSessionHandshake:477(449ns) {SwitchToState:179(103ns)} GenerateSessionVirtualId:988(222ns) IssueIncomingHandshakeReply:193(16324593ns) {SwitchToState:179(839ns)}}}
2025-12-24T02:11:15.334843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241302098280367:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:15.335846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213079:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.338861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241302355028203:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19839435ns) {Configure:104(19837587ns) {ProcessConfigured:135(19550807ns) {StartInitialHandshake:157(19549824ns) {DropHandshakes:470(916ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1269ns) {SwitchToState:179(588ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.339743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280371:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.338919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241299059213059:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.340271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241299059212998:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.340285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.340294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241299059212998:2048] [node 2] ICP22 created new session: [1:7587241299059213090:2048]
2025-12-24T02:11:15.340301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241299059213090:2048] [node 2] ICS09 handshake done sender: [1:7587241299059213059:2107] self: [1:7587241299059213058:0] peer: [2:7587241300321543472:0] socket: 498 qp: -1
2025-12-24T02:11:15.340310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241299059213090:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:15.340347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.342186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241299059213063:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.339442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241300321543473:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55111
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.342976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241302098280298:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19915338ns) {Configure:104(19914027ns) {ProcessConfigured:135(19592877ns) {StartInitialHandshake:157(19592295ns) {DropHandshakes:470(590ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:477(1397ns) {SwitchToState:179(79ns)}}}}}
2025-12-24T02:11:15.343049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299744750051:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.343123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750060:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.343200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241299744750049:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.343213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.343219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299744750049:2111] [node 6] ICP22 created new session: [3:7587241299744750062:2111]
2025-12-24T02:11:15.343226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299744750062:2111] [node 6] ICS09 handshake done sender: [3:7587241299744750051:2114] self: [3:7587241299744750050:0] peer: [6:7587241302355028272:0] socket: 525 qp: -1
2025-12-24T02:11:15.343234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299744750062:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:15.343244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.343246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750061:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.343662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241300321543452:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.347063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299744750036:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.347283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750060:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.347296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241299744749976:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.347311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.347318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299744749976:2048] [node 1] ICP22 created new session: [3:7587241299744750065:2048]
2025-12-24T02:11:15.347325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299744750065:2048] [node 1] ICS09 handshake done sender: [3:7587241299744750036:2107] self: [3:7587241299744750056:0] peer: [1:7587241299059213062:0] socket: 506 qp: -1
2025-12-24T02:11:15.347332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299744750065:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.347341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.347528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241299059212999:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.347553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.347559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241299059212999:2049] [node 3] ICP22 created new session: [1:7587241299059213094:2049]
2025-12-24T02:11:15.347566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241299059213094:2049] [node 3] ICS09 handshake done sender: [1:7587241299059213063:2109] self: [1:7587241299059213062:0] peer: [3:7587241299744750056:0] socket: 500 qp: -1
2025-12-24T02:11:15.347570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241299059213094:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:15.347578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.350786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302355028276:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.351304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280372:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.355065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118238:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.355096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.358828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241302098280367:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23865680ns) {Configure:104(23864047ns) {ProcessConfigured:135(23510670ns) {StartInitialHandshake:157(23509933ns) {DropHandshakes:470(889ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(2476ns) {SwitchToState:179(983ns)}}}}}
2025-12-24T02:11:15.358889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280376:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.361741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302098280359:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.361935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302098280358:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.362071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302098280363:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.362089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.362098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302098280363:2111] [node 6] ICP22 created new session: [5:7587241302098280379:2111]
2025-12-24T02:11:15.362105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302098280379:2111] [node 6] ICS09 handshake done sender: [5:7587241302098280358:2108] self: [5:7587241302098280364:0] peer: [6:7587241302355028265:0] socket: 513 qp: -1
2025-12-24T02:11:15.362113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302098280379:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:15.362137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.362353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302098280297:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.362594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302098280376:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.362724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241300321543392:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.362746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.362751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241300321543392:2048] [node 1] ICP22 created new session: [2:7587241300321543478:2048]
2025-12-24T02:11:15.362757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241300321543478:2048] [node 1] ICS09 handshake done sender: [2:7587241300321543452:2107] self: [2:7587241300321543472:0] peer: [1:7587241299059213058:0] socket: 499 qp: -1
2025-12-24T02:11:15.362762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241300321543478:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.362778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.362966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543467:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.362971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241300321543467:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.363465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241300321543474:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55111
2025-12-24T02:11:15.363602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241300321543477:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55111
2025-12-24T02:11:15.363895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241300321543459:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.364025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241300321543456:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.364200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241300321543457:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.364210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.364218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241300321543457:2108] [node 4] ICP22 created new session: [2:7587241300321543481:2108]
2025-12-24T02:11:15.364224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241300321543481:2108] [node 4] ICS09 handshake done sender: [2:7587241300321543459:2110] self: [2:7587241300321543458:0] peer: [4:7587241301410118236:0] socket: 505 qp: -1
2025-12-24T02:11:15.364229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241300321543481:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:15.364235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.364329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241300321543393:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.364335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.364339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241300321543393:2049] [node 3] ICP22 created new session: [2:7587241300321543483:2049]
2025-12-24T02:11:15.364342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241300321543483:2049] [node 3] ICS09 handshake done sender: [2:7587241300321543456:2109] self: [2:7587241300321543455:0] peer: [3:7587241299744750058:0] socket: 504 qp: -1
2025-12-24T02:11:15.364345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241300321543483:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:15.364351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.364961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241300321543467:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.363599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241299744750049:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(20366538ns) {HandleHandshakeStatus:329(20365656ns) {DropIncomingHandshake:442(263ns) DropOutgoingHandshake:457(358ns) SwitchToState:179(262ns) ProcessPendingSessionEvents:495(20265807ns) {ForwardSessionEventToSession:612(20253512ns) ForwardSessionEventToSession:612(1701ns) ForwardSessionEventToSession:612(910ns) ForwardSessionEventToSession:612(525ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(867ns) ForwardSessionEventToSession:612(573ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(999ns) ForwardSessionEventToSession:612(826ns)}}}
2025-12-24T02:11:15.366982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301410118214:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.368090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301410118153:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.368113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.368122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301410118153:2049] [node 2] ICP22 created new session: [4:7587241301410118240:2049]
2025-12-24T02:11:15.368127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301410118240:2049] [node 2] ICS09 handshake done sender: [4:7587241301410118214:2109] self: [4:7587241301410118236:0] peer: [2:7587241300321543458:0] socket: 512 qp: -1
2025-12-24T02:11:15.368135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301410118240:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.368155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.370999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280373:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.371181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302098280372:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.368156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543474:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.368211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241300321543463:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.368222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.368228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241300321543463:2111] [node 5] ICP22 created new session: [2:7587241300321543487:2111]
2025-12-24T02:11:15.368233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241300321543487:2111] [node 5] ICS09 handshake done sender: [2:7587241300321543467:2113] self: [2:7587241300321543466:0] peer: [5:7587241302098280381:0] socket: 509 qp: -1
2025-12-24T02:11:15.368239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241300321543487:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:15.368247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.374515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302098280296:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.374649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302098280298:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.374945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543477:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.374983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241300321543463:2111] [node 5] ICP18 handshake (actor [2:7587241300321543477:2118]) is too old
2025-12-24T02:11:15.374822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241302355028276:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23928642ns) {Configure:104(23927312ns) {ProcessConfigured:135(23580021ns) {StartInitialHandshake:157(23579216ns) {DropHandshakes:470(1122ns) {DropIncomingHandshake:442(165ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(404ns) PrepareNewSessionHandshake:477(1487ns) {SwitchToState:179(47ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.376987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750067:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.377018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241299744750068:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8221
2025-12-24T02:11:15.375313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118242:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.375401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213076:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.375416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241299059213076:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.377605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750048:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.376163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241299059213076:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.377617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241299744750048:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.376253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241299059213070:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.377987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299744750037:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.376269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.376278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241299059213070:2110] [node 5] ICP22 created new session: [1:7587241299059213097:2110]
2025-12-24T02:11:15.376286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241299059213097:2110] [node 5] ICS09 handshake done sender: [1:7587241299059213076:2114] self: [1:7587241299059213072:0] peer: [5:7587241302098280382:0] socket: 546 qp: -1
2025-12-24T02:11:15.376292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241299059213097:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:15.376304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.378509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241300321543489:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55111
2025-12-24T02:11:15.375392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241302098280373:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.375422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241302098280367:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.377324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241302098280297:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18391) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241302098280359:2109] held: no
2025-12-24T02:11:15.378797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241299744749977:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.378816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.378822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299744749977:2049] [node 2] ICP22 created new session: [3:7587241299744750072:2049]
2025-12-24T02:11:15.378829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299744750072:2049] [node 2] ICS09 handshake done sender: [3:7587241299744750037:2108] self: [3:7587241299744750058:0] peer: [2:7587241300321543455:0] socket: 507 qp: -1
2025-12-24T02:11:15.378836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299744750072:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.378843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.379085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299744750048:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.380229Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241302098280296:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241302098280376:2118] held: no
2025-12-24T02:11:15.380646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750067:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.380661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241299744750041:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.380671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.380679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299744750041:2110] [node 5] ICP22 created new session: [3:7587241299744750074:2110]
2025-12-24T02:11:15.380685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299744750074:2110] [node 5] ICS09 handshake done sender: [3:7587241299744750048:2113] self: [3:7587241299744750047:0] peer: [5:7587241302098280383:0] socket: 524 qp: -1
2025-12-24T02:11:15.380691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299744750074:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:15.380698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.380834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241299744750041:2110] [node 5] ICP18 handshake (actor [3:7587241299744750067:2119]) is too old
2025-12-24T02:11:15.379202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118239:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.379236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301410118154:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.380499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118244:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.382430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241299744750055:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.382441Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241299744750055:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.383067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241299744750055:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.383258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118244:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.383401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241299744750040:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.383422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.383429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241299744750040:2109] [node 4] ICP22 created new session: [3:7587241299744750077:2109]
2025-12-24T02:11:15.383434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241299744750077:2109] [node 4] ICS09 handshake done sender: [3:7587241299744750055:2115] self: [3:7587241299744750046:0] peer: [4:7587241301410118243:0] socket: 551 qp: -1
2025-12-24T02:11:15.383441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241299744750077:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:15.383448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.383584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028287:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.383653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028284:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.383716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028285:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.383765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028286:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.383880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241302098280298:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7456) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241302098280372:2115] held: no
2025-12-24T02:11:15.383963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.384359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280387:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.384503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280388:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.384620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302098280359:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.384693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302098280376:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.384748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302098280372:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.384800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302098280297:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.384814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.384820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302098280297:2049] [node 2] ICP22 created new session: [5:7587241302098280389:2049]
2025-12-24T02:11:15.384825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302098280389:2049] [node 2] ICS09 handshake done sender: [5:7587241302098280359:2109] self: [5:7587241302098280381:0] peer: [2:7587241300321543466:0] socket: 514 qp: -1
2025-12-24T02:11:15.384832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302098280389:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.384838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.384966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302098280296:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.384972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.384977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302098280296:2048] [node 1] ICP22 created new session: [5:7587241302098280391:2048]
2025-12-24T02:11:15.384981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302098280391:2048] [node 1] ICS09 handshake done sender: [5:7587241302098280376:2118] self: [5:7587241302098280382:0] peer: [1:7587241299059213072:0] socket: 549 qp: -1
2025-12-24T02:11:15.384985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302098280391:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.384990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.385064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302098280298:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.385070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.385074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302098280298:2050] [node 3] ICP22 created new session: [5:7587241302098280393:2050]
2025-12-24T02:11:15.385079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302098280393:2050] [node 3] ICS09 handshake done sender: [5:7587241302098280372:2115] self: [5:7587241302098280383:0] peer: [3:7587241299744750047:0] socket: 541 qp: -1
2025-12-24T02:11:15.385084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302098280393:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.385089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.386836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118227:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.386856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241301410118227:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.387661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302355028287:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.387703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302355028284:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.387728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302355028285:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.387749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302355028271:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.389252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302355028203:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.389275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302355028202:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.389288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302355028276:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.390965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302355028204:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.390986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.390995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302355028204:2050] [node 3] ICP22 created new session: [6:7587241302355028291:2050]
2025-12-24T02:11:15.391002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302355028291:2050] [node 3] ICS09 handshake done sender: [6:7587241302355028271:2110] self: [6:7587241302355028272:0] peer: [3:7587241299744750050:0] socket: 526 qp: -1
2025-12-24T02:11:15.391012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302355028291:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.391040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.390937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241299059213074:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.390963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241299059213074:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.392379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241301410118154:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7456) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241301410118239:2119] held: no
2025-12-24T02:11:15.392507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118230:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.392510Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241301410118230:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.393762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241302098280395:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55307
2025-12-24T02:11:15.394681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241299059213074:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.394850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241299059213069:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.394873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.394882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241299059213069:2111] [node 6] ICP22 created new session: [1:7587241299059213100:2111]
2025-12-24T02:11:15.394889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241299059213100:2111] [node 6] ICS09 handshake done sender: [1:7587241299059213074:2113] self: [1:7587241299059213073:0] peer: [6:7587241302355028289:0] socket: 522 qp: -1
2025-12-24T02:11:15.394896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241299059213100:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:15.394906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.394903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301410118227:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.395541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301410118230:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.395699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241302098280373:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.395811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241302098280367:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.395827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.395834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241302098280367:2112] [node 4] ICP22 created new session: [5:7587241302098280396:2112]
2025-12-24T02:11:15.395841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241302098280396:2112] [node 4] ICS09 handshake done sender: [5:7587241302098280373:2116] self: [5:7587241302098280384:0] peer: [4:7587241301410118225:0] socket: 542 qp: -1
2025-12-24T02:11:15.395848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241302098280396:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:15.395883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.402975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241300321543469:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.403004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241300321543469:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.407777Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241302355028202:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241302355028284:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.409315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301410118242:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.409347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241301410118224:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.409361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.409371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301410118224:2112] [node 6] ICP22 created new session: [4:7587241301410118249:2112]
2025-12-24T02:11:15.409380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301410118249:2112] [node 6] ICS09 handshake done sender: [4:7587241301410118230:2115] self: [4:7587241301410118226:0] peer: [6:7587241302355028290:0] socket: 517 qp: -1
2025-12-24T02:11:15.409389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301410118249:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.409398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.409540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241301410118223:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.409551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.409556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301410118223:2111] [node 5] ICP22 created new session: [4:7587241301410118251:2111]
2025-12-24T02:11:15.409560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301410118251:2111] [node 5] ICS09 handshake done sender: [4:7587241301410118227:2114] self: [4:7587241301410118225:0] peer: [5:7587241302098280384:0] socket: 530 qp: -1
2025-12-24T02:11:15.409565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301410118251:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:15.409569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.409746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301410118248:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60684
2025-12-24T02:11:15.410266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301410118239:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.410409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241301410118224:2112] [node 6] ICP18 handshake (actor [4:7587241301410118242:2120]) is too old
2025-12-24T02:11:15.410594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301410118154:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.410602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.410608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301410118154:2050] [node 3] ICP22 created new session: [4:7587241301410118253:2050]
2025-12-24T02:11:15.410613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301410118253:2050] [node 3] ICS09 handshake done sender: [4:7587241301410118239:2119] self: [4:7587241301410118243:0] peer: [3:7587241299744750046:0] socket: 528 qp: -1
2025-12-24T02:11:15.410619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301410118253:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.410637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.412532Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241302355028276:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241302355028285:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.415645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028293:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.415721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028294:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.415754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302355028295:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60763
2025-12-24T02:11:15.416008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302355028285:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.416107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302355028276:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.416124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.416130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302355028276:2111] [node 4] ICP22 created new session: [6:7587241302355028296:2111]
2025-12-24T02:11:15.416138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302355028296:2111] [node 4] ICS09 handshake done sender: [6:7587241302355028285:2116] self: [6:7587241302355028290:0] peer: [4:7587241301410118226:0] socket: 565 qp: -1
2025-12-24T02:11:15.416146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302355028296:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:15.416156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.416593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302355028284:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.416689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302355028202:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.416704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.416711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302355028202:2048] [node 1] ICP22 created new session: [6:7587241302355028298:2048]
2025-12-24T02:11:15.416716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302355028298:2048] [node 1] ICS09 handshake done sender: [6:7587241302355028284:2115] self: [6:7587241302355028289:0] peer: [1:7587241299059213073:0] socket: 536 qp: -1
2025-12-24T02:11:15.416723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302355028298:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.416729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.418853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241300321543469:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.419281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241300321543465:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.419303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.419311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241300321543465:2112] [node 6] ICP22 created new session: [2:7587241300321543491:2112]
2025-12-24T02:11:15.419317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241300321543491:2112] [node 6] ICS09 handshake done sender: [2:7587241300321543469:2114] self: [2:7587241300321543468:0] peer: [6:7587241302355028288:0] socket: 558 qp: -1
2025-12-24T02:11:15.419324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241300321543491:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.419332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.419133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302355028287:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.419742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302355028203:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.419755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.419760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302355028203:2049] [node 2] ICP22 created new session: [6:7587241302355028300:2049]
2025-12-24T02:11:15.419766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302355028300:2049] [node 2] ICS09 handshake done sender: [6:7587241302355028287:2118] self: [6:7587241302355028288:0] peer: [2:7587241300321543468:0] socket: 567 qp: -1
2025-12-24T02:11:15.419771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302355028300:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.419778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.430220Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:15.430313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241300321543478:2048] [node 1] ICS01 socket: 499 reason# EndOfStream
2025-12-24T02:11:15.430322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241300321543392:2048] [node 1] ICP30 unregister session Session# [2:7587241300321543478:2048] VirtualId# [2:7587241300321543472:0]
2025-12-24T02:11:15.430329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241300321543478:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.430374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.430397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.430831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:15.430896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241299744750065:2048] [node 1] ICS01 socket: 506 reason# EndOfStream
2025-12-24T02:11:15.430902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241299744749976:2048] [node 1] ICP30 unregister session Session# [3:7587241299744750065:2048] VirtualId# [3:7587241299744750056:0]
2025-12-24T02:11:15.430906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241299744750065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.430935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.430953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.433322Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:15.433397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241301410118233:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-12-24T02:11:15.433404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241301410118152:2048] [node 1] ICP30 unregister session Session# [4:7587241301410118233:2048] VirtualId# [4:7587241301410118231:0]
2025-12-24T02:11:15.433409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241301410118233:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.433458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.433482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.433512Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:15.433573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241302098280391:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-12-24T02:11:15.433577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241302098280296:2048] [node 1] ICP30 unregister session Session# [5:7587241302098280391:2048] VirtualId# [5:7587241302098280382:0]
2025-12-24T02:11:15.433581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241302098280391:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.433609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.433630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.433657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:15.433716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302355028298:2048] [node 1] ICS01 socket: 536 reason# EndOfStream
2025-12-24T02:11:15.433720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302355028202:2048] [node 1] ICP30 unregister session Session# [6:7587241302355028298:2048] VirtualId# [6:7587241302355028289:0]
2025-12-24T02:11:15.433724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302355028298:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.433753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.433772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.439035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:15.439120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241299744750072:2049] [node 2] ICS01 socket: 507 reason# EndOfStream
2025-12-24T02:11:15.439128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241299744749977:2049] [node 2] ICP30 unregister session Session# [3:7587241299744750072:2049] VirtualId# [3:7587241299744750058:0]
2025-12-24T02:11:15.439135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241299744750072:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.439179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.439201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.458391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:15.458485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241302098280389:2049] [node 2] ICS01 socket: 514 reason# EndOfStream
2025-12-24T02:11:15.458494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241302098280297:2049] [node 2] ICP30 unregister session Session# [5:7587241302098280389:2049] VirtualId# [5:7587241302098280381:0]
2025-12-24T02:11:15.458500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241302098280389:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.458558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.458583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.458623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:15.458684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302355028300:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-12-24T02:11:15.458688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302355028203:2049] [node 2] ICP30 unregister session Session# [6:7587241302355028300:2049] VirtualId# [6:7587241302355028288:0]
2025-12-24T02:11:15.458703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302355028300:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.458740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.458764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.458792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:15.458849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241301410118240:2049] [node 2] ICS01 socket: 512 reason# EndOfStream
2025-12-24T02:11:15.458853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241301410118153:2049] [node 2] ICP30 unregister session Session# [4:7587241301410118240:2049] VirtualId# [4:7587241301410118236:0]
2025-12-24T02:11:15.458857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241301410118240:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.458884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.458904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.465404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:15.465499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302355028291:2050] [node 3] ICS01 socket: 526 reason# EndOfStream
2025-12-24T02:11:15.465508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302355028204:2050] [node 3] ICP30 unregister session Session# [6:7587241302355028291:2050] VirtualId# [6:7587241302355028272:0]
2025-12-24T02:11:15.465515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302355028291:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.465575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.465600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.465693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:15.465761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241302098280393:2050] [node 3] ICS01 socket: 541 reason# EndOfStream
2025-12-24T02:11:15.465766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241302098280298:2050] [node 3] ICP30 unregister session Session# [5:7587241302098280393:2050] VirtualId# [5:7587241302098280383:0]
2025-12-24T02:11:15.465771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241302098280393:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.465799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.465824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.489838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:15.489938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302355028296:2111] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-12-24T02:11:15.489947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302355028276:2111] [node 4] ICP30 unregister session Session# [6:7587241302355028296:2111] VirtualId# [6:7587241302355028290:0]
2025-12-24T02:11:15.489954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302355028296:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.490027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:15.490058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:15.545517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241300415390474:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:15.545526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241300415390475:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.563349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241301474303451:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.563364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241301474303452:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.565179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241298721456289:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.565186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241298721456290:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.574377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301150521023:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.574398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301150521024:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.574403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301150521025:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.584786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241298492750785:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.584823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241298492750786:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.584835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241298492750787:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.592054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298127215260:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.592081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298127215261:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.592086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298127215262:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.612695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298127215322:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:15.614610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241300415390542:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.615139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303512:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.615016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241300415390539:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:15.615430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303513:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
2025-12-24T02:11:15.616999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241300415390537:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:15.618909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241301474303515:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:15.619601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241301474303521:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:15.627714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241298492750852:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.632292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215329:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.631042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241300415390475:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17873859ns) {Configure:104(17867826ns) {ProcessConfigured:135(17574211ns) {StartInitialHandshake:157(17573621ns) {DropHandshakes:470(396ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:477(1507ns) {SwitchToState:179(1092ns)}}}}}
2025-12-24T02:11:15.631154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241301474303525:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:15.632665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.635975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215336:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.637551Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241301150521023:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16227476ns) {Configure:104(16226387ns) {ProcessConfigured:135(15793614ns) {StartInitialHandshake:157(15793219ns) {DropHandshakes:470(487ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(742ns) {SwitchToState:179(260ns)}}}}}
2025-12-24T02:11:15.637701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301150521090:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:15.638806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241301150521024:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16715595ns) {Configure:104(16714620ns) {ProcessConfigured:135(16353188ns) {StartInitialHandshake:157(16352781ns) {DropHandshakes:470(594ns) {DropIncomingHandshake:442(104ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(866ns) {SwitchToState:179(476ns)}}}}}
2025-12-24T02:11:15.638666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241300415390553:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34905
2025-12-24T02:11:15.640122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303530:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.644275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750855:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.644348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750856:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.651034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241301150521088:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:15.650892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241298721456354:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:15.650917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241298721456355:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:15.651045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241298721456353:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:15.651417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241298127215328:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:15.651728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241298492750853:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:15.651819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750855:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.659014Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241298721456289:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38574838ns) {Configure:104(38573367ns) {ProcessConfigured:135(38206555ns) {StartInitialHandshake:157(38205774ns) {DropHandshakes:470(497ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(1943ns) {SwitchToState:179(1087ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.660073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215338:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.660707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.660867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456368:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
2025-12-24T02:11:15.660900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456366:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
2025-12-24T02:11:15.660921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456367:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.661139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241301474303451:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27517313ns) {Configure:104(27516660ns) {ProcessConfigured:135(27133559ns) {StartInitialHandshake:157(27133378ns) {DropHandshakes:470(587ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(797ns) {SwitchToState:179(742ns)}}}}}
2025-12-24T02:11:15.661199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303512:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.663017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456367:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.662910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241300415390554:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34905
2025-12-24T02:11:15.664604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521100:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.661053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241298127215260:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37409484ns) {Configure:104(37408626ns) {ProcessConfigured:135(37067929ns) {StartInitialHandshake:157(37067381ns) {DropHandshakes:470(352ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(736ns) {SwitchToState:179(501ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.670565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456368:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.667298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390553:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.667720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241298492750787:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44782966ns) {Configure:104(44782032ns) {ProcessConfigured:135(44426638ns) {StartInitialHandshake:157(44426342ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(601ns) {SwitchToState:179(246ns)}}}}}
2025-12-24T02:11:15.670319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215341:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.672859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390554:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456366:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456360:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673285Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241298721456360:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.673311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241298721456290:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.673798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241298721456360:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.673938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241298721456355:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.673953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.673961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241298721456355:2108] [node 5] ICP22 created new session: [3:7587241298721456375:2108]
2025-12-24T02:11:15.673967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241298721456375:2108] [node 5] ICS09 handshake done sender: [3:7587241298721456360:2111] self: [3:7587241298721456359:0] peer: [5:7587241298492750866:0] socket: 535 qp: -1
2025-12-24T02:11:15.673973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241298721456375:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:15.673985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.673767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390541:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241300415390541:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.674210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390549:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.674215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241300415390549:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.671482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750861:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.671526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241298492750787:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.671897Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241298492750785:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44085743ns) {Configure:104(44084602ns) {ProcessConfigured:135(43755406ns) {StartInitialHandshake:157(43754832ns) {DropHandshakes:470(1266ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(982ns) {SwitchToState:179(492ns)}}}}}
2025-12-24T02:11:15.672037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521102:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.672083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521103:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.673118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521100:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521102:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301150521024:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.674246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521103:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.674318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301150521025:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.673414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298127215336:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298127215329:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298127215261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.673547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298127215260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.674454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298127215338:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.674482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298127215328:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.674937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241300415390541:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.675765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456362:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.675781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241298721456362:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.675935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241300415390549:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.676067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241300415390474:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.676080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.676088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241300415390474:2048] [node 2] ICP22 created new session: [1:7587241300415390559:2048]
2025-12-24T02:11:15.676094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241300415390559:2048] [node 2] ICS09 handshake done sender: [1:7587241300415390541:2110] self: [1:7587241300415390535:0] peer: [2:7587241301474303535:0] socket: 510 qp: -1
2025-12-24T02:11:15.676100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241300415390559:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:15.676126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.676227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241300415390542:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.676234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.676240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241300415390542:2109] [node 6] ICP22 created new session: [1:7587241300415390561:2109]
2025-12-24T02:11:15.676243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241300415390561:2109] [node 6] ICS09 handshake done sender: [1:7587241300415390549:2112] self: [1:7587241300415390543:0] peer: [6:7587241298127215343:0] socket: 514 qp: -1
2025-12-24T02:11:15.676246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241300415390561:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:15.676258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.678141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
2025-12-24T02:11:15.677251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521094:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.677272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241301150521094:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.677969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521107:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.677987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521108:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.679164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301150521094:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.679479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241301150521090:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.679492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.679497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301150521090:2109] [node 6] ICP22 created new session: [4:7587241301150521110:2109]
2025-12-24T02:11:15.679503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301150521110:2109] [node 6] ICS09 handshake done sender: [4:7587241301150521094:2112] self: [4:7587241301150521093:0] peer: [6:7587241298127215344:0] socket: 528 qp: -1
2025-12-24T02:11:15.679508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301150521110:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:15.679526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.679644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301150521100:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.679874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301150521024:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.679881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.679885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301150521024:2049] [node 2] ICP22 created new session: [4:7587241301150521113:2049]
2025-12-24T02:11:15.679889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301150521113:2049] [node 2] ICS09 handshake done sender: [4:7587241301150521100:2114] self: [4:7587241301150521104:0] peer: [2:7587241301474303517:0] socket: 517 qp: -1
2025-12-24T02:11:15.679893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301150521113:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.679897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.673121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303513:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.673155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241301474303451:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.674792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303518:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.674798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241301474303518:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.676292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303527:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.676297Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241301474303527:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.677071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
2025-12-24T02:11:15.677113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303537:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
2025-12-24T02:11:15.678379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241301474303518:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.678497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241301474303527:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.678561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241301474303515:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.678572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.678581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241301474303515:2107] [node 4] ICP22 created new session: [2:7587241301474303540:2107]
2025-12-24T02:11:15.678588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241301474303540:2107] [node 4] ICS09 handshake done sender: [2:7587241301474303518:2110] self: [2:7587241301474303517:0] peer: [4:7587241301150521104:0] socket: 515 qp: -1
2025-12-24T02:11:15.678595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241301474303540:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:15.678606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.678749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303524:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.678758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241301474303525:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.678766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241301474303524:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.678770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.678775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241301474303525:2112] [node 6] ICP22 created new session: [2:7587241301474303542:2112]
2025-12-24T02:11:15.678780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241301474303542:2112] [node 6] ICS09 handshake done sender: [2:7587241301474303527:2114] self: [2:7587241301474303526:0] peer: [6:7587241298127215342:0] socket: 520 qp: -1
2025-12-24T02:11:15.678784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241301474303542:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:15.678788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: 2025-12-24T02:11:15.681100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750856:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.681278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241298492750785:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-12-24T02:11:15.681384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.681413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750868:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.680872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241301474303513:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.683457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298127215341:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.683511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298127215322:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.683704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241298721456362:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.683390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390546:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.683405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241300415390546:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.683993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241298721456353:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.684011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.684017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241298721456353:2107] [node 4] ICP22 created new session: [3:7587241298721456380:2107]
2025-12-24T02:11:15.684024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241298721456380:2107] [node 4] ICS09 handshake done sender: [3:7587241298721456362:2112] self: [3:7587241298721456361:0] peer: [4:7587241301150521106:0] socket: 536 qp: -1
2025-12-24T02:11:15.684031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241298721456380:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:15.684051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.684668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241298492750861:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.684000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521116:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.685826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301150521103:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.685882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301150521025:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.685901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.685909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301150521025:2050] [node 3] ICP22 created new session: [4:7587241301150521117:2050]
2025-12-24T02:11:15.685920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301150521117:2050] [node 3] ICS09 handshake done sender: [4:7587241301150521103:2117] self: [4:7587241301150521106:0] peer: [3:7587241298721456361:0] socket: 545 qp: -1
2025-12-24T02:11:15.685927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301150521117:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.685933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.684570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241301474303451:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.684586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.684592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241301474303451:2048] [node 1] ICP22 created new session: [2:7587241301474303548:2048]
2025-12-24T02:11:15.684596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241301474303548:2048] [node 1] ICS09 handshake done sender: [2:7587241301474303513:2109] self: [2:7587241301474303535:0] peer: [1:7587241300415390535:0] socket: 511 qp: -1
2025-12-24T02:11:15.684600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241301474303548:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.684605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.684740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303536:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.685644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241301474303547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59889
2025-12-24T02:11:15.687132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750862:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.687145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241298492750862:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.687658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750871:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.688071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241298492750862:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.688142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241298492750853:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.688157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.688163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241298492750853:2110] [node 6] ICP22 created new session: [5:7587241298492750873:2110]
2025-12-24T02:11:15.688169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241298492750873:2110] [node 6] ICS09 handshake done sender: [5:7587241298492750862:2115] self: [5:7587241298492750860:0] peer: [6:7587241298127215345:0] socket: 543 qp: -1
2025-12-24T02:11:15.688175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241298492750873:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:15.688204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.688875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241301474303524:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.688423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456378:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.688459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241298721456289:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.690987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241298127215261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61929) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241298127215336:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.695059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750869:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.695100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241298492750852:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.695174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241298492750787:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.695186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.695191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241298492750787:2050] [node 3] ICP22 created new session: [5:7587241298492750876:2050]
2025-12-24T02:11:15.695197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241298492750876:2050] [node 3] ICS09 handshake done sender: [5:7587241298492750861:2114] self: [5:7587241298492750866:0] peer: [3:7587241298721456359:0] socket: 537 qp: -1
2025-12-24T02:11:15.695203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241298492750876:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.695209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.695633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750878:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.697227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241298492750856:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.695912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241300415390546:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.696168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390552:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.696173Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241300415390552:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.696679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241300415390539:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.696690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.696697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241300415390539:2108] [node 5] ICP22 created new session: [1:7587241300415390566:2108]
2025-12-24T02:11:15.696701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241300415390566:2108] [node 5] ICS09 handshake done sender: [1:7587241300415390546:2111] self: [1:7587241300415390545:0] peer: [5:7587241298492750870:0] socket: 512 qp: -1
2025-12-24T02:11:15.696707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241300415390566:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:15.696715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.697076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241300415390552:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.697328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241300415390475:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.697341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.697346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241300415390475:2049] [node 3] ICP22 created new session: [1:7587241300415390569:2049]
2025-12-24T02:11:15.697351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241300415390569:2049] [node 3] ICS09 handshake done sender: [1:7587241300415390552:2114] self: [1:7587241300415390540:0] peer: [3:7587241298721456383:0] socket: 552 qp: -1
2025-12-24T02:11:15.697357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241300415390569:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:15.697363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.699179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521107:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.699219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241301150521023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.699020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241300415390572:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34905
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.698761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241298492750785:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.698781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.698789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241298492750785:2048] [node 1] ICP22 created new session: [5:7587241298492750881:2048]
2025-12-24T02:11:15.698795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241298492750881:2048] [node 1] ICS09 handshake done sender: [5:7587241298492750856:2112] self: [5:7587241298492750870:0] peer: [1:7587241300415390545:0] socket: 531 qp: -1
2025-12-24T02:11:15.698800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241298492750881:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.698807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.699724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215349:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
2025-12-24T02:11:15.699788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215350:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
2025-12-24T02:11:15.700262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241300415390551:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.700277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241300415390551:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.700393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241300415390539:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:15.701355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241298721456289:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241298721456378:2118] held: no
2025-12-24T02:11:15.701227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241300415390571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34905
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.702999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241301150521101:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.703018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241301150521101:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.702821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241298127215349:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.702892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215346:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
2025-12-24T02:11:15.706932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241300415390551:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.707267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241300415390537:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.707299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.707311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241300415390537:2107] [node 4] ICP22 created new session: [1:7587241300415390574:2107]
2025-12-24T02:11:15.707318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241300415390574:2107] [node 4] ICS09 handshake done sender: [1:7587241300415390551:2113] self: [1:7587241300415390548:0] peer: [4:7587241301150521119:0] socket: 551 qp: -1
2025-12-24T02:11:15.707326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241300415390574:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:15.707337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.710856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241301150521023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241301150521107:2118] held: no
2025-12-24T02:11:15.718868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241298492750871:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.718931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241298492750786:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.718736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215347:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.722857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521120:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.723878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301150521107:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.724714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241301150521023:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.724734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.724740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301150521023:2048] [node 1] ICP22 created new session: [4:7587241301150521121:2048]
2025-12-24T02:11:15.724746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301150521121:2048] [node 1] ICS09 handshake done sender: [4:7587241301150521107:2118] self: [4:7587241301150521119:0] peer: [1:7587241300415390548:0] socket: 557 qp: -1
2025-12-24T02:11:15.724752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301150521121:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.724758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.723033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456384:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
2025-12-24T02:11:15.723314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241298721456385:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26971
2025-12-24T02:11:15.723438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241298721456366:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.723471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241298721456378:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.723515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241298721456289:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.723536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.723545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241298721456289:2048] [node 1] ICP22 created new session: [3:7587241298721456386:2048]
2025-12-24T02:11:15.723554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241298721456386:2048] [node 1] ICS09 handshake done sender: [3:7587241298721456378:2118] self: [3:7587241298721456383:0] peer: [1:7587241300415390540:0] socket: 560 qp: -1
2025-12-24T02:11:15.723563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241298721456386:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.723577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.723968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241298721456290:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.723978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.723984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241298721456290:2049] [node 2] ICP22 created new session: [3:7587241298721456388:2049]
2025-12-24T02:11:15.723990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241298721456388:2049] [node 2] ICS09 handshake done sender: [3:7587241298721456366:2113] self: [3:7587241298721456373:0] peer: [2:7587241301474303516:0] socket: 533 qp: -1
2025-12-24T02:11:15.723996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241298721456388:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.724015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.727299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241301474303531:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.727323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241301474303531:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.729753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241301150521123:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54132
2025-12-24T02:11:15.727124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298127215329:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:15.728127Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241298127215322:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13396) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241298127215341:2118] held: no
2025-12-24T02:11:15.728168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298127215260:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.728180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.728187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298127215260:2048] [node 1] ICP22 created new session: [6:7587241298127215351:2048]
2025-12-24T02:11:15.728194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298127215351:2048] [node 1] ICS09 handshake done sender: [6:7587241298127215329:2111] self: [6:7587241298127215343:0] peer: [1:7587241300415390543:0] socket: 516 qp: -1
2025-12-24T02:11:15.728203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298127215351:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.728210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.728472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
2025-12-24T02:11:15.728940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298127215336:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.729060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298127215261:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.729071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.729077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298127215261:2049] [node 2] ICP22 created new session: [6:7587241298127215353:2049]
2025-12-24T02:11:15.729084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298127215353:2049] [node 2] ICS09 handshake done sender: [6:7587241298127215336:2114] self: [6:7587241298127215342:0] peer: [2:7587241301474303526:0] socket: 524 qp: -1
2025-12-24T02:11:15.729090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298127215353:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.729097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.729296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298127215341:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.729589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298127215338:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.729933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241298127215262:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.729959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298127215328:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.729969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.729973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298127215328:2110] [node 4] ICP22 created new session: [6:7587241298127215358:2110]
2025-12-24T02:11:15.729980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298127215358:2110] [node 4] ICS09 handshake done sender: [6:7587241298127215338:2115] self: [6:7587241298127215344:0] peer: [4:7587241301150521093:0] socket: 529 qp: -1
2025-12-24T02:11:15.729987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298127215358:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:15.729994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.730356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241298721456370:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.730369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241298721456370:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.731307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241298721456370:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:15.731654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241298721456354:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.731678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:15.731687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241298721456354:2109] [node 6] ICP22 created new session: [3:7587241298721456391:2109]
2025-12-24T02:11:15.731692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241298721456391:2109] [node 6] ICS09 handshake done sender: [3:7587241298721456370:2117] self: [3:7587241298721456357:0] peer: [6:7587241298127215357:0] socket: 566 qp: -1
2025-12-24T02:11:15.731697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241298721456391:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:15.731703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:15.732084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241301474303531:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:15.732356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241301474303521:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.732390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.732398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241301474303521:2111] [node 5] ICP22 created new session: [2:7587241301474303552:2111]
2025-12-24T02:11:15.732405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241301474303552:2111] [node 5] ICS09 handshake done sender: [2:7587241301474303531:2116] self: [2:7587241301474303522:0] peer: [5:7587241298492750883:0] socket: 569 qp: -1
2025-12-24T02:11:15.732409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241301474303552:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:15.732415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.733321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298127215322:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.733335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.733341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298127215322:2108] [node 5] ICP22 created new session: [6:7587241298127215360:2108]
2025-12-24T02:11:15.733347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298127215360:2108] [node 5] ICS09 handshake done sender: [6:7587241298127215341:2118] self: [6:7587241298127215345:0] peer: [5:7587241298492750860:0] socket: 544 qp: -1
2025-12-24T02:11:15.733352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298127215360:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:15.733357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.734240Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241298492750786:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61929) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241298492750871:2120] held: no
2025-12-24T02:11:15.734448Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241298127215262:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23401) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241298127215349:2122] held: no
2025-12-24T02:11:15.735003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241301150521101:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.735455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750885:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.735650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241298492750884:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21556
2025-12-24T02:11:15.734882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241301474303452:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.734915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.734925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241301474303452:2049] [node 3] ICP22 created new session: [2:7587241301474303554:2049]
2025-12-24T02:11:15.734934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241301474303554:2049] [node 3] ICS09 handshake done sender: [2:7587241301474303524:2113] self: [2:7587241301474303516:0] peer: [3:7587241298721456373:0] socket: 519 qp: -1
2025-12-24T02:11:15.734940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241301474303554:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:15.734978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.735835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241298492750871:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:15.735991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241298492750869:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.736079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241298492750786:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.736091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:15.736097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241298492750786:2049] [node 2] ICP22 created new session: [5:7587241298492750886:2049]
2025-12-24T02:11:15.736103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241298492750886:2049] [node 2] ICS09 handshake done sender: [5:7587241298492750871:2120] self: [5:7587241298492750883:0] peer: [2:7587241301474303522:0] socket: 523 qp: -1
2025-12-24T02:11:15.736109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241298492750886:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:15.736116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:15.736782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241298492750852:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.736797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.736804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241298492750852:2109] [node 4] ICP22 created new session: [5:7587241298492750888:2109]
2025-12-24T02:11:15.736810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241298492750888:2109] [node 4] ICS09 handshake done sender: [5:7587241298492750869:2119] self: [5:7587241298492750875:0] peer: [4:7587241301150521099:0] socket: 563 qp: -1
2025-12-24T02:11:15.736817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241298492750888:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:15.736824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.738864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241301150521088:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.738895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:15.738904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241301150521088:2108] [node 5] ICP22 created new session: [4:7587241301150521125:2108]
2025-12-24T02:11:15.738910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241301150521125:2108] [node 5] ICS09 handshake done sender: [4:7587241301150521101:2115] self: [4:7587241301150521099:0] peer: [5:7587241298492750875:0] socket: 561 qp: -1
2025-12-24T02:11:15.738916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241301150521125:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:15.738925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:15.739696Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241301474303452:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(44860217ns) {HandleHandshakeStatus:329(44859381ns) {DropIncomingHandshake:442(9126ns) DropOutgoingHandshake:457(149ns) SwitchToState:179(114ns) ProcessPendingSessionEvents:495(30546ns) {ForwardSessionEventToSession:612(20355ns) ForwardSessionEventToSession:612(994ns) ForwardSessionEventToSession:612(816ns) ForwardSessionEventToSession:612(793ns) ForwardSessionEventToSession:612(795ns) ForwardSessionEventToSession:612(1108ns) ForwardSessionEventToSession:612(571ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(635ns) ForwardSessionEventToSession:612(1068ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.743291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241298127215362:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6388
2025-12-24T02:11:15.743651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241298127215349:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:15.744054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241298127215262:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.744088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:15.744096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241298127215262:2050] [node 3] ICP22 created new session: [6:7587241298127215363:2050]
2025-12-24T02:11:15.744105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241298127215363:2050] [node 3] ICS09 handshake done sender: [6:7587241298127215349:2122] self: [6:7587241298127215357:0] peer: [3:7587241298721456357:0] socket: 577 qp: -1
2025-12-24T02:11:15.744113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241298127215363:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:15.744124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:15.755207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:15.755314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241301474303548:2048] [node 1] ICS01 socket: 511 reason# EndOfStream
2025-12-24T02:11:15.755325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241301474303451:2048] [node 1] ICP30 unregister session Session# [2:7587241301474303548:2048] VirtualId# [2:7587241301474303535:0]
2025-12-24T02:11:15.755333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241301474303548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.755412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.755436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.756009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:15.756089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241298721456386:2048] [node 1] ICS01 socket: 560 reason# EndOfStream
2025-12-24T02:11:15.756097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241298721456289:2048] [node 1] ICP30 unregister session Session# [3:7587241298721456386:2048] VirtualId# [3:7587241298721456383:0]
2025-12-24T02:11:15.756105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241298721456386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.756142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.756162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.763145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:15.763242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298127215351:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:15.763251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241298127215260:2048] [node 1] ICP30 unregister session Session# [6:7587241298127215351:2048] VirtualId# [6:7587241298127215343:0]
2025-12-24T02:11:15.763259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298127215351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.763316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.763342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.766778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:15.766891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241301150521121:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:15.766903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241301150521023:2048] [node 1] ICP30 unregister session Session# [4:7587241301150521121:2048] VirtualId# [4:7587241301150521119:0]
2025-12-24T02:11:15.766912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241301150521121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.766979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.767018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.767063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:15.767130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241298492750881:2048] [node 1] ICS01 socket: 531 reason# EndOfStream
2025-12-24T02:11:15.767134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241298492750785:2048] [node 1] ICP30 unregister session Session# [5:7587241298492750881:2048] VirtualId# [5:7587241298492750870:0]
2025-12-24T02:11:15.767139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241298492750881:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.767175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:15.767199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:15.774463Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:15.774578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241298721456388:2049] [node 2] ICS01 socket: 533 reason# EndOfStream
2025-12-24T02:11:15.774589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241298721456290:2049] [node 2] ICP30 unregister session Session# [3:7587241298721456388:2049] VirtualId# [3:7587241298721456373:0]
2025-12-24T02:11:15.774597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241298721456388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.774646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.774671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.778181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:15.778280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241298492750886:2049] [node 2] ICS01 socket: 523 reason# EndOfStream
2025-12-24T02:11:15.778289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241298492750786:2049] [node 2] ICP30 unregister session Session# [5:7587241298492750886:2049] VirtualId# [5:7587241298492750883:0]
2025-12-24T02:11:15.778298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241298492750886:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.778355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.778388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.778478Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:15.778549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241301150521113:2049] [node 2] ICS01 socket: 517 reason# EndOfStream
2025-12-24T02:11:15.778555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241301150521024:2049] [node 2] ICP30 unregister session Session# [4:7587241301150521113:2049] VirtualId# [4:7587241301150521104:0]
2025-12-24T02:11:15.778559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241301150521113:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.778593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.778615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.780469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:15.780551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298127215353:2049] [node 2] ICS01 socket: 524 reason# EndOfStream
2025-12-24T02:11:15.780559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241298127215261:2049] [node 2] ICP30 unregister session Session# [6:7587241298127215353:2049] VirtualId# [6:7587241298127215342:0]
2025-12-24T02:11:15.780565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298127215353:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.780622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:15.780646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:15.789716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:15.789818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241298492750876:2050] [node 3] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:15.789829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241298492750787:2050] [node 3] ICP30 unregister session Session# [5:7587241298492750876:2050] VirtualId# [5:7587241298492750866:0]
2025-12-24T02:11:15.789836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241298492750876:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.789892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.789919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.790915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:15.790987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298127215363:2050] [node 3] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:15.790993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241298127215262:2050] [node 3] ICP30 unregister session Session# [6:7587241298127215363:2050] VirtualId# [6:7587241298127215357:0]
2025-12-24T02:11:15.790998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298127215363:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.791042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:15.791069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:15.810803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:15.810897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241298127215358:2110] [node 4] ICS01 socket: 529 reason# EndOfStream
2025-12-24T02:11:15.810906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241298127215328:2110] [node 4] ICP30 unregister session Session# [6:7587241298127215358:2110] VirtualId# [6:7587241298127215344:0]
2025-12-24T02:11:15.810911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241298127215358:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:15.810962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:15.810989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:15.858327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241299814818177:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.858344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241299814818178:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.858898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241301808205675:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.858908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241301808205676:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.858673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241298334977748:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:15.858682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241298334977749:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:15.886889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241298588341082:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.886902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241298588341083:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.886907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241298588341084:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.896815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241300044470593:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.896825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241300044470594:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.896830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241300044470595:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.900589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241299510440721:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:15.900598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241299510440722:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:15.900603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241299510440723:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:15.952044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241298334977811:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:15.952959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241298334977812:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:15.953845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241299510440741:2065] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.954096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241299814818237:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.954862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241298588341143:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.954156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241300044470644:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:15.955498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241299510440743:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:15.958938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241300044470656:2093] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.959490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241301808205735:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:15.956372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241298334977824:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.963024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241299510440754:2074] [node 4] ICP01 ready to work
2025-12-24T02:11:15.963197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241298334977827:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65088
2025-12-24T02:11:15.964502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241299814818237:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.971092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241301808205735:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.972413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241299814818245:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:15.973080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241299814818249:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
2025-12-24T02:11:15.975054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241298588341143:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.975512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241298588341082:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.975754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977817:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.975766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241298334977817:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.976419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241298334977817:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:15.976599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241298334977811:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:15.976616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:15.976624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241298334977811:2107] [node 4] ICP22 created new session: [1:7587241298334977829:2107]
2025-12-24T02:11:15.976630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241298334977829:2107] [node 4] ICS09 handshake done sender: [1:7587241298334977817:2110] self: [1:7587241298334977816:0] peer: [4:7587241298588341144:0] socket: 511 qp: -1
2025-12-24T02:11:15.976637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241298334977829:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:15.976647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:15.977294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241301808205675:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.978818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977827:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.978947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977814:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.978958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241298334977814:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.978931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241299814818177:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.979131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241299814818251:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:15.981225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241299510440743:2068] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.987036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241299814818249:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.987183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241299510440723:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16352283ns) {Configure:104(16351049ns) {ProcessConfigured:135(16017572ns) {StartInitialHandshake:157(16016531ns) {DropHandshakes:470(864ns) {DropIncomingHandshake:442(138ns) DropOutgoingHandshake:457(125ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(967ns) {SwitchToState:179(346ns)}}}}}
2025-12-24T02:11:15.986895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241298334977833:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65088
2025-12-24T02:11:15.987896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977833:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.987637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241298588341147:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:15.987699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241298588341148:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:15.990900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241298588341147:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.990946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241298588341143:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.991060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241298588341082:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:15.991080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:15.991088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241298588341082:2048] [node 1] ICP22 created new session: [4:7587241298588341150:2048]
2025-12-24T02:11:15.991096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241298588341150:2048] [node 1] ICS09 handshake done sender: [4:7587241298588341143:2108] self: [4:7587241298588341144:0] peer: [1:7587241298334977816:0] socket: 514 qp: -1
2025-12-24T02:11:15.991105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241298588341150:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:15.991128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:15.991173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241298588341145:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:15.991009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241299510440798:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:15.991288Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241299510440741:2065] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37377559ns) {Configure:104(37376638ns) {ProcessConfigured:135(29233011ns) {StartInitialHandshake:157(29231887ns) {DropHandshakes:470(274ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(823ns) {SwitchToState:179(84ns)}}}}}
2025-12-24T02:11:15.991333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241299510440741:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.991511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241298588341145:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.993993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977820:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.994013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241298334977820:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.994191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241299510440798:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.994238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241299510440722:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.994939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241299814818243:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:15.998953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241298588341154:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:15.999341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241298588341154:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:15.998927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241300044470665:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:15.999629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241299510440789:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:15.999643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241299510440789:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:15.999993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241300044470665:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.001364Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241299814818178:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22524305ns) {Configure:104(22522765ns) {ProcessConfigured:135(22153808ns) {StartInitialHandshake:157(22153366ns) {DropHandshakes:470(661ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:477(1554ns) {SwitchToState:179(1010ns)}}}}}
2025-12-24T02:11:16.002798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241299814818258:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
2025-12-24T02:11:16.003304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241300044470646:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.003322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241300044470646:2098] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.002939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241299814818251:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23748024ns) {Configure:104(23747475ns) {ProcessConfigured:135(23448395ns) {StartInitialHandshake:157(23448119ns) {DropHandshakes:470(514ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1166ns) {SwitchToState:179(586ns)}}}}}
2025-12-24T02:11:16.006747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241299814818247:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.006765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241299814818247:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.006853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241298334977814:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.007125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241299814818237:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.009625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241300044470593:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18012017ns) {Configure:104(18010907ns) {ProcessConfigured:135(17667688ns) {StartInitialHandshake:157(17667233ns) {DropHandshakes:470(620ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(134ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1131ns) {SwitchToState:179(771ns)}}}}}
2025-12-24T02:11:16.009673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241300044470593:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.009261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241298588341083:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17668584ns) {Configure:104(17667462ns) {ProcessConfigured:135(17357330ns) {StartInitialHandshake:157(17356595ns) {DropHandshakes:470(470ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1957ns) {SwitchToState:179(1091ns)}}}}}
2025-12-24T02:11:16.010814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977822:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.010830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241298334977822:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.015046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241298334977748:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.015075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.015085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241298334977748:2048] [node 2] ICP22 created new session: [1:7587241302629945132:2048]
2025-12-24T02:11:16.015094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302629945132:2048] [node 2] ICS09 handshake done sender: [1:7587241298334977814:2109] self: [1:7587241298334977809:0] peer: [2:7587241299814818241:0] socket: 510 qp: -1
2025-12-24T02:11:16.015104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302629945132:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:16.015128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.019283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241298334977820:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.018812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241299814818243:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23783677ns) {Configure:104(23782952ns) {ProcessConfigured:135(23454483ns) {StartInitialHandshake:157(23453917ns) {DropHandshakes:470(574ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(143ns)} GenerateSessionVirtualId:988(162ns) PrepareNewSessionHandshake:477(1332ns) {SwitchToState:179(741ns)}}}}}
2025-12-24T02:11:16.018845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241299814818177:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.018864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.018873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241299814818177:2048] [node 1] ICP22 created new session: [2:7587241304109785557:2048]
2025-12-24T02:11:16.018878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241304109785557:2048] [node 1] ICS09 handshake done sender: [2:7587241299814818237:2107] self: [2:7587241299814818241:0] peer: [1:7587241298334977809:0] socket: 512 qp: -1
2025-12-24T02:11:16.018888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241304109785557:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.018910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.020304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302883308457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.020817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437967:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.020629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241304109785559:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
2025-12-24T02:11:16.022385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304339437967:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.023024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241299510440789:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.023239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241299510440754:2074] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.023255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.023263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241299510440754:2074] [node 4] ICP22 created new session: [6:7587241303805408101:2074]
2025-12-24T02:11:16.023270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303805408101:2074] [node 4] ICS09 handshake done sender: [6:7587241299510440789:2107] self: [6:7587241299510440756:0] peer: [4:7587241298588341155:0] socket: 528 qp: -1
2025-12-24T02:11:16.023278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303805408101:2074] [node 4] ICS10 traffic start
2025-12-24T02:11:16.023302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.023689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241298334977749:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.023709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.023716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241298334977749:2049] [node 3] ICP22 created new session: [1:7587241302629945136:2049]
2025-12-24T02:11:16.023722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302629945136:2049] [node 3] ICS09 handshake done sender: [1:7587241298334977820:2111] self: [1:7587241298334977813:0] peer: [3:7587241301808205736:0] socket: 513 qp: -1
2025-12-24T02:11:16.023727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302629945136:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.023734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.023121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241301808205742:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:16.023514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241301808205743:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:16.026838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241301808205741:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:16.027047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241300044470594:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24075134ns) {Configure:104(24073878ns) {ProcessConfigured:135(23731457ns) {StartInitialHandshake:157(23730193ns) {DropHandshakes:470(955ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(249ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1393ns) {SwitchToState:179(745ns)}}}}}
2025-12-24T02:11:16.027097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241300044470594:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.026804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241298588341152:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.028776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408104:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:16.031589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241299814818247:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.031719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241299814818245:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.031742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.031749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241299814818245:2110] [node 6] ICP22 created new session: [2:7587241304109785562:2110]
2025-12-24T02:11:16.031756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241304109785562:2110] [node 6] ICS09 handshake done sender: [2:7587241299814818247:2111] self: [2:7587241299814818246:0] peer: [6:7587241299510440802:0] socket: 530 qp: -1
2025-12-24T02:11:16.031764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241304109785562:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:16.031774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.028316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241298588341147:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.028420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241298588341154:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.028432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.028441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241298588341154:2113] [node 6] ICP22 created new session: [4:7587241302883308459:2113]
2025-12-24T02:11:16.028446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302883308459:2113] [node 6] ICS09 handshake done sender: [4:7587241298588341147:2111] self: [4:7587241298588341155:0] peer: [6:7587241299510440756:0] socket: 534 qp: -1
2025-12-24T02:11:16.028454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302883308459:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.028476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.030920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241300044470646:2098] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.034820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241300044470595:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27840584ns) {Configure:104(27839098ns) {ProcessConfigured:135(27475879ns) {StartInitialHandshake:157(27475114ns) {DropHandshakes:470(794ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(173ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(1381ns) {SwitchToState:179(834ns)}}}}}
2025-12-24T02:11:16.034827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241300044470644:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.034842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.034850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241300044470644:2096] [node 6] ICP22 created new session: [5:7587241304339437972:2096]
2025-12-24T02:11:16.034857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304339437972:2096] [node 6] ICS09 handshake done sender: [5:7587241300044470646:2098] self: [5:7587241300044470645:0] peer: [6:7587241299510440800:0] socket: 516 qp: -1
2025-12-24T02:11:16.034865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304339437972:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:16.034876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.035349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241304109785565:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.035837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241299510440722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241299510440798:2114] held: no
2025-12-24T02:11:16.035897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408105:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.036283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241299510440743:2068] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.036380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241299510440741:2065] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.036396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.036402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241299510440741:2065] [node 5] ICP22 created new session: [6:7587241303805408106:2065]
2025-12-24T02:11:16.036409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303805408106:2065] [node 5] ICS09 handshake done sender: [6:7587241299510440743:2068] self: [6:7587241299510440800:0] peer: [5:7587241300044470645:0] socket: 517 qp: -1
2025-12-24T02:11:16.036417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303805408106:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:16.036438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.036621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241299510440798:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.036987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241299510440722:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.037002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.037009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241299510440722:2049] [node 2] ICP22 created new session: [6:7587241303805408108:2049]
2025-12-24T02:11:16.037014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303805408108:2049] [node 2] ICS09 handshake done sender: [6:7587241299510440798:2114] self: [6:7587241299510440802:0] peer: [2:7587241299814818246:0] socket: 535 qp: -1
2025-12-24T02:11:16.037020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303805408108:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.037026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.036053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302883308461:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:16.038960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302883308461:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.039009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241298588341083:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.041725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241304109785559:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.040750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241302629945139:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65088
2025-12-24T02:11:16.040919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241298334977822:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.042730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408100:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:16.043200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241304109785565:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.042805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241301808205744:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:16.043396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303805408100:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.043433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241299510440721:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.043592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302629945139:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.046230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241304109785555:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.046255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241304109785555:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.051723Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241301808205676:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(72150425ns) {Configure:104(72148750ns) {ProcessConfigured:135(71883137ns) {StartInitialHandshake:157(71882497ns) {DropHandshakes:470(448ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(2120ns) {SwitchToState:179(974ns)}}}}}
2025-12-24T02:11:16.051827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241301808205744:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.058825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241301808205742:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(35611202ns) {Configure:104(35609291ns) {ProcessConfigured:135(35280622ns) {StartInitialHandshake:157(35279993ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1938ns) {SwitchToState:179(732ns)}}}}}
2025-12-24T02:11:16.059069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241298334977749:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35336664ns) {HandleHandshakeStatus:329(35335884ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(205ns) SwitchToState:179(187ns) ProcessPendingSessionEvents:495(35256111ns) {ForwardSessionEventToSession:612(35237964ns) ForwardSessionEventToSession:612(2634ns) ForwardSessionEventToSession:612(1122ns) ForwardSessionEventToSession:612(703ns) ForwardSessionEventToSession:612(877ns) ForwardSessionEventToSession:612(1103ns) ForwardSessionEventToSession:612(474ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(532ns) ForwardSessionEventToSession:612(1494ns)}}}
2025-12-24T02:11:16.060546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241298334977826:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.060562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241298334977826:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.059698Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241301808205743:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32783195ns) {Configure:104(32781939ns) {ProcessConfigured:135(32431615ns) {StartInitialHandshake:157(32430969ns) {DropHandshakes:470(508ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(608ns) {SwitchToState:179(250ns)}}}}}
2025-12-24T02:11:16.059463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241298588341152:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32512002ns) {Configure:104(32511801ns) {ProcessConfigured:135(32195014ns) {StartInitialHandshake:157(32179575ns) {DropHandshakes:470(680ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(438ns) PrepareNewSessionHandshake:477(1294ns) {SwitchToState:179(683ns)}} IncomingHandshake:261(6738ns) {DropIncomingHandshake:442(105ns) PrepareNewSessionHandshake:477(133ns) {SwitchToState:179(76ns)} GenerateSessionVirtualId:988(608ns) IssueIncomingHandshakeReply:193(182ns)}}}}
2025-12-24T02:11:16.063490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241304109785556:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.063509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241304109785556:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.063337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241298334977812:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.063366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.063393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241298334977812:2108] [node 5] ICP22 created new session: [1:7587241302629945140:2108]
2025-12-24T02:11:16.063404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302629945140:2108] [node 5] ICS09 handshake done sender: [1:7587241298334977822:2112] self: [1:7587241298334977819:0] peer: [5:7587241304339437965:0] socket: 533 qp: -1
2025-12-24T02:11:16.063415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302629945140:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:16.063444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.071132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306103173051:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:16.071222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306103173052:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:16.071314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306103173053:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.075390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.074838Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241299510440754:2074] [node 4] ICP35 event processing took too much time PendingConnection:279(51546898ns) {HandleHandshakeStatus:329(51545323ns) {DropIncomingHandshake:442(112ns) DropOutgoingHandshake:457(335ns) SwitchToState:179(192ns) ProcessPendingSessionEvents:495(31919708ns) {ForwardSessionEventToSession:612(31906232ns) ForwardSessionEventToSession:612(2135ns) ForwardSessionEventToSession:612(1398ns) ForwardSessionEventToSession:612(982ns) ForwardSessionEventToSession:612(728ns) ForwardSessionEventToSession:612(1093ns) ForwardSessionEventToSession:612(710ns) ForwardSessionEventToSession:612(680ns)}}}
2025-12-24T02:11:16.075472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437976:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.075540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437977:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.078831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173051:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.078872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173053:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.078915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241301808205676:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.079172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241301808205735:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.079406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241299814818256:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.079421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241299814818256:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.080284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304339437976:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.080340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241300044470656:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.080527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302883308463:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.080540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241302883308463:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.081897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241299814818256:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.081218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241301808205741:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38286542ns) {Configure:104(38285095ns) {ProcessConfigured:135(37962064ns) {StartInitialHandshake:157(37961125ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(158ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(524ns) PrepareNewSessionHandshake:477(1012ns) {SwitchToState:179(468ns)}}}}}
2025-12-24T02:11:16.081336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306103173056:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:16.082896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241304109785555:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.082933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241298334977826:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.083033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241301808205675:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.083050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.083057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241301808205675:2048] [node 1] ICP22 created new session: [3:7587241306103173058:2048]
2025-12-24T02:11:16.083063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306103173058:2048] [node 1] ICS09 handshake done sender: [3:7587241301808205735:2107] self: [3:7587241301808205736:0] peer: [1:7587241298334977813:0] socket: 521 qp: -1
2025-12-24T02:11:16.083070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306103173058:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.083093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.083079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241298334977824:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.083105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.083113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241298334977824:2113] [node 6] ICP22 created new session: [1:7587241302629945143:2113]
2025-12-24T02:11:16.083121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302629945143:2113] [node 6] ICS09 handshake done sender: [1:7587241298334977826:2114] self: [1:7587241298334977825:0] peer: [6:7587241303805408110:0] socket: 548 qp: -1
2025-12-24T02:11:16.083128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302629945143:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.083137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.083414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241299814818251:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.083431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.083436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241299814818251:2113] [node 5] ICP22 created new session: [2:7587241304109785571:2113]
2025-12-24T02:11:16.083442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241304109785571:2113] [node 5] ICS09 handshake done sender: [2:7587241304109785555:2116] self: [2:7587241299814818253:0] peer: [5:7587241304339437969:0] socket: 518 qp: -1
2025-12-24T02:11:16.083447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241304109785571:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:16.083454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.083256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302883308463:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.083328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241298588341152:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.083341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.083346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241298588341152:2110] [node 5] ICP22 created new session: [4:7587241302883308467:2110]
2025-12-24T02:11:16.083350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302883308467:2110] [node 5] ICS09 handshake done sender: [4:7587241302883308463:2118] self: [4:7587241302883308458:0] peer: [5:7587241304339437978:0] socket: 567 qp: -1
2025-12-24T02:11:16.083354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302883308467:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:16.083369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.081119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241300044470665:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.087341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241299814818178:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.087367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.087392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241299814818178:2049] [node 3] ICP22 created new session: [2:7587241304109785573:2049]
2025-12-24T02:11:16.087402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241304109785573:2049] [node 3] ICS09 handshake done sender: [2:7587241299814818256:2114] self: [2:7587241299814818252:0] peer: [3:7587241306103173054:0] socket: 541 qp: -1
2025-12-24T02:11:16.087412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241304109785573:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.087422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.088418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241304109785569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60006
2025-12-24T02:11:16.089041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241300044470593:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.089068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.089075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241300044470593:2048] [node 1] ICP22 created new session: [5:7587241304339437979:2048]
2025-12-24T02:11:16.089083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304339437979:2048] [node 1] ICS09 handshake done sender: [5:7587241300044470665:2112] self: [5:7587241304339437965:0] peer: [1:7587241298334977819:0] socket: 536 qp: -1
2025-12-24T02:11:16.089092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304339437979:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.089115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.087986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306103173051:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.091136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241304109785556:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.091954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241298334977812:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(49991198ns) {HandleHandshakeStatus:329(49990518ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(781ns) SwitchToState:179(138ns) ProcessPendingSessionEvents:495(9267676ns) {ForwardSessionEventToSession:612(9254518ns) ForwardSessionEventToSession:612(2404ns) ForwardSessionEventToSession:612(1319ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(943ns) ForwardSessionEventToSession:612(540ns) ForwardSessionEventToSession:612(594ns) ForwardSessionEventToSession:612(885ns) ForwardSessionEventToSession:612(1085ns)}}}
2025-12-24T02:11:16.091991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241298334977812:2108] [node 5] ICP18 handshake (actor [1:7587241302629945139:2117]) is too old
2025-12-24T02:11:16.092226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304339437977:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.091395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241301808205676:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.091415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.091423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241301808205676:2049] [node 2] ICP22 created new session: [3:7587241306103173060:2049]
2025-12-24T02:11:16.091449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306103173060:2049] [node 2] ICS09 handshake done sender: [3:7587241306103173051:2115] self: [3:7587241306103173054:0] peer: [2:7587241299814818252:0] socket: 544 qp: -1
2025-12-24T02:11:16.091457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306103173060:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.091474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.092434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241300044470594:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241304339437967:2114] held: no
2025-12-24T02:11:16.092513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241300044470595:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.092596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.092842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241304109785569:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.092957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241299814818178:2049] [node 3] ICP18 handshake (actor [2:7587241304109785569:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.094052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437982:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.094964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241299510440721:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241303805408100:2117] held: no
2025-12-24T02:11:16.095304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304339437967:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.095442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241300044470594:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.095454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.095461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241300044470594:2049] [node 2] ICP22 created new session: [5:7587241304339437985:2049]
2025-12-24T02:11:16.095467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304339437985:2049] [node 2] ICS09 handshake done sender: [5:7587241304339437967:2114] self: [5:7587241304339437969:0] peer: [2:7587241299814818253:0] socket: 552 qp: -1
2025-12-24T02:11:16.095473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304339437985:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.095492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.095578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173049:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.095589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306103173049:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.096045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306103173063:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15136
2025-12-24T02:11:16.096148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306103173049:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.096259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241301808205743:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.096278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.096284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241301808205743:2110] [node 5] ICP22 created new session: [3:7587241306103173064:2110]
2025-12-24T02:11:16.096291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306103173064:2110] [node 5] ICS09 handshake done sender: [3:7587241306103173049:2114] self: [3:7587241306103173042:0] peer: [5:7587241304339437983:0] socket: 568 qp: -1
2025-12-24T02:11:16.096296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306103173064:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:16.096304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.096382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304339437976:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.097914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241299814818243:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.097940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.097947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241299814818243:2109] [node 4] ICP22 created new session: [2:7587241304109785576:2109]
2025-12-24T02:11:16.097953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241304109785576:2109] [node 4] ICS09 handshake done sender: [2:7587241304109785556:2117] self: [2:7587241299814818255:0] peer: [4:7587241302883308462:0] socket: 558 qp: -1
2025-12-24T02:11:16.097958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241304109785576:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:16.097966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.098242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302883308469:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.099472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241301808205675:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16404098ns) {HandleHandshakeStatus:329(16403311ns) {DropIncomingHandshake:442(279ns) DropOutgoingHandshake:457(126ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(8515580ns) {ForwardSessionEventToSession:612(8503370ns) ForwardSessionEventToSession:612(1357ns) ForwardSessionEventToSession:612(1390ns) ForwardSessionEventToSession:612(843ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(836ns) ForwardSessionEventToSession:612(857ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(953ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.101070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304339437987:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51280
2025-12-24T02:11:16.101755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241300044470656:2093] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.101772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.101778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241300044470656:2093] [node 4] ICP22 created new session: [5:7587241304339437988:2093]
2025-12-24T02:11:16.101783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304339437988:2093] [node 4] ICS09 handshake done sender: [5:7587241304339437976:2118] self: [5:7587241304339437978:0] peer: [4:7587241302883308458:0] socket: 572 qp: -1
2025-12-24T02:11:16.101790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304339437988:2093] [node 4] ICS10 traffic start
2025-12-24T02:11:16.101796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.101860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241300044470656:2093] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:16.101958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304339437977:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.102540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408112:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:16.099823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302883308469:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.099874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241298588341084:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.102663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408111:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:16.100108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241298588341083:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241302883308461:2117] held: no
2025-12-24T02:11:16.100454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302883308470:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:16.100529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302883308461:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.100620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241298588341083:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.100631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.100638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241298588341083:2049] [node 2] ICP22 created new session: [4:7587241302883308472:2049]
2025-12-24T02:11:16.100645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302883308472:2049] [node 2] ICS09 handshake done sender: [4:7587241302883308461:2117] self: [4:7587241302883308462:0] peer: [2:7587241299814818255:0] socket: 561 qp: -1
2025-12-24T02:11:16.100652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302883308472:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.100665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.102809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303805408100:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.102889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241299510440721:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.102912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.102918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241299510440721:2048] [node 1] ICP22 created new session: [6:7587241303805408113:2048]
2025-12-24T02:11:16.102923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303805408113:2048] [node 1] ICS09 handshake done sender: [6:7587241303805408100:2117] self: [6:7587241303805408110:0] peer: [1:7587241298334977825:0] socket: 553 qp: -1
2025-12-24T02:11:16.102929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303805408113:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.102940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.106893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303805408112:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.107208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173046:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.107221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306103173046:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.108157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306103173046:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.108239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173063:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.108262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241301808205743:2110] [node 5] ICP18 handshake (actor [3:7587241306103173063:2120]) is too old
2025-12-24T02:11:16.108512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241301808205742:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.108530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.108536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241301808205742:2108] [node 4] ICP22 created new session: [3:7587241306103173068:2108]
2025-12-24T02:11:16.108543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306103173068:2108] [node 4] ICS09 handshake done sender: [3:7587241306103173046:2113] self: [3:7587241306103173041:0] peer: [4:7587241302883308471:0] socket: 566 qp: -1
2025-12-24T02:11:16.108550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306103173068:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:16.108558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.108922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306103173055:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.108931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306103173055:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.109080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241299510440723:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.110617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303805408116:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:858
2025-12-24T02:11:16.112275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306103173055:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.113180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303805408112:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.113281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241299510440723:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.113300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.113307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241299510440723:2050] [node 3] ICP22 created new session: [6:7587241303805408117:2050]
2025-12-24T02:11:16.113313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303805408117:2050] [node 3] ICS09 handshake done sender: [6:7587241303805408112:2121] self: [6:7587241303805408115:0] peer: [3:7587241306103173043:0] socket: 525 qp: -1
2025-12-24T02:11:16.113319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303805408117:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.113331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.114228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241300044470595:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8251) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241304339437977:2119] held: no
2025-12-24T02:11:16.114630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241301808205741:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.114652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.114659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241301808205741:2109] [node 6] ICP22 created new session: [3:7587241306103173071:2109]
2025-12-24T02:11:16.114665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306103173071:2109] [node 6] ICS09 handshake done sender: [3:7587241306103173055:2118] self: [3:7587241306103173043:0] peer: [6:7587241303805408115:0] socket: 585 qp: -1
2025-12-24T02:11:16.114670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306103173071:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:16.114677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.115795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241300044470595:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.115810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.115818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241300044470595:2050] [node 3] ICP22 created new session: [5:7587241304339437990:2050]
2025-12-24T02:11:16.115824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304339437990:2050] [node 3] ICS09 handshake done sender: [5:7587241304339437977:2119] self: [5:7587241304339437983:0] peer: [3:7587241306103173042:0] socket: 573 qp: -1
2025-12-24T02:11:16.115830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304339437990:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.115841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.109733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302883308474:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16711
2025-12-24T02:11:16.110835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302883308469:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.110998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241298588341084:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.111017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.111025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241298588341084:2050] [node 3] ICP22 created new session: [4:7587241302883308475:2050]
2025-12-24T02:11:16.111032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302883308475:2050] [node 3] ICS09 handshake done sender: [4:7587241302883308469:2119] self: [4:7587241302883308471:0] peer: [3:7587241306103173041:0] socket: 569 qp: -1
2025-12-24T02:11:16.111039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302883308475:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.111047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.130815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:16.130908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241304109785557:2048] [node 1] ICS07 socket disconnect 512 reason# EndOfStream
2025-12-24T02:11:16.130916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241304109785557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.130980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.131001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241304109785557:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.138873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:16.138997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241306103173058:2048] [node 1] ICS07 socket disconnect 521 reason# EndOfStream
2025-12-24T02:11:16.139004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241306103173058:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.139052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.139073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241306103173058:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.140113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241299814818177:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:16.140128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241304109785557:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.140579Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241301808205675:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.140593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241306103173058:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.140599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241301808205675:2048] [node 1] ICP30 unregister session Session# [3:7587241306103173058:2048] VirtualId# [3:7587241301808205736:0]
2025-12-24T02:11:16.140629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.140635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241301808205675:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.140638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.140470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241299814818177:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.140484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241304109785557:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.140490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241299814818177:2048] [node 1] ICP30 unregister session Session# [2:7587241304109785557:2048] VirtualId# [2:7587241299814818241:0]
2025-12-24T02:11:16.140509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.140516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241299814818177:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.140519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.144142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:16.144223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241304339437979:2048] [node 1] ICS07 socket disconnect 536 reason# EndOfStream
2025-12-24T02:11:16.144229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304339437979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.144262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.144285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241304339437979:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.144456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:16.144525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303805408113:2048] [node 1] ICS01 socket: 553 reason# EndOfStream
2025-12-24T02:11:16.144532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241299510440721:2048] [node 1] ICP30 unregister session Session# [6:7587241303805408113:2048] VirtualId# [6:7587241303805408110:0]
2025-12-24T02:11:16.144538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303805408113:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.144567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.144594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.143743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:16.143835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241298588341150:2048] [node 1] ICS07 socket disconnect 514 reason# EndOfStream
2025-12-24T02:11:16.143842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241298588341150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.143901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.143924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241298588341150:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.151617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241301808205675:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:16.152722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241300044470593:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.152743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304339437979:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.152749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241300044470593:2048] [node 1] ICP30 unregister session Session# [5:7587241304339437979:2048] VirtualId# [5:7587241304339437965:0]
2025-12-24T02:11:16.152767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.152773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241300044470593:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.152776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.153040Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241298588341082:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.153058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241298588341150:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.153063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241298588341082:2048] [node 1] ICP30 unregister session Session# [4:7587241298588341150:2048] VirtualId# [4:7587241298588341144:0]
2025-12-24T02:11:16.153074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.153078Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241298588341082:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.153080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16638) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.157773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:16.157873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241306103173060:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-12-24T02:11:16.157882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241301808205676:2049] [node 2] ICP30 unregister session Session# [3:7587241306103173060:2049] VirtualId# [3:7587241306103173054:0]
2025-12-24T02:11:16.157890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241306103173060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.157932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.157954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.162242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241300044470593:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:16.162289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:16.162370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304339437985:2049] [node 2] ICS01 socket: 552 reason# EndOfStream
2025-12-24T02:11:16.162380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241300044470594:2049] [node 2] ICP30 unregister session Session# [5:7587241304339437985:2049] VirtualId# [5:7587241304339437969:0]
2025-12-24T02:11:16.162386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304339437985:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.162459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.162486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.162629Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:16.162823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:16.162883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241302883308472:2049] [node 2] ICS01 socket: 561 reason# EndOfStream
2025-12-24T02:11:16.162888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241298588341083:2049] [node 2] ICP30 unregister session Session# [4:7587241302883308472:2049] VirtualId# [4:7587241302883308462:0]
2025-12-24T02:11:16.162892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241302883308472:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.162927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.162952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.162988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241303805408108:2049] [node 2] ICS07 socket disconnect 535 reason# EndOfStream
2025-12-24T02:11:16.162995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303805408108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.163023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.163043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241303805408108:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:16.163636Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241299510440722:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.163648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303805408108:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.163653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241299510440722:2049] [node 2] ICP30 unregister session Session# [6:7587241303805408108:2049] VirtualId# [6:7587241299510440802:0]
2025-12-24T02:11:16.163665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.163670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241299510440722:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.163672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.163358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241298588341082:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:16.174518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241299510440722:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:16.174555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:16.174632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303805408117:2050] [node 3] ICS01 socket: 525 reason# EndOfStream
2025-12-24T02:11:16.174641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241299510440723:2050] [node 3] ICP30 unregister session Session# [6:7587241303805408117:2050] VirtualId# [6:7587241303805408115:0]
2025-12-24T02:11:16.174646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303805408117:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.174753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:16.174815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304339437990:2050] [node 3] ICS01 socket: 573 reason# EndOfStream
2025-12-24T02:11:16.174820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241300044470595:2050] [node 3] ICP30 unregister session Session# [5:7587241304339437990:2050] VirtualId# [5:7587241304339437983:0]
2025-12-24T02:11:16.174825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304339437990:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.174857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.174881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.175157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.175189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.198356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:16.198459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241303805408101:2074] [node 4] ICS07 socket disconnect 528 reason# EndOfStream
2025-12-24T02:11:16.198465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303805408101:2074] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.198529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:16.198562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241303805408101:2074] [node 4] ICS15 start handshake
2025-12-24T02:11:16.199190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241299510440754:2074] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18241) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.199202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303805408101:2074] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.199207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241299510440754:2074] [node 4] ICP30 unregister session Session# [6:7587241303805408101:2074] VirtualId# [6:7587241299510440756:0]
2025-12-24T02:11:16.199224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:16.199231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241299510440754:2074] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18241) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.199234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18241) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:16.234665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304508446178:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:16.234678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304508446179:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.249137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241304156734714:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.249157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241304156734715:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.249163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241304156734716:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.241036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306689220728:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.241051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306689220729:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.245671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241304889205841:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.245690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241304889205842:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.245695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241304889205843:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.244137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305500891866:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.244187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305500891867:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.244191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305500891868:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.266809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304047083440:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.266827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304047083441:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.290035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241304156734776:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:16.292643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304508446244:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.294106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205901:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.294397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220788:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
2025-12-24T02:11:16.297027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304889205901:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.297242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241304889205907:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:16.297641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241304889205907:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.300569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304508446250:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:16.300671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304508446251:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:16.301975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241304156734778:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.301991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241304156734778:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.302640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241304156734778:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.298895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083500:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.302773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241304156734776:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.302790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.302798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241304156734776:2107] [node 5] ICP22 created new session: [6:7587241304156734781:2107]
2025-12-24T02:11:16.302804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241304156734781:2107] [node 5] ICS09 handshake done sender: [6:7587241304156734778:2109] self: [6:7587241304156734777:0] peer: [5:7587241304889205908:0] socket: 517 qp: -1
2025-12-24T02:11:16.302811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241304156734781:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:16.302821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.302661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220788:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.303144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241306689220728:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.303142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205909:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.306731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083501:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.306887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446239:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.306910Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241304508446239:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.309151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241304508446257:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33290
2025-12-24T02:11:16.309112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304889205901:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.309261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241304889205907:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.309274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.309282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241304889205907:2110] [node 6] ICP22 created new session: [5:7587241304889205910:2110]
2025-12-24T02:11:16.309289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304889205910:2110] [node 6] ICS09 handshake done sender: [5:7587241304889205901:2107] self: [5:7587241304889205908:0] peer: [6:7587241304156734777:0] socket: 521 qp: -1
2025-12-24T02:11:16.309297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304889205910:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:16.309306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.311015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083500:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.311338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241304508446239:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.311211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306689220794:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:16.311298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304047083503:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:16.311685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241304047083503:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.313117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304889205904:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.313136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241304889205904:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.313075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891927:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-12-24T02:11:16.313680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304889205904:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.314084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083505:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.314250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304047083500:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.314494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304508446178:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.314523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.314532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304508446178:2048] [node 2] ICP22 created new session: [1:7587241304508446259:2048]
2025-12-24T02:11:16.314539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241304508446259:2048] [node 2] ICS09 handshake done sender: [1:7587241304508446239:2107] self: [1:7587241304508446238:0] peer: [2:7587241306689220791:0] socket: 516 qp: -1
2025-12-24T02:11:16.314545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241304508446259:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:16.314552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.314926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241304889205843:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.314948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.314956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241304889205843:2050] [node 3] ICP22 created new session: [5:7587241304889205913:2050]
2025-12-24T02:11:16.314963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304889205913:2050] [node 3] ICS09 handshake done sender: [5:7587241304889205904:2109] self: [5:7587241304889205903:0] peer: [3:7587241304047083504:0] socket: 522 qp: -1
2025-12-24T02:11:16.314971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304889205913:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.314978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.315911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306689220797:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:16.316505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220795:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
2025-12-24T02:11:16.316934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241306689220788:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.317012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241306689220728:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.317025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.317031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306689220728:2048] [node 1] ICP22 created new session: [2:7587241306689220804:2048]
2025-12-24T02:11:16.317037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241306689220804:2048] [node 1] ICS09 handshake done sender: [2:7587241306689220788:2107] self: [2:7587241306689220791:0] peer: [1:7587241304508446238:0] socket: 519 qp: -1
2025-12-24T02:11:16.317043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241306689220804:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.317059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.318759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241304047083503:2109] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.318780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.318790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304047083503:2109] [node 5] ICP22 created new session: [3:7587241304047083506:2109]
2025-12-24T02:11:16.318797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304047083506:2109] [node 5] ICS09 handshake done sender: [3:7587241304047083500:2107] self: [3:7587241304047083504:0] peer: [5:7587241304889205903:0] socket: 524 qp: -1
2025-12-24T02:11:16.318804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304047083506:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:16.318823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.318965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083501:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.319286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241304047083440:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.327369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304047083512:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:16.331145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305500891932:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:16.331289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305500891933:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:16.330980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304047083514:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:16.332162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306689220807:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:16.334545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241304156734784:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:16.341487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241304889205920:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.351538Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241305500891932:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16061470ns) {Configure:104(16060595ns) {ProcessConfigured:135(15788539ns) {StartInitialHandshake:157(15788210ns) {DropHandshakes:470(747ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1390ns) {SwitchToState:179(784ns)}}}}}
2025-12-24T02:11:16.351127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734793:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.351495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446242:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.351511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241304508446242:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.355591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220810:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
2025-12-24T02:11:16.355623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220811:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
2025-12-24T02:11:16.355669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220812:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.359070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241304508446178:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:16.359734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205926:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.362290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241304508446261:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33290
2025-12-24T02:11:16.362337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241304508446262:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33290
2025-12-24T02:11:16.362581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241304508446242:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.362303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220812:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.362588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220811:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.362806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205927:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.362990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446262:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.363016Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446261:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.363024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304508446179:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.363037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.363046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304508446179:2049] [node 3] ICP22 created new session: [1:7587241304508446265:2049]
2025-12-24T02:11:16.363052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241304508446265:2049] [node 3] ICS09 handshake done sender: [1:7587241304508446242:2108] self: [1:7587241304508446241:0] peer: [3:7587241304047083508:0] socket: 525 qp: -1
2025-12-24T02:11:16.363058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241304508446265:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.363067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.366084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891927:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.366177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891941:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.366230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891942:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.366270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891943:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-12-24T02:11:16.366406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305500891866:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.366621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891942:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.366884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734797:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.367233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241304156734793:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.367298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241304156734716:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.367449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304889205926:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.367478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241304889205842:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.367109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083522:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.367151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083523:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.367182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083524:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.367712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083525:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.367755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083522:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.367785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083519:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.367789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241304047083519:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.368193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446248:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.368203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241304508446248:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.369631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241304508446270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33290
2025-12-24T02:11:16.368182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083523:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.368200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304047083501:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.370084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891941:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.370117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891943:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.367128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241306689220818:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20495
2025-12-24T02:11:16.367973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220810:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.370146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220803:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.370152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241306689220803:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.371183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241306689220803:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.371347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220799:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.371350Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241306689220799:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.371722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241306689220797:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.371736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.371741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306689220797:2111] [node 5] ICP22 created new session: [2:7587241306689220821:2111]
2025-12-24T02:11:16.371747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241306689220821:2111] [node 5] ICS09 handshake done sender: [2:7587241306689220803:2114] self: [2:7587241306689220802:0] peer: [5:7587241304889205928:0] socket: 553 qp: -1
2025-12-24T02:11:16.371753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241306689220821:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:16.371761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.372853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241306689220799:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.370779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241304047083440:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.370801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.370809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304047083440:2048] [node 1] ICP22 created new session: [3:7587241304047083527:2048]
2025-12-24T02:11:16.370816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304047083527:2048] [node 1] ICS09 handshake done sender: [3:7587241304047083501:2108] self: [3:7587241304047083508:0] peer: [1:7587241304508446241:0] socket: 527 qp: -1
2025-12-24T02:11:16.370823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304047083527:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.370831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.371060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241304047083441:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.371262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304047083519:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.372108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241304047083514:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.372118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.372124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304047083514:2113] [node 6] ICP22 created new session: [3:7587241304047083531:2113]
2025-12-24T02:11:16.372129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304047083531:2113] [node 6] ICS09 handshake done sender: [3:7587241304047083519:2115] self: [3:7587241304047083518:0] peer: [6:7587241304156734798:0] socket: 539 qp: -1
2025-12-24T02:11:16.372135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304047083531:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.372142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.372775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304047083533:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54115
2025-12-24T02:11:16.373005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304047083523:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.373053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241304047083441:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.373062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.373067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304047083441:2049] [node 2] ICP22 created new session: [3:7587241304047083534:2049]
2025-12-24T02:11:16.373071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304047083534:2049] [node 2] ICS09 handshake done sender: [3:7587241304047083523:2117] self: [3:7587241304047083530:0] peer: [2:7587241306689220798:0] socket: 568 qp: -1
2025-12-24T02:11:16.373076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304047083534:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.373081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.374861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304889205927:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.381334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305500891868:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.381365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305500891867:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.381399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-12-24T02:11:16.382583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083516:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.382598Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241304047083516:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.384037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241304889205842:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241304889205926:2116] held: no
2025-12-24T02:11:16.384141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241304889205841:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.384171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205929:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.384398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304889205926:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.384478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241304889205842:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.384493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.384499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241304889205842:2049] [node 2] ICP22 created new session: [5:7587241304889205931:2049]
2025-12-24T02:11:16.384506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304889205931:2049] [node 2] ICS09 handshake done sender: [5:7587241304889205926:2116] self: [5:7587241304889205928:0] peer: [2:7587241306689220802:0] socket: 556 qp: -1
2025-12-24T02:11:16.384513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304889205931:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.384523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.385304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241304508446248:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.385622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446253:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.385629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241304508446253:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.386086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304508446244:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.386095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.386102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304508446244:2109] [node 4] ICP22 created new session: [1:7587241304508446272:2109]
2025-12-24T02:11:16.386107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241304508446272:2109] [node 4] ICS09 handshake done sender: [1:7587241304508446248:2112] self: [1:7587241304508446245:0] peer: [4:7587241305500891948:0] socket: 532 qp: -1
2025-12-24T02:11:16.386114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241304508446272:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:16.386123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.386589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205933:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.381655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241306689220729:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.381684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.381692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306689220729:2049] [node 3] ICP22 created new session: [2:7587241306689220825:2049]
2025-12-24T02:11:16.381700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241306689220825:2049] [node 3] ICS09 handshake done sender: [2:7587241306689220799:2112] self: [2:7587241306689220798:0] peer: [3:7587241304047083530:0] socket: 551 qp: -1
2025-12-24T02:11:16.381706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241306689220825:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.381716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.381961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220801:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.381965Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241306689220801:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.384856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220818:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.388028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305500891927:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.388345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305500891866:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.388363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.388370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305500891866:2048] [node 1] ICP22 created new session: [4:7587241305500891954:2048]
2025-12-24T02:11:16.388375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305500891954:2048] [node 1] ICS09 handshake done sender: [4:7587241305500891927:2108] self: [4:7587241305500891948:0] peer: [1:7587241304508446245:0] socket: 534 qp: -1
2025-12-24T02:11:16.388381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305500891954:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.388387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.386908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304047083516:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.387046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241304047083512:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.387054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.387060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304047083512:2111] [node 4] ICP22 created new session: [3:7587241304047083537:2111]
2025-12-24T02:11:16.387066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304047083537:2111] [node 4] ICS09 handshake done sender: [3:7587241304047083516:2114] self: [3:7587241304047083515:0] peer: [4:7587241305500891951:0] socket: 538 qp: -1
2025-12-24T02:11:16.387071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304047083537:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:16.387079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.387464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304047083525:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.387489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241304047083512:2111] [node 4] ICP18 handshake (actor [3:7587241304047083525:2119]) is too old
2025-12-24T02:11:16.388203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241304156734797:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.388243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241304156734716:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241304156734793:2115] held: no
2025-12-24T02:11:16.388282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734799:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.389330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734800:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.389396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241304156734714:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.389407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241304156734793:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.389600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241304156734716:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.389609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.389616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241304156734716:2050] [node 3] ICP22 created new session: [6:7587241304156734803:2050]
2025-12-24T02:11:16.389622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241304156734803:2050] [node 3] ICS09 handshake done sender: [6:7587241304156734793:2115] self: [6:7587241304156734798:0] peer: [3:7587241304047083518:0] socket: 544 qp: -1
2025-12-24T02:11:16.389628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241304156734803:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.389633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.393528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305500891868:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241305500891941:2115] held: no
2025-12-24T02:11:16.393745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891953:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.393909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205934:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.394036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241304508446253:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.394242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304889205927:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.394307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241304889205841:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.394325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.394332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241304889205841:2048] [node 1] ICP22 created new session: [5:7587241304889205935:2048]
2025-12-24T02:11:16.394339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304889205935:2048] [node 1] ICS09 handshake done sender: [5:7587241304889205927:2117] self: [5:7587241304889205930:0] peer: [1:7587241304508446252:0] socket: 560 qp: -1
2025-12-24T02:11:16.394345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304889205935:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.394353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.395432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446270:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.395565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241304508446255:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.395569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241304508446255:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.396020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304508446250:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.396036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.396042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304508446250:2110] [node 5] ICP22 created new session: [1:7587241304508446275:2110]
2025-12-24T02:11:16.396049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241304508446275:2110] [node 5] ICS09 handshake done sender: [1:7587241304508446253:2113] self: [1:7587241304508446252:0] peer: [5:7587241304889205930:0] socket: 559 qp: -1
2025-12-24T02:11:16.396057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241304508446275:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:16.396067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.396881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891956:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-12-24T02:11:16.396937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305500891957:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31537
2025-12-24T02:11:16.397617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305500891941:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.396648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241306689220801:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.397158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241306689220794:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.397171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.397178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306689220794:2109] [node 4] ICP22 created new session: [2:7587241306689220829:2109]
2025-12-24T02:11:16.397184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241306689220829:2109] [node 4] ICS09 handshake done sender: [2:7587241306689220801:2113] self: [2:7587241306689220800:0] peer: [4:7587241305500891952:0] socket: 552 qp: -1
2025-12-24T02:11:16.397189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241306689220829:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:16.397196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.398065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241306689220814:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.398070Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241306689220814:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.397755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241304156734800:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.397841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241304156734715:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.397906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734805:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.398849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891956:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.399828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241304508446244:2109] [node 4] ICP18 handshake (actor [1:7587241304508446270:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.400237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241304508446255:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.399555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734807:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.400085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734808:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
2025-12-24T02:11:16.400413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241304156734797:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.400463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241304156734714:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.400476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.400482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241304156734714:2048] [node 1] ICP22 created new session: [6:7587241304156734809:2048]
2025-12-24T02:11:16.400489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241304156734809:2048] [node 1] ICS09 handshake done sender: [6:7587241304156734797:2116] self: [6:7587241304156734802:0] peer: [1:7587241304508446254:0] socket: 569 qp: -1
2025-12-24T02:11:16.400495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241304156734809:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.400503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.403327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304508446251:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.403351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.403359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304508446251:2111] [node 6] ICP22 created new session: [1:7587241304508446278:2111]
2025-12-24T02:11:16.403366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241304508446278:2111] [node 6] ICS09 handshake done sender: [1:7587241304508446255:2114] self: [1:7587241304508446254:0] peer: [6:7587241304156734802:0] socket: 563 qp: -1
2025-12-24T02:11:16.403373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241304508446278:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:16.403394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.403337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305500891868:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.403357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.403364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305500891868:2050] [node 3] ICP22 created new session: [4:7587241305500891960:2050]
2025-12-24T02:11:16.403369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305500891960:2050] [node 3] ICS09 handshake done sender: [4:7587241305500891941:2115] self: [4:7587241305500891951:0] peer: [3:7587241304047083515:0] socket: 548 qp: -1
2025-12-24T02:11:16.403376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305500891960:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.403396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.403542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305500891867:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3920) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241305500891943:2117] held: no
2025-12-24T02:11:16.404534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241304156734800:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.404752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241304156734715:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.404770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.404776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241304156734715:2049] [node 2] ICP22 created new session: [6:7587241304156734811:2049]
2025-12-24T02:11:16.404782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241304156734811:2049] [node 2] ICS09 handshake done sender: [6:7587241304156734800:2118] self: [6:7587241304156734806:0] peer: [2:7587241306689220813:0] socket: 581 qp: -1
2025-12-24T02:11:16.404793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241304156734811:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.404800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.402934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241306689220814:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.403849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241306689220807:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.403861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.403867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306689220807:2115] [node 6] ICP22 created new session: [2:7587241306689220832:2115]
2025-12-24T02:11:16.403873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241306689220832:2115] [node 6] ICS09 handshake done sender: [2:7587241306689220814:2119] self: [2:7587241306689220813:0] peer: [6:7587241304156734806:0] socket: 562 qp: -1
2025-12-24T02:11:16.403878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241306689220832:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:16.403887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.407179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241304156734715:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:16.407717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305500891943:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.407857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305500891867:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.407873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.407893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305500891867:2049] [node 2] ICP22 created new session: [4:7587241305500891963:2049]
2025-12-24T02:11:16.407906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305500891963:2049] [node 2] ICS09 handshake done sender: [4:7587241305500891943:2117] self: [4:7587241305500891952:0] peer: [2:7587241306689220800:0] socket: 561 qp: -1
2025-12-24T02:11:16.407914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305500891963:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.407926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.408170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241304156734805:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.408208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241304156734784:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.408341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241304889205934:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.408371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241304889205920:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.408503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891945:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.408514Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241305500891945:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.408804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305500891939:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.408812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241305500891939:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.409443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241304156734814:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15102
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.409602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241304889205938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40785
2025-12-24T02:11:16.409581Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305500891945:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.409887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241304156734805:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.409767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305500891939:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.409831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241305500891933:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.409844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.409850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305500891933:2111] [node 6] ICP22 created new session: [4:7587241305500891967:2111]
2025-12-24T02:11:16.409854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305500891967:2111] [node 6] ICS09 handshake done sender: [4:7587241305500891945:2118] self: [4:7587241305500891944:0] peer: [6:7587241304156734813:0] socket: 542 qp: -1
2025-12-24T02:11:16.409863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305500891967:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:16.409871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.410093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241304889205934:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.409993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241305500891932:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.410002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.410005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305500891932:2109] [node 5] ICP22 created new session: [4:7587241305500891969:2109]
2025-12-24T02:11:16.410007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305500891969:2109] [node 5] ICS09 handshake done sender: [4:7587241305500891939:2114] self: [4:7587241305500891938:0] peer: [5:7587241304889205937:0] socket: 583 qp: -1
2025-12-24T02:11:16.410010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305500891969:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:16.410014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.410198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241304889205920:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.410212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.410218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241304889205920:2112] [node 4] ICP22 created new session: [5:7587241304889205939:2112]
2025-12-24T02:11:16.410223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241304889205939:2112] [node 4] ICS09 handshake done sender: [5:7587241304889205934:2120] self: [5:7587241304889205937:0] peer: [4:7587241305500891938:0] socket: 584 qp: -1
2025-12-24T02:11:16.410229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241304889205939:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:16.410238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.411864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241304156734784:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.411884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.411891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241304156734784:2110] [node 4] ICP22 created new session: [6:7587241304156734815:2110]
2025-12-24T02:11:16.411897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241304156734815:2110] [node 4] ICS09 handshake done sender: [6:7587241304156734805:2119] self: [6:7587241304156734813:0] peer: [4:7587241305500891944:0] socket: 585 qp: -1
2025-12-24T02:11:16.411904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241304156734815:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:16.411913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.411989Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241304156734784:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:16.416783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:16.416860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241304047083527:2048] [node 1] ICS01 socket: 527 reason# EndOfStream
2025-12-24T02:11:16.416873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241304047083440:2048] [node 1] ICP30 unregister session Session# [3:7587241304047083527:2048] VirtualId# [3:7587241304047083508:0]
2025-12-24T02:11:16.416879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241304047083527:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.416938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.416977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.416317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:16.416418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241306689220804:2048] [node 1] ICS01 socket: 519 reason# EndOfStream
2025-12-24T02:11:16.416428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241306689220728:2048] [node 1] ICP30 unregister session Session# [2:7587241306689220804:2048] VirtualId# [2:7587241306689220791:0]
2025-12-24T02:11:16.416435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241306689220804:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.416483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.416505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.419475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:16.419539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241305500891954:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:16.419543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305500891866:2048] [node 1] ICP30 unregister session Session# [4:7587241305500891954:2048] VirtualId# [4:7587241305500891948:0]
2025-12-24T02:11:16.419547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241305500891954:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.419588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.419609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.424799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:16.424904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241304156734809:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:16.424913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241304156734714:2048] [node 1] ICP30 unregister session Session# [6:7587241304156734809:2048] VirtualId# [6:7587241304156734802:0]
2025-12-24T02:11:16.424921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241304156734809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.424991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.425025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.425129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:16.425197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304889205935:2048] [node 1] ICS01 socket: 560 reason# EndOfStream
2025-12-24T02:11:16.425203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241304889205841:2048] [node 1] ICP30 unregister session Session# [5:7587241304889205935:2048] VirtualId# [5:7587241304889205930:0]
2025-12-24T02:11:16.425208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304889205935:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.425237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.425259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.426198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:16.426276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241304047083534:2049] [node 2] ICS01 socket: 568 reason# EndOfStream
2025-12-24T02:11:16.426283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241304047083441:2049] [node 2] ICP30 unregister session Session# [3:7587241304047083534:2049] VirtualId# [3:7587241304047083530:0]
2025-12-24T02:11:16.426288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241304047083534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.426327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.426355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.428136Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:16.428217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304889205931:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:16.428227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241304889205842:2049] [node 2] ICP30 unregister session Session# [5:7587241304889205931:2049] VirtualId# [5:7587241304889205928:0]
2025-12-24T02:11:16.428233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304889205931:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.428278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.428304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.429340Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:16.429420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241305500891963:2049] [node 2] ICS01 socket: 561 reason# EndOfStream
2025-12-24T02:11:16.429426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305500891867:2049] [node 2] ICP30 unregister session Session# [4:7587241305500891963:2049] VirtualId# [4:7587241305500891952:0]
2025-12-24T02:11:16.429432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241305500891963:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.429484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.429524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.429361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:16.429439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241304156734811:2049] [node 2] ICS01 socket: 581 reason# EndOfStream
2025-12-24T02:11:16.429446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241304156734715:2049] [node 2] ICP30 unregister session Session# [6:7587241304156734811:2049] VirtualId# [6:7587241304156734806:0]
2025-12-24T02:11:16.429451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241304156734811:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.429499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.429535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.434856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:16.434946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241304889205913:2050] [node 3] ICS07 socket disconnect 522 reason# EndOfStream
2025-12-24T02:11:16.434954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304889205913:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.435013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.435035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241304889205913:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:16.435538Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:16.435605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241304156734803:2050] [node 3] ICS01 socket: 544 reason# EndOfStream
2025-12-24T02:11:16.435609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241304156734716:2050] [node 3] ICP30 unregister session Session# [6:7587241304156734803:2050] VirtualId# [6:7587241304156734798:0]
2025-12-24T02:11:16.435615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241304156734803:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.435658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.435681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.435456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241304889205843:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.435470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304889205913:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.435476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241304889205843:2050] [node 3] ICP30 unregister session Session# [5:7587241304889205913:2050] VirtualId# [5:7587241304889205903:0]
2025-12-24T02:11:16.435491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.435499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241304889205843:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.435502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.441108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:16.441201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241304889205939:2112] [node 4] ICS01 socket: 584 reason# EndOfStream
2025-12-24T02:11:16.441210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241304889205920:2112] [node 4] ICP30 unregister session Session# [5:7587241304889205939:2112] VirtualId# [5:7587241304889205937:0]
2025-12-24T02:11:16.441216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241304889205939:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.441267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:16.441299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:16.441403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:16.441469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241304156734815:2110] [node 4] ICS07 socket disconnect 585 reason# EndOfStream
2025-12-24T02:11:16.441474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241304156734815:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.441504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:16.441523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241304156734815:2110] [node 4] ICS15 start handshake
2025-12-24T02:11:16.441876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241304156734784:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.441888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241304156734815:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.441892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241304156734784:2110] [node 4] ICP30 unregister session Session# [6:7587241304156734815:2110] VirtualId# [6:7587241304156734813:0]
2025-12-24T02:11:16.441907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:16.441914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241304156734784:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.441916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.447520Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:16.447651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241304156734781:2107] [node 5] ICS07 socket disconnect 517 reason# EndOfStream
2025-12-24T02:11:16.447660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241304156734781:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.447732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:16.447757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241304156734781:2107] [node 5] ICS15 start handshake
2025-12-24T02:11:16.448203Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241304156734776:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20895) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.448221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241304156734781:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.448228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241304156734776:2107] [node 5] ICP30 unregister session Session# [6:7587241304156734781:2107] VirtualId# [6:7587241304156734777:0]
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:16.470854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241302778194031:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:16.470868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241302778194032:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.472214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241303097238184:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.472244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241303097238185:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.473860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241306107112286:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.473870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241306107112287:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.490966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241303574031874:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.490976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241303574031875:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.490985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241303574031876:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.484731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241302515941107:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.484741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241302515941108:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.484751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241302515941109:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.486304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305559525392:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.486313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305559525393:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.486319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305559525394:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.524032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241303574031936:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.524800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238244:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.524955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525453:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.527060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241302778194095:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:16.528034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241302778194103:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:16.528289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241302778194102:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:16.528990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241303097238250:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:16.529059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241303097238251:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.529721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941168:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.530859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241303097238260:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:16.530863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241306107112350:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.532284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241306107112360:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:16.532366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241306107112361:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:16.529760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525454:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.535326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112346:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.536696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941169:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.538143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241302778194108:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20805
2025-12-24T02:11:16.538176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241302778194109:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20805
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:16.538177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238244:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.538250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241303097238184:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.538712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238264:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
2025-12-24T02:11:16.538838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238261:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
2025-12-24T02:11:16.543968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241302515941178:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:16.543917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112346:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.544093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241302515941179:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:16.544142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941168:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.544642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305559525463:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:16.544656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525454:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.544735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305559525464:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:16.544783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305559525393:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.545519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238255:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.545534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241303097238255:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.546159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238267:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
2025-12-24T02:11:16.546174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238268:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.546414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241303097238255:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.546551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241303097238251:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.546567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.546573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241303097238251:2109] [node 5] ICP22 created new session: [2:7587241303097238270:2109]
2025-12-24T02:11:16.546578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241303097238270:2109] [node 5] ICS09 handshake done sender: [2:7587241303097238255:2112] self: [2:7587241303097238254:0] peer: [5:7587241305559525468:0] socket: 531 qp: -1
2025-12-24T02:11:16.546583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241303097238270:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:16.546590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.546441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525473:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.547610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241305559525454:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.548833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525453:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.548849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305559525393:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.548862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.548869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305559525393:2049] [node 2] ICP22 created new session: [5:7587241305559525474:2049]
2025-12-24T02:11:16.548876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241305559525474:2049] [node 2] ICS09 handshake done sender: [5:7587241305559525454:2109] self: [5:7587241305559525468:0] peer: [2:7587241303097238254:0] socket: 534 qp: -1
2025-12-24T02:11:16.548883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241305559525474:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.548915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.555049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241306107112287:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.556427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241303574031948:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:16.556313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238267:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.559120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194092:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.559137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241302778194092:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.560284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241302778194112:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20805
2025-12-24T02:11:16.560305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241302778194113:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20805
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.560768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194113:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.560791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241302778194092:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.560915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241302778194031:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.560944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.560952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241302778194031:2048] [node 2] ICP22 created new session: [1:7587241302778194116:2048]
2025-12-24T02:11:16.560964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302778194116:2048] [node 2] ICS09 handshake done sender: [1:7587241302778194092:2107] self: [1:7587241302778194091:0] peer: [2:7587241303097238265:0] socket: 522 qp: -1
2025-12-24T02:11:16.560973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302778194116:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:16.561000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.562029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194108:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.562791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241303574031874:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20096386ns) {Configure:104(20094855ns) {ProcessConfigured:135(19690654ns) {StartInitialHandshake:157(19689851ns) {DropHandshakes:470(999ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(931ns) {SwitchToState:179(545ns)}}}}}
2025-12-24T02:11:16.561844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525478:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.563361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238249:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.563374Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241303097238249:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.563838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238261:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.563852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241303097238184:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241303097238244:2107] held: no
2025-12-24T02:11:16.564045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241303097238249:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.565528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194107:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.565543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241302778194107:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.566118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241302778194107:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.566206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241302778194102:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.566221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.566227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241302778194102:2109] [node 5] ICP22 created new session: [1:7587241302778194121:2109]
2025-12-24T02:11:16.566232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302778194121:2109] [node 5] ICS09 handshake done sender: [1:7587241302778194107:2114] self: [1:7587241302778194105:0] peer: [5:7587241305559525479:0] socket: 562 qp: -1
2025-12-24T02:11:16.566243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302778194121:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:16.566251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.565385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525478:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.565414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305559525392:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.566045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.566293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241305559525478:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.566548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305559525392:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.566562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.566569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305559525392:2048] [node 1] ICP22 created new session: [5:7587241305559525481:2048]
2025-12-24T02:11:16.566576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241305559525481:2048] [node 1] ICS09 handshake done sender: [5:7587241305559525478:2118] self: [5:7587241305559525479:0] peer: [1:7587241302778194105:0] socket: 565 qp: -1
2025-12-24T02:11:16.566583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241305559525481:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.566590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.571270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112367:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.571316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112368:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.571350Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.571361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112370:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.571179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241302515941108:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.571445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241302515941109:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27170187ns) {Configure:104(27169592ns) {ProcessConfigured:135(26834750ns) {StartInitialHandshake:157(26834074ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(97ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(538ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.571592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941180:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.571640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941182:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.571820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941169:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.570072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241303097238274:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1621
2025-12-24T02:11:16.575575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241302515941109:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.586911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241303097238185:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.586939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.586948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241303097238185:2049] [node 3] ICP22 created new session: [2:7587241303097238278:2049]
2025-12-24T02:11:16.586955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241303097238278:2049] [node 3] ICS09 handshake done sender: [2:7587241303097238249:2110] self: [2:7587241303097238248:0] peer: [3:7587241306107112366:0] socket: 528 qp: -1
2025-12-24T02:11:16.586964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241303097238278:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.586989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.587344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238253:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.587354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241303097238253:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.588190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241303097238244:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.588298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238264:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.588335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241303097238184:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.588346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.588356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241303097238184:2048] [node 1] ICP22 created new session: [2:7587241303097238281:2048]
2025-12-24T02:11:16.588361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241303097238281:2048] [node 1] ICS09 handshake done sender: [2:7587241303097238244:2107] self: [2:7587241303097238265:0] peer: [1:7587241302778194091:0] socket: 524 qp: -1
2025-12-24T02:11:16.588366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241303097238281:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.588376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.588970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112368:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.589019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112354:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.589023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306107112354:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.588775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941191:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.590599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941180:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.590648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241302515941107:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.590717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194109:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.593831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031952:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.593938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241306107112287:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241306107112346:2107] held: no
2025-12-24T02:11:16.593968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194101:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.593981Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241302778194101:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.594098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031953:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.594564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241302778194101:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.594577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.594669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302515941180:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.594655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241302778194095:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.594673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.594682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241302778194095:2108] [node 4] ICP22 created new session: [1:7587241302778194126:2108]
2025-12-24T02:11:16.594689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302778194126:2108] [node 4] ICS09 handshake done sender: [1:7587241302778194101:2112] self: [1:7587241302778194100:0] peer: [4:7587241302515941192:0] socket: 548 qp: -1
2025-12-24T02:11:16.594741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302778194126:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:16.594750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.594813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303574031952:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.594850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241303574031874:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.594770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241302515941107:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.594795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.594803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241302515941107:2048] [node 1] ICP22 created new session: [4:7587241302515941194:2048]
2025-12-24T02:11:16.594810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302515941194:2048] [node 1] ICS09 handshake done sender: [4:7587241302515941180:2114] self: [4:7587241302515941192:0] peer: [1:7587241302778194100:0] socket: 550 qp: -1
2025-12-24T02:11:16.594818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302515941194:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.594838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.599059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194106:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.599071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241302778194106:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.599731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241302778194106:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.599801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241302778194103:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.599817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.599825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241302778194103:2110] [node 6] ICP22 created new session: [1:7587241302778194129:2110]
2025-12-24T02:11:16.599831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302778194129:2110] [node 6] ICS09 handshake done sender: [1:7587241302778194106:2113] self: [1:7587241302778194104:0] peer: [6:7587241303574031954:0] socket: 568 qp: -1
2025-12-24T02:11:16.599837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302778194129:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:16.599844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.600316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941182:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.601886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241303097238253:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.602152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302515941168:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.602017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241303097238250:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.602035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.602044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241303097238250:2108] [node 4] ICP22 created new session: [2:7587241303097238284:2108]
2025-12-24T02:11:16.602050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241303097238284:2108] [node 4] ICS09 handshake done sender: [2:7587241303097238253:2111] self: [2:7587241303097238252:0] peer: [4:7587241302515941183:0] socket: 529 qp: -1
2025-12-24T02:11:16.602055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241303097238284:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:16.602062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.600217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525483:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.602901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031955:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.602960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031956:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.603166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303574031952:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.602381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241302515941108:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.602400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.602407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241302515941108:2049] [node 2] ICP22 created new session: [4:7587241302515941198:2049]
2025-12-24T02:11:16.602414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302515941198:2049] [node 2] ICS09 handshake done sender: [4:7587241302515941168:2108] self: [4:7587241302515941183:0] peer: [2:7587241303097238252:0] socket: 532 qp: -1
2025-12-24T02:11:16.602423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302515941198:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.602430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.603267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241303574031874:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.603282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.603288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241303574031874:2048] [node 1] ICP22 created new session: [6:7587241303574031957:2048]
2025-12-24T02:11:16.603293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303574031957:2048] [node 1] ICS09 handshake done sender: [6:7587241303574031952:2115] self: [6:7587241303574031954:0] peer: [1:7587241302778194104:0] socket: 569 qp: -1
2025-12-24T02:11:16.603300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303574031957:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.603307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.603671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112367:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.603767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306107112346:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.603840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112371:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.603959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241306107112286:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.604154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241306107112287:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.604171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.604180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241306107112287:2049] [node 2] ICP22 created new session: [3:7587241306107112376:2049]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.604186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306107112376:2049] [node 2] ICS09 handshake done sender: [3:7587241306107112346:2107] self: [3:7587241306107112366:0] peer: [2:7587241303097238248:0] socket: 530 qp: -1
2025-12-24T02:11:16.604195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306107112376:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.604203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.604355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112369:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.604450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306107112354:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.604718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241306107112350:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.604733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.604739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241306107112350:2108] [node 4] ICP22 created new session: [3:7587241306107112380:2108]
2025-12-24T02:11:16.604744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306107112380:2108] [node 4] ICS09 handshake done sender: [3:7587241306107112354:2111] self: [3:7587241306107112353:0] peer: [4:7587241302515941190:0] socket: 537 qp: -1
2025-12-24T02:11:16.604750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306107112380:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:16.604761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.604201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525485:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.607045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241302778194099:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.607062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241302778194099:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.608997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941202:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.609110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241302778194099:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.609215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241302778194032:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.609233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.609238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241302778194032:2049] [node 3] ICP22 created new session: [1:7587241302778194132:2049]
2025-12-24T02:11:16.609244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241302778194132:2049] [node 3] ICS09 handshake done sender: [1:7587241302778194099:2111] self: [1:7587241302778194097:0] peer: [3:7587241306107112375:0] socket: 547 qp: -1
2025-12-24T02:11:16.609250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241302778194132:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.609256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.609583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241302515941201:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:358
2025-12-24T02:11:16.609599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302515941169:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.604285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303574031956:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.606799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241303574031936:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.610760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241302515941109:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.610783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.610792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241302515941109:2050] [node 3] ICP22 created new session: [4:7587241302515941203:2050]
2025-12-24T02:11:16.610799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302515941203:2050] [node 3] ICS09 handshake done sender: [4:7587241302515941169:2109] self: [4:7587241302515941190:0] peer: [3:7587241306107112353:0] socket: 538 qp: -1
2025-12-24T02:11:16.610806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302515941203:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.610813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.612453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112371:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.612658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241306107112350:2108] [node 4] ICP18 handshake (actor [3:7587241306107112371:2120]) is too old
2025-12-24T02:11:16.611203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525472:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.611218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241305559525472:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.612081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525483:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.612198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305559525463:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.612546Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941186:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.612556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241302515941186:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.613062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302515941186:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.613120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241302515941178:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.613131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.613137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241302515941178:2112] [node 5] ICP22 created new session: [4:7587241302515941206:2112]
2025-12-24T02:11:16.613142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302515941206:2112] [node 5] ICS09 handshake done sender: [4:7587241302515941186:2117] self: [4:7587241302515941185:0] peer: [5:7587241305559525487:0] socket: 536 qp: -1
2025-12-24T02:11:16.613146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302515941206:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:16.613152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.614842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241305559525472:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.614900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241305559525485:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.616511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305559525463:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241305559525483:2120] held: no
2025-12-24T02:11:16.616537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241305559525464:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.616550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.616556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305559525464:2111] [node 6] ICP22 created new session: [5:7587241305559525489:2111]
2025-12-24T02:11:16.616563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241305559525489:2111] [node 6] ICS09 handshake done sender: [5:7587241305559525472:2116] self: [5:7587241305559525470:0] peer: [6:7587241303574031960:0] socket: 579 qp: -1
2025-12-24T02:11:16.616569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241305559525489:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:16.616575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.616870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525488:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.621875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303574031953:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.622452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241305559525483:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.622583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305559525463:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.622596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.622604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305559525463:2110] [node 4] ICP22 created new session: [5:7587241305559525491:2110]
2025-12-24T02:11:16.622610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241305559525491:2110] [node 4] ICS09 handshake done sender: [5:7587241305559525483:2120] self: [5:7587241305559525487:0] peer: [4:7587241302515941185:0] socket: 546 qp: -1
2025-12-24T02:11:16.622616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241305559525491:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:16.622619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305559525394:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.622626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.623155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241303574031936:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11757) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241303574031956:2118] held: no
2025-12-24T02:11:16.623181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031961:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.623221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.623241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031963:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.623270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241303574031875:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.623508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303574031962:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.623591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241303574031948:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.623610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941201:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.623691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303574031956:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.623410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241303097238266:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.623424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241303097238266:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.624379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241303097238266:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.624455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241303097238260:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.624468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.624476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241303097238260:2113] [node 6] ICP22 created new session: [2:7587241303097238287:2113]
2025-12-24T02:11:16.624482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241303097238287:2113] [node 6] ICS09 handshake done sender: [2:7587241303097238266:2117] self: [2:7587241303097238262:0] peer: [6:7587241303574031964:0] socket: 575 qp: -1
2025-12-24T02:11:16.624488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241303097238287:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.624497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.630811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241306107112286:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241306107112367:2116] held: no
No RDMA context found for GID: 2025-12-24T02:11:16.631248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241302515941189:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:16.631272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241302515941189:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.632539Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241302515941189:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.632645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241302515941179:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.632664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.632675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241302515941179:2113] [node 6] ICP22 created new session: [4:7587241302515941210:2113]
2025-12-24T02:11:16.632681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241302515941210:2113] [node 6] ICS09 handshake done sender: [4:7587241302515941189:2118] self: [4:7587241302515941188:0] peer: [6:7587241303574031965:0] socket: 587 qp: -1
2025-12-24T02:11:16.632687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241302515941210:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.632696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.632125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241303574031936:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.632152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.632163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241303574031936:2107] [node 5] ICP22 created new session: [6:7587241303574031966:2107]
2025-12-24T02:11:16.632171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303574031966:2107] [node 5] ICS09 handshake done sender: [6:7587241303574031956:2118] self: [6:7587241303574031960:0] peer: [5:7587241305559525470:0] socket: 580 qp: -1
2025-12-24T02:11:16.632181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303574031966:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:16.632192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.632374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112363:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.632386Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306107112363:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.634862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306107112363:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.634899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241306107112382:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41001
2025-12-24T02:11:16.635232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241303574031961:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.639017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241305559525494:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60717
2025-12-24T02:11:16.639115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241306107112360:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.639141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.639149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241306107112360:2112] [node 5] ICP22 created new session: [3:7587241306107112384:2112]
2025-12-24T02:11:16.639157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306107112384:2112] [node 5] ICS09 handshake done sender: [3:7587241306107112363:2114] self: [3:7587241306107112362:0] peer: [5:7587241305559525492:0] socket: 583 qp: -1
2025-12-24T02:11:16.639165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306107112384:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:16.639175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.641527Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241303574031948:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26113) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241303574031962:2120] held: no
2025-12-24T02:11:16.641691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306107112367:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.642492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241303574031876:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.642514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241303574031875:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241303574031953:2116] held: no
2025-12-24T02:11:16.642669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031968:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.642823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031970:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.643728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303574031962:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.642832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241306107112365:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.642841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241306107112365:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.643260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241306107112286:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.643279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.643287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241306107112286:2048] [node 1] ICP22 created new session: [3:7587241306107112386:2048]
2025-12-24T02:11:16.643293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306107112386:2048] [node 1] ICS09 handshake done sender: [3:7587241306107112367:2116] self: [3:7587241306107112375:0] peer: [1:7587241302778194097:0] socket: 549 qp: -1
2025-12-24T02:11:16.643303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306107112386:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.643311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.643868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241303574031971:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:106
2025-12-24T02:11:16.643882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241303574031948:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.643890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.643896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241303574031948:2112] [node 4] ICP22 created new session: [6:7587241303574031972:2112]
2025-12-24T02:11:16.643901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303574031972:2112] [node 4] ICS09 handshake done sender: [6:7587241303574031962:2120] self: [6:7587241303574031965:0] peer: [4:7587241302515941188:0] socket: 525 qp: -1
2025-12-24T02:11:16.643907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303574031972:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:16.643913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.644222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303574031953:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.644349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241306107112365:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.644558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241303574031961:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.644617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241303574031876:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.644631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.644637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241303574031876:2050] [node 3] ICP22 created new session: [6:7587241303574031974:2050]
2025-12-24T02:11:16.644641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303574031974:2050] [node 3] ICS09 handshake done sender: [6:7587241303574031961:2119] self: [6:7587241303574031969:0] peer: [3:7587241306107112364:0] socket: 574 qp: -1
2025-12-24T02:11:16.644647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303574031974:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.644653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.645368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241303574031875:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.645383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.645389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241303574031875:2049] [node 2] ICP22 created new session: [6:7587241303574031976:2049]
2025-12-24T02:11:16.645395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241303574031976:2049] [node 2] ICS09 handshake done sender: [6:7587241303574031953:2116] self: [6:7587241303574031964:0] peer: [2:7587241303097238262:0] socket: 557 qp: -1
2025-12-24T02:11:16.645401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241303574031976:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.645408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.645745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241306107112361:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.645764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.645770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241306107112361:2113] [node 6] ICP22 created new session: [3:7587241306107112389:2113]
2025-12-24T02:11:16.645774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241306107112389:2113] [node 6] ICS09 handshake done sender: [3:7587241306107112365:2115] self: [3:7587241306107112364:0] peer: [6:7587241303574031969:0] socket: 585 qp: -1
2025-12-24T02:11:16.645779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241306107112389:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:16.645784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.647191Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305559525394:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10656) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241305559525485:2121] held: no
2025-12-24T02:11:16.647246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241305559525485:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.647357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305559525394:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.647366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.647373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305559525394:2050] [node 3] ICP22 created new session: [5:7587241305559525495:2050]
2025-12-24T02:11:16.647378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241305559525495:2050] [node 3] ICS09 handshake done sender: [5:7587241305559525485:2121] self: [5:7587241305559525492:0] peer: [3:7587241306107112362:0] socket: 584 qp: -1
2025-12-24T02:11:16.647397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241305559525495:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.647404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.654833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:16.654930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241303097238281:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-12-24T02:11:16.654938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241303097238184:2048] [node 1] ICP30 unregister session Session# [2:7587241303097238281:2048] VirtualId# [2:7587241303097238265:0]
2025-12-24T02:11:16.654945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241303097238281:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.654982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.655003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.655354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:16.655439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241306107112386:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-12-24T02:11:16.655444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241306107112286:2048] [node 1] ICP30 unregister session Session# [3:7587241306107112386:2048] VirtualId# [3:7587241306107112375:0]
2025-12-24T02:11:16.655449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241306107112386:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.655480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.655497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.659691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:16.659763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241302515941194:2048] [node 1] ICS01 socket: 550 reason# EndOfStream
2025-12-24T02:11:16.659770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241302515941107:2048] [node 1] ICP30 unregister session Session# [4:7587241302515941194:2048] VirtualId# [4:7587241302515941192:0]
2025-12-24T02:11:16.659776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241302515941194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.659841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.659872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.660010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:16.660093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241305559525481:2048] [node 1] ICS01 socket: 565 reason# EndOfStream
2025-12-24T02:11:16.660101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305559525392:2048] [node 1] ICP30 unregister session Session# [5:7587241305559525481:2048] VirtualId# [5:7587241305559525479:0]
2025-12-24T02:11:16.660108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241305559525481:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.660160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.660183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.661253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:16.661334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303574031957:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:16.661341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241303574031874:2048] [node 1] ICP30 unregister session Session# [6:7587241303574031957:2048] VirtualId# [6:7587241303574031954:0]
2025-12-24T02:11:16.661347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303574031957:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.661398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.661423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.670819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:16.670920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241306107112376:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-12-24T02:11:16.670928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241306107112287:2049] [node 2] ICP30 unregister session Session# [3:7587241306107112376:2049] VirtualId# [3:7587241306107112366:0]
2025-12-24T02:11:16.670935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241306107112376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.671018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.671039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.688943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:16.689023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241302515941198:2049] [node 2] ICS07 socket disconnect 532 reason# EndOfStream
2025-12-24T02:11:16.689029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241302515941198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.689078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.689099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241302515941198:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:16.689226Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:16.689310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241305559525474:2049] [node 2] ICS07 socket disconnect 534 reason# EndOfStream
2025-12-24T02:11:16.689315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241305559525474:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.689350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.689369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241305559525474:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:16.689568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241302515941108:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.689581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241302515941198:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.689620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241302515941108:2049] [node 2] ICP30 unregister session Session# [4:7587241302515941198:2049] VirtualId# [4:7587241302515941183:0]
2025-12-24T02:11:16.689640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.689646Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241302515941108:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.689648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.690876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:16.690960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303574031976:2049] [node 2] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:16.690968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241303574031875:2049] [node 2] ICP30 unregister session Session# [6:7587241303574031976:2049] VirtualId# [6:7587241303574031964:0]
2025-12-24T02:11:16.690974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303574031976:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.691012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.691042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.692610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305559525393:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.692627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241305559525474:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.692632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305559525393:2049] [node 2] ICP30 unregister session Session# [5:7587241305559525474:2049] VirtualId# [5:7587241305559525468:0]
2025-12-24T02:11:16.692647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.692653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241305559525393:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.692655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21766) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.700525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241302515941108:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:16.705490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241305559525393:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:16.705593Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:16.705670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241305559525495:2050] [node 3] ICS01 socket: 584 reason# EndOfStream
2025-12-24T02:11:16.705677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305559525394:2050] [node 3] ICP30 unregister session Session# [5:7587241305559525495:2050] VirtualId# [5:7587241305559525492:0]
2025-12-24T02:11:16.705685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241305559525495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.705736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.705760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.706783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:16.706860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303574031974:2050] [node 3] ICS01 socket: 574 reason# EndOfStream
2025-12-24T02:11:16.706867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241303574031876:2050] [node 3] ICP30 unregister session Session# [6:7587241303574031974:2050] VirtualId# [6:7587241303574031969:0]
2025-12-24T02:11:16.706872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303574031974:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.706931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.706955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.714773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:16.714872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241303574031972:2112] [node 4] ICS01 socket: 525 reason# EndOfStream
2025-12-24T02:11:16.714881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241303574031948:2112] [node 4] ICP30 unregister session Session# [6:7587241303574031972:2112] VirtualId# [6:7587241303574031965:0]
2025-12-24T02:11:16.714889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241303574031972:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.714948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:16.714986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:16.739163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241305631719743:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.739179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241305631719744:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.740678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304012974961:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.740687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304012974962:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.742253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305315109065:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.742263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305315109066:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.742267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305315109067:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.737528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241303403535354:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:16.737543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241303403535355:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.744064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241306185806277:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.744081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241306185806278:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.744085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241306185806279:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.746675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302790892049:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.746684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302790892050:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.746704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302790892051:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:16.758951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304012975020:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:16.759788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241303403535414:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:16.760590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241303403535423:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:16.760843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241303403535427:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:16.761414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302790892110:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.762481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975028:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.775208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241305631719809:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:16.775315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241305631719810:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.775411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241305631719811:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:16.777282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719808:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
2025-12-24T02:11:16.781260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241304012974962:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18710969ns) {Configure:104(18710351ns) {ProcessConfigured:135(18356164ns) {StartInitialHandshake:157(18355400ns) {DropHandshakes:470(453ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(908ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.781282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241305631719744:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19670685ns) {Configure:104(19668864ns) {ProcessConfigured:135(18200298ns) {StartInitialHandshake:157(18199949ns) {DropHandshakes:470(171ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1186ns) {SwitchToState:179(618ns)}}}}}
2025-12-24T02:11:16.781374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975033:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.781452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241306185806344:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.781718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719822:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.782127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806342:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.782210Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806343:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:16.782881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719823:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
2025-12-24T02:11:16.782777Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241303403535423:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22131842ns) {Configure:104(22131414ns) {ProcessConfigured:135(21896257ns) {StartInitialHandshake:157(21895915ns) {DropHandshakes:470(396ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1039ns) {SwitchToState:179(539ns)}}}}}
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.783298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975038:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.785838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719822:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.787419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305315109132:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:16.787519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305315109133:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:16.794776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304012975032:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:16.795267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241303403535432:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29489
2025-12-24T02:11:16.799118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241303403535427:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38212382ns) {Configure:104(38211586ns) {ProcessConfigured:135(37981736ns) {StartInitialHandshake:157(37981199ns) {DropHandshakes:470(351ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(2370ns) {SwitchToState:179(1401ns)}}}}}
2025-12-24T02:11:16.799228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535432:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.802825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241302790892110:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41323907ns) {Configure:104(41323193ns) {ProcessConfigured:135(40917637ns) {StartInitialHandshake:157(40916811ns) {DropHandshakes:470(432ns) {DropIncomingHandshake:442(37ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(1937ns) {SwitchToState:179(1040ns)}}}}}
2025-12-24T02:11:16.803829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892118:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.804211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241305315109065:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40397851ns) {Configure:104(40397023ns) {ProcessConfigured:135(40175609ns) {StartInitialHandshake:157(40175269ns) {DropHandshakes:470(324ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(1317ns) {SwitchToState:179(1094ns)}}}}}
2025-12-24T02:11:16.806848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241304012975034:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:16.806885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975040:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.807266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241302790892115:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.810726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241306185806340:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:16.810827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241306185806342:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.811049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241306185806279:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.811485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806354:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.811622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975038:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.811655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975025:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.811659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241304012975025:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.812350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304012975025:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.812465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241304012975020:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.812484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.812490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304012975020:2105] [node 5] ICP22 created new session: [3:7587241304012975044:2105]
2025-12-24T02:11:16.812496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304012975044:2105] [node 5] ICS09 handshake done sender: [3:7587241304012975025:2107] self: [3:7587241304012975024:0] peer: [5:7587241306185806355:0] socket: 530 qp: -1
2025-12-24T02:11:16.812503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304012975044:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:16.812512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.812841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241306185806348:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.812849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241306185806348:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.813548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806358:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.813879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241306185806348:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.810725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241302790892051:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19579942ns) {Configure:104(19578830ns) {ProcessConfigured:135(19271119ns) {StartInitialHandshake:157(19270975ns) {DropHandshakes:470(406ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:477(898ns) {SwitchToState:179(461ns)}}}}}
2025-12-24T02:11:16.810784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892119:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.811419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302790892119:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.811448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302790892110:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.814704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241306185806279:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7663) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241306185806342:2110] held: no
2025-12-24T02:11:16.814729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241306185806344:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.814738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.814744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241306185806344:2105] [node 6] ICP22 created new session: [5:7587241306185806360:2105]
2025-12-24T02:11:16.814750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241306185806360:2105] [node 6] ICS09 handshake done sender: [5:7587241306185806348:2113] self: [5:7587241306185806347:0] peer: [6:7587241302790892128:0] socket: 550 qp: -1
2025-12-24T02:11:16.814755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241306185806360:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:16.814764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.814872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241306185806342:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.814997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241306185806279:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.815007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.815011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241306185806279:2050] [node 3] ICP22 created new session: [5:7587241306185806362:2050]
2025-12-24T02:11:16.815014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241306185806362:2050] [node 3] ICS09 handshake done sender: [5:7587241306185806342:2110] self: [5:7587241306185806355:0] peer: [3:7587241304012975024:0] socket: 535 qp: -1
2025-12-24T02:11:16.815017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241306185806362:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.815021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.816735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109143:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.818298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975033:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.819819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975028:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.819855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241304012974961:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.821218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241305631719743:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59734288ns) {Configure:104(59733433ns) {ProcessConfigured:135(58255328ns) {StartInitialHandshake:157(58254890ns) {DropHandshakes:470(490ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(1203ns) {SwitchToState:179(448ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.821466Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241306185806277:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55615586ns) {Configure:104(55614708ns) {ProcessConfigured:135(55266082ns) {StartInitialHandshake:157(55265651ns) {DropHandshakes:470(522ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:477(866ns) {SwitchToState:179(349ns)}}}}}
2025-12-24T02:11:16.822969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719823:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.826807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241304012975032:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31917459ns) {Configure:104(31916498ns) {ProcessConfigured:135(31513747ns) {StartInitialHandshake:157(31512959ns) {DropHandshakes:470(890ns) {DropIncomingHandshake:442(100ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(1052ns) {SwitchToState:179(557ns)}}}}}
2025-12-24T02:11:16.826935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241304012975034:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19998355ns) {Configure:104(19997561ns) {ProcessConfigured:135(19644338ns) {StartInitialHandshake:157(19644009ns) {DropHandshakes:470(649ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(429ns) {SwitchToState:179(273ns)}}}}}
2025-12-24T02:11:16.827148Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241302790892115:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19611433ns) {Configure:104(19609551ns) {ProcessConfigured:135(19355423ns) {StartInitialHandshake:157(19355004ns) {DropHandshakes:470(526ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(875ns) {SwitchToState:179(739ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.831078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109144:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.831098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109145:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.830909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892131:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.830947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892130:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.831310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302790892119:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.831404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302790892110:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.831417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.831423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302790892110:2107] [node 5] ICP22 created new session: [6:7587241302790892132:2107]
2025-12-24T02:11:16.831429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302790892132:2107] [node 5] ICS09 handshake done sender: [6:7587241302790892119:2113] self: [6:7587241302790892128:0] peer: [5:7587241306185806347:0] socket: 557 qp: -1
2025-12-24T02:11:16.831435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302790892132:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:16.831470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.838945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241303403535434:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29489
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.839419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719827:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
2025-12-24T02:11:16.839476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719828:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
2025-12-24T02:11:16.839165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241303403535435:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29489
2025-12-24T02:11:16.839601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719827:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.838995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302790892131:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.839043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302790892051:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.839183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109143:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.839232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305315109065:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.844047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109145:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.843115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535434:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.843165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535421:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.843170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241303403535421:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.844184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719808:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.844217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241305631719743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.844345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241303403535421:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.844607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535417:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.844629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241303403535417:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.846739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975047:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.846764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241304012975047:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.847540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304012975047:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.848512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241304012975032:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.848526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.848532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304012975032:2111] [node 6] ICP22 created new session: [3:7587241304012975052:2111]
2025-12-24T02:11:16.848537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304012975052:2111] [node 6] ICS09 handshake done sender: [3:7587241304012975047:2117] self: [3:7587241304012975039:0] peer: [6:7587241302790892135:0] socket: 574 qp: -1
2025-12-24T02:11:16.848542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304012975052:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:16.848549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.847420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.847726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302790892131:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.847791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302790892051:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.847807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.847814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302790892051:2050] [node 3] ICP22 created new session: [6:7587241302790892137:2050]
2025-12-24T02:11:16.847821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302790892137:2050] [node 3] ICS09 handshake done sender: [6:7587241302790892131:2118] self: [6:7587241302790892135:0] peer: [3:7587241304012975039:0] socket: 575 qp: -1
2025-12-24T02:11:16.847828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302790892137:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.847837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.850845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535419:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.850862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241303403535419:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.851503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241303403535414:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.851527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.851535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241303403535414:2106] [node 4] ICP22 created new session: [1:7587241303403535440:2106]
2025-12-24T02:11:16.851541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241303403535440:2106] [node 4] ICS09 handshake done sender: [1:7587241303403535421:2110] self: [1:7587241303403535420:0] peer: [4:7587241305315109146:0] socket: 533 qp: -1
2025-12-24T02:11:16.851549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241303403535440:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:16.851558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.863230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975049:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.863242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241304012975049:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.863942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304012975049:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.862896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.862906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109150:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.863041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109149:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.863082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109150:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.863107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305315109143:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.863149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305315109067:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.863260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305315109065:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.863277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.863283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305315109065:2048] [node 1] ICP22 created new session: [4:7587241305315109152:2048]
2025-12-24T02:11:16.863288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305315109152:2048] [node 1] ICS09 handshake done sender: [4:7587241305315109143:2115] self: [4:7587241305315109146:0] peer: [1:7587241303403535420:0] socket: 536 qp: -1
2025-12-24T02:11:16.863295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305315109152:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.863302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.863868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241303403535439:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29489
2025-12-24T02:11:16.864005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241304012975034:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.864019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.864027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304012975034:2110] [node 4] ICP22 created new session: [3:7587241304012975056:2110]
2025-12-24T02:11:16.864032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304012975056:2110] [node 4] ICS09 handshake done sender: [3:7587241304012975049:2118] self: [3:7587241304012975041:0] peer: [4:7587241305315109151:0] socket: 558 qp: -1
2025-12-24T02:11:16.864036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304012975056:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:16.864043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.866881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535439:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.866961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241303403535419:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.869498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241305631719831:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12592
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.869671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241305631719808:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.869588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975058:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.869947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241305631719743:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.869892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.869968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.869975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241305631719743:2048] [node 1] ICP22 created new session: [2:7587241305631719832:2048]
2025-12-24T02:11:16.869982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241305631719832:2048] [node 1] ICS09 handshake done sender: [2:7587241305631719808:2110] self: [2:7587241305631719830:0] peer: [1:7587241303403535418:0] socket: 534 qp: -1
2025-12-24T02:11:16.869989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241305631719832:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.870006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.867470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241303403535354:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.867490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.867498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241303403535354:2048] [node 2] ICP22 created new session: [1:7587241303403535444:2048]
2025-12-24T02:11:16.867504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241303403535444:2048] [node 2] ICS09 handshake done sender: [1:7587241303403535419:2109] self: [1:7587241303403535418:0] peer: [2:7587241305631719830:0] socket: 532 qp: -1
2025-12-24T02:11:16.867512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241303403535444:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:16.867519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.870686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975059:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.870047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241306185806343:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.870083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241306185806278:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.873969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241303403535417:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.874794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304012975028:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.875971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241306185806367:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.876018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241306185806277:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.874948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241304012974961:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.874968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.874977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304012974961:2048] [node 1] ICP22 created new session: [3:7587241304012975060:2048]
2025-12-24T02:11:16.874983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304012975060:2048] [node 1] ICS09 handshake done sender: [3:7587241304012975028:2109] self: [3:7587241304012975046:0] peer: [1:7587241303403535416:0] socket: 537 qp: -1
2025-12-24T02:11:16.874990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304012975060:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.875001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.881627Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241305315109065:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18330698ns) {HandleHandshakeStatus:329(18330140ns) {DropIncomingHandshake:442(130ns) DropOutgoingHandshake:457(3724ns) SwitchToState:179(64ns) ProcessPendingSessionEvents:495(14680870ns) {ForwardSessionEventToSession:612(14666865ns) ForwardSessionEventToSession:612(3038ns) ForwardSessionEventToSession:612(1391ns) ForwardSessionEventToSession:612(690ns) ForwardSessionEventToSession:612(454ns) ForwardSessionEventToSession:612(753ns) ForwardSessionEventToSession:612(590ns) ForwardSessionEventToSession:612(560ns) ForwardSessionEventToSession:612(681ns) ForwardSessionEventToSession:612(940ns)}}}
2025-12-24T02:11:16.881671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109144:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.881613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719828:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.881654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719817:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.881657Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241305631719817:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.880661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892140:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.881415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302790892118:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.881453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302790892050:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.882852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241303403535355:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.882884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.882893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241303403535355:2049] [node 3] ICP22 created new session: [1:7587241303403535449:2049]
2025-12-24T02:11:16.882902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241303403535449:2049] [node 3] ICS09 handshake done sender: [1:7587241303403535417:2108] self: [1:7587241303403535416:0] peer: [3:7587241304012975046:0] socket: 531 qp: -1
2025-12-24T02:11:16.882912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241303403535449:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.882928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.883237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719815:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.883255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241305631719815:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.883364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535431:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.883376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241303403535431:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.886841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241305631719817:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.887137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305315109067:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7663) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241305315109150:2120] held: no
2025-12-24T02:11:16.887274Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241305631719743:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17280217ns) {HandleHandshakeStatus:329(17279437ns) {DropIncomingHandshake:442(323ns) DropOutgoingHandshake:457(1872ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(17161050ns) {ForwardSessionEventToSession:612(17152407ns) ForwardSessionEventToSession:612(1366ns) ForwardSessionEventToSession:612(1018ns) ForwardSessionEventToSession:612(383ns) ForwardSessionEventToSession:612(483ns) ForwardSessionEventToSession:612(546ns) ForwardSessionEventToSession:612(480ns) ForwardSessionEventToSession:612(426ns) ForwardSessionEventToSession:612(371ns) ForwardSessionEventToSession:612(794ns)}}}
2025-12-24T02:11:16.887837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302790892140:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.887919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302790892049:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.888233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241305631719815:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.888586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535435:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.888621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241303403535433:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.888625Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241303403535433:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.889606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241303403535431:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.889643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109149:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.889713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305315109066:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.889734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109154:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.889673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241305631719811:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.889690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.889696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241305631719811:2109] [node 6] ICP22 created new session: [2:7587241305631719838:2109]
2025-12-24T02:11:16.889701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241305631719838:2109] [node 6] ICS09 handshake done sender: [2:7587241305631719817:2113] self: [2:7587241305631719816:0] peer: [6:7587241302790892141:0] socket: 544 qp: -1
2025-12-24T02:11:16.889706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241305631719838:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:16.889712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.889876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241303403535433:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:16.889867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305315109150:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.889947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241305631719810:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.889974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.889977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241305631719810:2108] [node 5] ICP22 created new session: [2:7587241305631719840:2108]
2025-12-24T02:11:16.889979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241305631719840:2108] [node 5] ICS09 handshake done sender: [2:7587241305631719815:2112] self: [2:7587241305631719814:0] peer: [5:7587241306185806368:0] socket: 543 qp: -1
2025-12-24T02:11:16.889983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241305631719840:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:16.889986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.890846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241303403535427:2111] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:16.890861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-12-24T02:11:16.890867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241303403535427:2111] [node 6] ICP22 created new session: [1:7587241303403535454:2111]
2025-12-24T02:11:16.890871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241303403535454:2111] [node 6] ICS09 handshake done sender: [1:7587241303403535433:2115] self: [1:7587241303403535429:0] peer: [6:7587241302790892142:0] socket: 591 qp: -1
2025-12-24T02:11:16.890876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241303403535454:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:16.890881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.890827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241304012975059:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.890914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241304012974962:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.891019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305315109067:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.891039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.891046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305315109067:2050] [node 3] ICP22 created new session: [4:7587241305315109159:2050]
2025-12-24T02:11:16.891052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305315109159:2050] [node 3] ICS09 handshake done sender: [4:7587241305315109150:2120] self: [4:7587241305315109151:0] peer: [3:7587241304012975041:0] socket: 583 qp: -1
2025-12-24T02:11:16.891057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305315109159:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:16.891065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:16.891656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241303403535423:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.891683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.891691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241303403535423:2107] [node 5] ICP22 created new session: [1:7587241303403535456:2107]
2025-12-24T02:11:16.891697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241303403535456:2107] [node 5] ICS09 handshake done sender: [1:7587241303403535431:2113] self: [1:7587241303403535428:0] peer: [5:7587241306185806369:0] socket: 587 qp: -1
2025-12-24T02:11:16.891704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241303403535456:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:16.891716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.892004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719821:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.892016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241305631719821:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.892794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241305631719821:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:16.892945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241305631719744:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.892968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:16.892976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241305631719744:2049] [node 3] ICP22 created new session: [2:7587241305631719843:2049]
2025-12-24T02:11:16.892981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241305631719813:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.892989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241305631719843:2049] [node 3] ICS09 handshake done sender: [2:7587241305631719821:2114] self: [2:7587241305631719807:0] peer: [3:7587241304012975062:0] socket: 541 qp: -1
2025-12-24T02:11:16.892994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241305631719813:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.892996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241305631719843:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:16.893005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.894335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241302790892050:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241302790892118:2112] held: no
2025-12-24T02:11:16.894369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241302790892049:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241302790892140:2120] held: no
2025-12-24T02:11:16.891833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806371:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.891965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806370:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.895292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241305631719813:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.895412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241305631719809:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.895458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.895463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241305631719809:2107] [node 4] ICP22 created new session: [2:7587241305631719846:2107]
2025-12-24T02:11:16.895469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241305631719846:2107] [node 4] ICS09 handshake done sender: [2:7587241305631719813:2111] self: [2:7587241305631719812:0] peer: [4:7587241305315109156:0] socket: 542 qp: -1
2025-12-24T02:11:16.895474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241305631719846:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:16.895481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.898072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892143:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.898607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241305315109161:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50707
2025-12-24T02:11:16.895146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241306185806278:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241306185806343:2111] held: no
2025-12-24T02:11:16.895195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241306185806367:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.895366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241306185806343:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.894845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241304012974962:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12337) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241304012975059:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:16.898807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305315109144:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.898977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305315109066:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.898997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.899006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305315109066:2049] [node 2] ICP22 created new session: [4:7587241305315109162:2049]
2025-12-24T02:11:16.899012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305315109162:2049] [node 2] ICS09 handshake done sender: [4:7587241305315109144:2116] self: [4:7587241305315109156:0] peer: [2:7587241305631719812:0] socket: 549 qp: -1
2025-12-24T02:11:16.899024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305315109162:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.899030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.899746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892144:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.899756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892145:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.899935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109138:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.899947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241305315109138:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.900348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241302790892145:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.900364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302790892118:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.900673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302790892140:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:16.901740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305315109138:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:16.901715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302790892050:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.901735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302790892049:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.901736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.901750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.901752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302790892050:2049] [node 2] ICP22 created new session: [6:7587241302790892146:2049]
2025-12-24T02:11:16.901758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302790892049:2048] [node 1] ICP22 created new session: [6:7587241302790892147:2048]
2025-12-24T02:11:16.901761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302790892146:2049] [node 2] ICS09 handshake done sender: [6:7587241302790892118:2112] self: [6:7587241302790892141:0] peer: [2:7587241305631719816:0] socket: 556 qp: -1
2025-12-24T02:11:16.901764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302790892147:2048] [node 1] ICS09 handshake done sender: [6:7587241302790892140:2120] self: [6:7587241302790892142:0] peer: [1:7587241303403535429:0] socket: 528 qp: -1
2025-12-24T02:11:16.901770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302790892146:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.901771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302790892147:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.901779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.901779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.901790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241302790892115:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.901899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241305315109132:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.901925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:16.898621Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241306185806277:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12659) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241306185806367:2119] held: no
2025-12-24T02:11:16.901927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241305315109140:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.901932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305315109132:2109] [node 5] ICP22 created new session: [4:7587241305315109165:2109]
2025-12-24T02:11:16.901932Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241305315109140:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:16.898640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241306185806277:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.901939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305315109165:2109] [node 5] ICS09 handshake done sender: [4:7587241305315109138:2113] self: [4:7587241305315109137:0] peer: [5:7587241306185806377:0] socket: 595 qp: -1
2025-12-24T02:11:16.898651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:16.901946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305315109165:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:16.898657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241306185806277:2048] [node 1] ICP22 created new session: [5:7587241306185806373:2048]
2025-12-24T02:11:16.901954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:16.898663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241306185806373:2048] [node 1] ICS09 handshake done sender: [5:7587241306185806367:2119] self: [5:7587241306185806369:0] peer: [1:7587241303403535428:0] socket: 588 qp: -1
2025-12-24T02:11:16.898670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241306185806373:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:16.898703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:16.899776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241304012975063:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61213
2025-12-24T02:11:16.899893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241304012975059:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:16.900726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241304012974962:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.900743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.900750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241304012974962:2049] [node 2] ICP22 created new session: [3:7587241304012975064:2049]
2025-12-24T02:11:16.900758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241304012975064:2049] [node 2] ICS09 handshake done sender: [3:7587241304012975059:2120] self: [3:7587241304012975062:0] peer: [2:7587241305631719807:0] socket: 593 qp: -1
2025-12-24T02:11:16.900765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241304012975064:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.900772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:16.902847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241305315109140:2114] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:16.902887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241302790892151:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13
2025-12-24T02:11:16.902924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241305315109133:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:16.902945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:16.902951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305315109133:2110] [node 6] ICP22 created new session: [4:7587241305315109168:2110]
2025-12-24T02:11:16.902957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241305315109168:2110] [node 6] ICS09 handshake done sender: [4:7587241305315109140:2114] self: [4:7587241305315109139:0] peer: [6:7587241302790892150:0] socket: 596 qp: -1
2025-12-24T02:11:16.902963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241305315109168:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:16.902969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:16.903132Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241302790892145:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.903288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241302790892115:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.903315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.903322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241302790892115:2109] [node 4] ICP22 created new session: [6:7587241302790892152:2109]
2025-12-24T02:11:16.903328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241302790892152:2109] [node 4] ICS09 handshake done sender: [6:7587241302790892145:2123] self: [6:7587241302790892150:0] peer: [4:7587241305315109139:0] socket: 563 qp: -1
2025-12-24T02:11:16.903335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241302790892152:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:16.903348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.898831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806372:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.898862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241306185806278:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.898872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:16.898878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241306185806278:2049] [node 2] ICP22 created new session: [5:7587241306185806375:2049]
2025-12-24T02:11:16.898885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241306185806375:2049] [node 2] ICS09 handshake done sender: [5:7587241306185806343:2111] self: [5:7587241306185806368:0] peer: [2:7587241305631719814:0] socket: 546 qp: -1
2025-12-24T02:11:16.898890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241306185806375:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:16.898896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:16.899144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241306185806372:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:16.899167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241306185806340:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:16.902845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241306185806378:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55916
2025-12-24T02:11:16.902989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241306185806372:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:16.903078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241306185806340:2104] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:16.903087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:16.903092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241306185806340:2104] [node 4] ICP22 created new session: [5:7587241306185806379:2104]
2025-12-24T02:11:16.903098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241306185806379:2104] [node 4] ICS09 handshake done sender: [5:7587241306185806372:2122] self: [5:7587241306185806377:0] peer: [4:7587241305315109137:0] socket: 599 qp: -1
2025-12-24T02:11:16.903103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241306185806379:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:16.903108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:16.914753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:16.914855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241305631719832:2048] [node 1] ICS07 socket disconnect 534 reason# EndOfStream
2025-12-24T02:11:16.914861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241305631719832:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.914937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.914963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241305631719832:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.920016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:16.920099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241305315109152:2048] [node 1] ICS07 socket disconnect 536 reason# EndOfStream
2025-12-24T02:11:16.920106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241305315109152:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.920163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.920192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241305315109152:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:16.920401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:16.920456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302790892147:2048] [node 1] ICS01 socket: 528 reason# EndOfStream
2025-12-24T02:11:16.920461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302790892049:2048] [node 1] ICP30 unregister session Session# [6:7587241302790892147:2048] VirtualId# [6:7587241302790892142:0]
2025-12-24T02:11:16.920465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302790892147:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.920508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.920527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.920589Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305315109065:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12659) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:16.920603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241305315109152:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:16.920608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305315109065:2048] [node 1] ICP30 unregister session Session# [4:7587241305315109152:2048] VirtualId# [4:7587241305315109146:0]
2025-12-24T02:11:16.920624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.920628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241305315109065:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12659) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.920631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12659) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:16.920260Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:16.920331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241306185806373:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:16.918831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:16.918925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241304012975060:2048] [node 1] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:16.918935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241304012974961:2048] [node 1] ICP30 unregister session Session# [3:7587241304012975060:2048] VirtualId# [3:7587241304012975046:0]
2025-12-24T02:11:16.918943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241304012975060:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.919001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.919030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.920338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241306185806277:2048] [node 1] ICP30 unregister session Session# [5:7587241306185806373:2048] VirtualId# [5:7587241306185806369:0]
2025-12-24T02:11:16.920343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241306185806373:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.920392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:16.920414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:16.933760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:16.933844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241305315109162:2049] [node 2] ICS01 socket: 549 reason# EndOfStream
2025-12-24T02:11:16.933853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305315109066:2049] [node 2] ICP30 unregister session Session# [4:7587241305315109162:2049] VirtualId# [4:7587241305315109156:0]
2025-12-24T02:11:16.933860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241305315109162:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.933917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.933944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.934234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:16.934317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302790892146:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:16.934324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302790892050:2049] [node 2] ICP30 unregister session Session# [6:7587241302790892146:2049] VirtualId# [6:7587241302790892141:0]
2025-12-24T02:11:16.934330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302790892146:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.934392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.934416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.932112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:16.932215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241304012975064:2049] [node 2] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:16.932229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241304012974962:2049] [node 2] ICP30 unregister session Session# [3:7587241304012975064:2049] VirtualId# [3:7587241304012975062:0]
2025-12-24T02:11:16.932237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241304012975064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.932309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.932332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.934186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:16.934523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241306185806375:2049] [node 2] ICS01 socket: 546 reason# EndOfStream
2025-12-24T02:11:16.934530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241306185806278:2049] [node 2] ICP30 unregister session Session# [5:7587241306185806375:2049] VirtualId# [5:7587241306185806368:0]
2025-12-24T02:11:16.934535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241306185806375:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.934568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:16.934588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:16.943752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241305315109065:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:16.943761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:16.943918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:16.943992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302790892137:2050] [node 3] ICS01 socket: 575 reason# EndOfStream
2025-12-24T02:11:16.943848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241305315109159:2050] [node 3] ICS01 socket: 583 reason# EndOfStream
2025-12-24T02:11:16.943854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305315109067:2050] [node 3] ICP30 unregister session Session# [4:7587241305315109159:2050] VirtualId# [4:7587241305315109151:0]
2025-12-24T02:11:16.943998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302790892051:2050] [node 3] ICP30 unregister session Session# [6:7587241302790892137:2050] VirtualId# [6:7587241302790892135:0]
2025-12-24T02:11:16.943860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241305315109159:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.944004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302790892137:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.943917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.943953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.944058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.944100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.943420Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:16.943561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241306185806362:2050] [node 3] ICS01 socket: 535 reason# EndOfStream
2025-12-24T02:11:16.943572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241306185806279:2050] [node 3] ICP30 unregister session Session# [5:7587241306185806362:2050] VirtualId# [5:7587241306185806355:0]
2025-12-24T02:11:16.943581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241306185806362:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.943650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:16.943681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:16.965358Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:16.965451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302790892152:2109] [node 4] ICS01 socket: 563 reason# EndOfStream
2025-12-24T02:11:16.965460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302790892115:2109] [node 4] ICP30 unregister session Session# [6:7587241302790892152:2109] VirtualId# [6:7587241302790892150:0]
2025-12-24T02:11:16.965467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302790892152:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.965530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:16.965558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:16.980481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:16.980583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241302790892132:2107] [node 5] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:16.980595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241302790892110:2107] [node 5] ICP30 unregister session Session# [6:7587241302790892132:2107] VirtualId# [6:7587241302790892128:0]
2025-12-24T02:11:16.980602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241302790892132:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:16.980673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:16.980703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:16.993074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304810653097:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:16.993075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241304810653096:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:16.996520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241302796820769:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.996547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241302796820770:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.998153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305778719784:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.998161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305778719785:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.998166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241305778719786:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.000066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305461466865:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.000076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305461466866:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.000081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241305461466867:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.003273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241307500467630:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.003288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241307500467632:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.003317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241307500467631:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:16.994825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306441402234:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:16.994838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241306441402235:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.014732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241307500467692:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.015236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241309105620455:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.015974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241309105620460:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:17.015995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241309105620461:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.017031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687138:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.018386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434222:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.023423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687138:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.023908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310073687142:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:17.024143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241307091788129:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:17.024219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241307091788130:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:17.024273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241307091788131:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:17.024197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434223:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.030865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434223:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.031719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241307500467700:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:17.037038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310073687144:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:17.034935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310736369599:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:17.036582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310736369595:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.042524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310736369603:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37491
2025-12-24T02:11:17.042546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310736369604:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37491
2025-12-24T02:11:17.042557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310736369602:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37491
2025-12-24T02:11:17.043882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241307091788130:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18979038ns) {Configure:104(12402359ns) {ProcessConfigured:135(11990567ns) {StartInitialHandshake:157(11989793ns) {DropHandshakes:470(670ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1086ns) {SwitchToState:179(607ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.044051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788138:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.044070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788139:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.044122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788140:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.044618Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241305461466865:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19264162ns) {Configure:104(19262509ns) {ProcessConfigured:135(17465988ns) {StartInitialHandshake:157(17465309ns) {DropHandshakes:470(522ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(129ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(808ns) {SwitchToState:179(582ns)}}}}}
2025-12-24T02:11:17.044663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305461466865:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.044714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434222:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.044903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434237:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.044922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434238:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.045884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620466:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.045897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241309105620466:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.046551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305778719784:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.046677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687156:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
2025-12-24T02:11:17.046711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687157:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
2025-12-24T02:11:17.046778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620463:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.046786Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241309105620463:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.046813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241309105620466:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.050740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241309105620460:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.050769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.050780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241309105620460:2109] [node 5] ICP22 created new session: [1:7587241309105620472:2109]
2025-12-24T02:11:17.050789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241309105620472:2109] [node 5] ICS09 handshake done sender: [1:7587241309105620466:2112] self: [1:7587241309105620465:0] peer: [5:7587241309756434236:0] socket: 537 qp: -1
2025-12-24T02:11:17.050799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241309105620472:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:17.050837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.050990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788139:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.050792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241307500467630:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24819759ns) {Configure:104(24818966ns) {ProcessConfigured:135(24455088ns) {StartInitialHandshake:157(24454153ns) {DropHandshakes:470(720ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(1345ns) {SwitchToState:179(890ns)}}}}}
2025-12-24T02:11:17.050803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309756434233:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:17.051011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309756434234:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:17.051780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310736369598:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.051869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369602:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.052272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241306441402234:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.053439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687156:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.052655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241302796820770:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32038748ns) {Configure:104(32037939ns) {ProcessConfigured:135(31844797ns) {StartInitialHandshake:157(31844582ns) {DropHandshakes:470(380ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:477(888ns) {SwitchToState:179(306ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.054325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434242:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.054915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620454:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.054931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241309105620454:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.056665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788140:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.056713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241302796820770:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.056775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241309105620454:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.056876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304810653096:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.056891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.056897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304810653096:2048] [node 2] ICP22 created new session: [1:7587241309105620476:2048]
2025-12-24T02:11:17.056902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241309105620476:2048] [node 2] ICS09 handshake done sender: [1:7587241309105620454:2108] self: [1:7587241309105620453:0] peer: [2:7587241310736369609:0] socket: 534 qp: -1
2025-12-24T02:11:17.056907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241309105620476:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:17.056921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.053833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241310736369599:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18834843ns) {Configure:104(18834401ns) {ProcessConfigured:135(18422368ns) {StartInitialHandshake:157(18422028ns) {DropHandshakes:470(384ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(1013ns) {SwitchToState:179(79ns)}}}}}
2025-12-24T02:11:17.056572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310736369613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37491
2025-12-24T02:11:17.056832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369597:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.056837Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310736369597:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.058730Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241307500467631:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32343363ns) {Configure:104(32342756ns) {ProcessConfigured:135(32080218ns) {StartInitialHandshake:157(32079874ns) {DropHandshakes:470(279ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(810ns) {SwitchToState:179(481ns)}}}}}
2025-12-24T02:11:17.058747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310736369602:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.058771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687157:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.059155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.058983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241309105620463:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.059297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
2025-12-24T02:11:17.059427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310073687138:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.059719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241309105620479:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18203
2025-12-24T02:11:17.059868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241302796820769:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30797678ns) {Configure:104(30796203ns) {ProcessConfigured:135(30378926ns) {StartInitialHandshake:157(30378308ns) {DropHandshakes:470(951ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(442ns) PrepareNewSessionHandshake:477(1259ns) {SwitchToState:179(672ns)}}}}}
2025-12-24T02:11:17.059084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434243:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.059233Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309756434223:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.059312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434237:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.059475Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434238:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.059520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309756434234:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.059535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305461466867:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.060129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434243:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.060154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241305461466866:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.060743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788138:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.060782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305461466865:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.060806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.060814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305461466865:2048] [node 1] ICP22 created new session: [5:7587241309756434250:2048]
2025-12-24T02:11:17.060820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309756434250:2048] [node 1] ICS09 handshake done sender: [5:7587241309756434223:2109] self: [5:7587241309756434236:0] peer: [1:7587241309105620465:0] socket: 542 qp: -1
2025-12-24T02:11:17.060828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309756434250:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.060848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.058956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369604:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.058975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241306441402234:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.058988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.058994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306441402234:2048] [node 1] ICP22 created new session: [2:7587241310736369614:2048]
2025-12-24T02:11:17.059001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310736369614:2048] [node 1] ICS09 handshake done sender: [2:7587241310736369602:2112] self: [2:7587241310736369609:0] peer: [1:7587241309105620453:0] socket: 543 qp: -1
2025-12-24T02:11:17.059007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310736369614:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.059023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.062861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241307091788131:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26888372ns) {Configure:104(26886813ns) {ProcessConfigured:135(26582800ns) {StartInitialHandshake:157(26582343ns) {DropHandshakes:470(449ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(520ns) PrepareNewSessionHandshake:477(814ns) {SwitchToState:179(275ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.063920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467709:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.064561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467708:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.064638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788137:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.064649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241307091788137:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.066260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241309105620455:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.066285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.066294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241309105620455:2107] [node 4] ICP22 created new session: [1:7587241309105620481:2107]
2025-12-24T02:11:17.066302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241309105620481:2107] [node 4] ICS09 handshake done sender: [1:7587241309105620463:2111] self: [1:7587241309105620459:0] peer: [4:7587241310073687158:0] socket: 536 qp: -1
2025-12-24T02:11:17.066308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241309105620481:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:17.066317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.066655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788135:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.066667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241307091788135:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.062876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687152:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.062888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241310073687152:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.063350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310073687152:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.063541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241310073687144:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.063551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.063557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310073687144:2109] [node 5] ICP22 created new session: [4:7587241310073687164:2109]
2025-12-24T02:11:17.063563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241310073687164:2109] [node 5] ICS09 handshake done sender: [4:7587241310073687152:2112] self: [4:7587241310073687151:0] peer: [5:7587241309756434247:0] socket: 548 qp: -1
2025-12-24T02:11:17.063569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241310073687164:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:17.063576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.064026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241305778719786:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26926817ns) {Configure:104(26926006ns) {ProcessConfigured:135(26659812ns) {StartInitialHandshake:157(26659214ns) {DropHandshakes:470(429ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(867ns) {SwitchToState:179(341ns)}}}}}
2025-12-24T02:11:17.064058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305778719786:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.065630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687169:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.066719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241307500467708:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.066782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241307500467630:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.067038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788151:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.064727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369608:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.064745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310736369608:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.064912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310736369597:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.065204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241306441402235:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.065216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.065225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241306441402235:2049] [node 3] ICP22 created new session: [2:7587241310736369619:2049]
2025-12-24T02:11:17.065231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310736369619:2049] [node 3] ICS09 handshake done sender: [2:7587241310736369597:2111] self: [2:7587241310736369594:0] peer: [3:7587241307091788146:0] socket: 554 qp: -1
2025-12-24T02:11:17.065238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310736369619:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.065244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.070831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310736369608:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.070896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369603:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.070850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241307091788140:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.070999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305778719784:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.071040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.071051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305778719784:2048] [node 1] ICP22 created new session: [4:7587241310073687170:2048]
2025-12-24T02:11:17.071063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241310073687170:2048] [node 1] ICS09 handshake done sender: [4:7587241310073687138:2105] self: [4:7587241310073687158:0] peer: [1:7587241309105620459:0] socket: 540 qp: -1
2025-12-24T02:11:17.071076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241310073687170:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.071104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.073365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788147:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.073384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241302796820770:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.073571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.073585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241302796820770:2049] [node 2] ICP22 created new session: [3:7587241307091788152:2049]
2025-12-24T02:11:17.073597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241307091788152:2049] [node 2] ICS09 handshake done sender: [3:7587241307091788140:2114] self: [3:7587241307091788146:0] peer: [2:7587241310736369594:0] socket: 559 qp: -1
2025-12-24T02:11:17.073630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241307091788152:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.073639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.074723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310736369598:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.074751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.074759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310736369598:2108] [node 5] ICP22 created new session: [2:7587241310736369624:2108]
2025-12-24T02:11:17.074764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310736369624:2108] [node 5] ICS09 handshake done sender: [2:7587241310736369608:2115] self: [2:7587241310736369607:0] peer: [5:7587241309756434249:0] socket: 572 qp: -1
2025-12-24T02:11:17.074771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310736369624:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:17.074778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.074859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687169:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.074884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241309105620483:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18203
2025-12-24T02:11:17.074944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241305778719785:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.075036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620469:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.075049Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241309105620469:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.075135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241309105620484:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18203
2025-12-24T02:11:17.075165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241307500467709:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.075225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241307500467700:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.075314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434252:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.075481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309756434237:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.075661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620470:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.075669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241309105620470:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.075320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310073687160:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.075329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241310073687160:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.075903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310073687160:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.075524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241302796820769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.076261Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241305778719784:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16540720ns) {HandleHandshakeStatus:329(16540101ns) {DropIncomingHandshake:442(981ns) DropOutgoingHandshake:457(12199ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(5061913ns) {ForwardSessionEventToSession:612(5051086ns) ForwardSessionEventToSession:612(1520ns) ForwardSessionEventToSession:612(1405ns) ForwardSessionEventToSession:612(605ns) ForwardSessionEventToSession:612(506ns) ForwardSessionEventToSession:612(1011ns) ForwardSessionEventToSession:612(613ns) ForwardSessionEventToSession:612(547ns) ForwardSessionEventToSession:612(482ns) ForwardSessionEventToSession:612(787ns)}}}
2025-12-24T02:11:17.076402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241310073687142:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.076425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.076437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310073687142:2108] [node 6] ICP22 created new session: [4:7587241310073687174:2108]
2025-12-24T02:11:17.076349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241309105620485:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18203
2025-12-24T02:11:17.076442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241310073687174:2108] [node 6] ICS09 handshake done sender: [4:7587241310073687160:2116] self: [4:7587241310073687148:0] peer: [6:7587241307500467711:0] socket: 587 qp: -1
2025-12-24T02:11:17.076458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241310073687174:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:17.076466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.076597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241307091788135:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.076773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241307091788137:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.076943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369611:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.076956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310736369611:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.077020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241307091788129:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.077033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.077039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241307091788129:2106] [node 4] ICP22 created new session: [3:7587241307091788157:2106]
2025-12-24T02:11:17.077079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241307091788157:2106] [node 4] ICS09 handshake done sender: [3:7587241307091788135:2110] self: [3:7587241307091788132:0] peer: [4:7587241310073687168:0] socket: 560 qp: -1
2025-12-24T02:11:17.077087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241307091788157:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:17.077070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241309105620485:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.077099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.077351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241307091788130:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.077368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.077374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241307091788130:2107] [node 5] ICP22 created new session: [3:7587241307091788159:2107]
2025-12-24T02:11:17.077384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241307091788159:2107] [node 5] ICS09 handshake done sender: [3:7587241307091788137:2111] self: [3:7587241307091788134:0] peer: [5:7587241309756434248:0] socket: 561 qp: -1
2025-12-24T02:11:17.077390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241307091788159:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:17.077412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.077463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310736369611:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.077554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310736369595:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.077570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.077574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310736369595:2107] [node 4] ICP22 created new session: [2:7587241310736369627:2107]
2025-12-24T02:11:17.077579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310736369627:2107] [node 4] ICS09 handshake done sender: [2:7587241310736369611:2116] self: [2:7587241310736369601:0] peer: [4:7587241310073687172:0] socket: 591 qp: -1
2025-12-24T02:11:17.077584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310736369627:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:17.077589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.078457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241309105620469:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.078734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467714:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.078886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241304810653096:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.079042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241309105620461:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.079054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.079061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241309105620461:2110] [node 6] ICP22 created new session: [1:7587241309105620488:2110]
2025-12-24T02:11:17.079071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241309105620488:2110] [node 6] ICS09 handshake done sender: [1:7587241309105620469:2113] self: [1:7587241309105620467:0] peer: [6:7587241307500467710:0] socket: 576 qp: -1
2025-12-24T02:11:17.079077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241309105620488:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:17.079094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.080545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467712:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:17.080542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305461466867:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241309756434238:2116] held: no
2025-12-24T02:11:17.080644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467713:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.080803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241307500467712:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.080817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241307500467709:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.080839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241307500467631:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.082262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241309105620470:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.082811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309756434234:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.082866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.082876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309756434234:2110] [node 4] ICP22 created new session: [5:7587241309756434255:2110]
2025-12-24T02:11:17.082883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309756434255:2110] [node 4] ICS09 handshake done sender: [5:7587241309756434237:2115] self: [5:7587241309756434247:0] peer: [4:7587241310073687151:0] socket: 564 qp: -1
2025-12-24T02:11:17.082892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309756434255:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:17.082920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.082944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241307500467700:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.082963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.082970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241307500467700:2110] [node 4] ICP22 created new session: [6:7587241307500467717:2110]
2025-12-24T02:11:17.082977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241307500467717:2110] [node 4] ICS09 handshake done sender: [6:7587241307500467709:2116] self: [6:7587241307500467711:0] peer: [4:7587241310073687148:0] socket: 588 qp: -1
2025-12-24T02:11:17.082983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241307500467717:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:17.082991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.083081Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241307500467700:2110] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.083160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434254:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.083280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467716:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.083498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241304810653097:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.083534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.083547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241304810653097:2049] [node 3] ICP22 created new session: [1:7587241309105620491:2049]
2025-12-24T02:11:17.083561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241309105620491:2049] [node 3] ICS09 handshake done sender: [1:7587241309105620470:2114] self: [1:7587241309105620458:0] peer: [3:7587241307091788156:0] socket: 581 qp: -1
2025-12-24T02:11:17.083568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241309105620491:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.083580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.083623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309756434238:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.083718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241309105620460:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.083737Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241309105620455:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.084054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369612:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.084071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310736369612:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.084048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309756434253:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20242
2025-12-24T02:11:17.084414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241302796820769:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241307091788147:2117] held: no
2025-12-24T02:11:17.084437Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241307500467708:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.084560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305461466866:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29586) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241309756434243:2118] held: no
2025-12-24T02:11:17.084666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305461466867:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.084681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.084687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305461466867:2050] [node 3] ICP22 created new session: [5:7587241309756434259:2050]
2025-12-24T02:11:17.084693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309756434259:2050] [node 3] ICS09 handshake done sender: [5:7587241309756434238:2116] self: [5:7587241309756434248:0] peer: [3:7587241307091788134:0] socket: 565 qp: -1
2025-12-24T02:11:17.084699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309756434259:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.084706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.084859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309756434243:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.085072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241305461466866:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.085103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.085116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241305461466866:2049] [node 2] ICP22 created new session: [5:7587241309756434261:2049]
2025-12-24T02:11:17.085123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309756434261:2049] [node 2] ICS09 handshake done sender: [5:7587241309756434243:2118] self: [5:7587241309756434249:0] peer: [2:7587241310736369607:0] socket: 574 qp: -1
2025-12-24T02:11:17.085132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309756434261:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.085137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.085187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310736369612:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.085277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310736369599:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.085293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.085305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310736369599:2109] [node 6] ICP22 created new session: [2:7587241310736369630:2109]
2025-12-24T02:11:17.085315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310736369630:2109] [node 6] ICS09 handshake done sender: [2:7587241310736369612:2117] self: [2:7587241310736369600:0] peer: [6:7587241307500467715:0] socket: 596 qp: -1
2025-12-24T02:11:17.085327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310736369630:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:17.085348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.085585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241307500467630:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.085601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.085614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241307500467630:2048] [node 1] ICP22 created new session: [6:7587241307500467719:2048]
2025-12-24T02:11:17.085620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241307500467719:2048] [node 1] ICS09 handshake done sender: [6:7587241307500467708:2115] self: [6:7587241307500467710:0] peer: [1:7587241309105620467:0] socket: 577 qp: -1
2025-12-24T02:11:17.085632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241307500467719:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.085650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.086212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241307500467714:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.086612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241307500467632:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.084047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687176:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
2025-12-24T02:11:17.086670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788162:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.084135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310073687177:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52534
2025-12-24T02:11:17.086704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241307091788163:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35109
2025-12-24T02:11:17.084362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305778719785:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29586) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241310073687169:2119] held: no
2025-12-24T02:11:17.084421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310073687169:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.084532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310073687156:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.087381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305778719785:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.087411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.087416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305778719785:2049] [node 2] ICP22 created new session: [4:7587241310073687179:2049]
2025-12-24T02:11:17.087420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241310073687179:2049] [node 2] ICS09 handshake done sender: [4:7587241310073687169:2119] self: [4:7587241310073687172:0] peer: [2:7587241310736369601:0] socket: 592 qp: -1
2025-12-24T02:11:17.087426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241310073687179:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.087432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.087542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241305778719786:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.087555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.087559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241305778719786:2050] [node 3] ICP22 created new session: [4:7587241310073687181:2050]
2025-12-24T02:11:17.087564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241310073687181:2050] [node 3] ICS09 handshake done sender: [4:7587241310073687156:2114] self: [4:7587241310073687168:0] peer: [3:7587241307091788132:0] socket: 568 qp: -1
2025-12-24T02:11:17.087569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241310073687181:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.087573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.087334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241307091788147:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.087337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241307091788149:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.087342Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241307091788149:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.087417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241302796820769:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.087427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.087433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241302796820769:2048] [node 1] ICP22 created new session: [3:7587241307091788164:2048]
2025-12-24T02:11:17.087440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241307091788164:2048] [node 1] ICS09 handshake done sender: [3:7587241307091788147:2117] self: [3:7587241307091788156:0] peer: [1:7587241309105620458:0] socket: 582 qp: -1
2025-12-24T02:11:17.087446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241307091788164:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.087452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.088081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241307091788149:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.088175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241307091788131:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.088202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.088214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241307091788131:2108] [node 6] ICP22 created new session: [3:7587241307091788167:2108]
2025-12-24T02:11:17.088221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241307091788167:2108] [node 6] ICS09 handshake done sender: [3:7587241307091788149:2118] self: [3:7587241307091788136:0] peer: [6:7587241307500467721:0] socket: 603 qp: -1
2025-12-24T02:11:17.088232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241307091788167:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:17.088240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.089284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241307500467632:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9609) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241307500467714:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.089562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310736369632:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.090961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467723:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.091177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310736369632:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.091223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241310736369599:2109] [node 6] ICP18 handshake (actor [2:7587241310736369632:2119]) is too old
2025-12-24T02:11:17.091617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241307500467631:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29586) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241307500467712:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.091695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467724:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.091768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241307500467712:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.092297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241307500467723:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.092313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241307500467631:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.092325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.092338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241307500467631:2049] [node 2] ICP22 created new session: [6:7587241307500467725:2049]
2025-12-24T02:11:17.092349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241307500467725:2049] [node 2] ICS09 handshake done sender: [6:7587241307500467712:2117] self: [6:7587241307500467715:0] peer: [2:7587241310736369600:0] socket: 599 qp: -1
2025-12-24T02:11:17.092355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241307500467725:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.092365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.092516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241307500467692:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.092623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309756434244:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.092636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241309756434244:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.092704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467728:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.092758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241307091788129:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.092860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241307500467714:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.092924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241307500467632:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.092943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.092952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241307500467632:2050] [node 3] ICP22 created new session: [6:7587241307500467729:2050]
2025-12-24T02:11:17.092957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241307500467729:2050] [node 3] ICS09 handshake done sender: [6:7587241307500467714:2119] self: [6:7587241307500467721:0] peer: [3:7587241307091788136:0] socket: 553 qp: -1
2025-12-24T02:11:17.092964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241307500467729:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.092972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.094151Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241307500467731:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4451
2025-12-24T02:11:17.094243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309756434244:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.094298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241307500467723:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.094397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241309756434233:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.094419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.094426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309756434233:2111] [node 6] ICP22 created new session: [5:7587241309756434264:2111]
2025-12-24T02:11:17.094432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309756434264:2111] [node 6] ICS09 handshake done sender: [5:7587241309756434244:2119] self: [5:7587241309756434240:0] peer: [6:7587241307500467727:0] socket: 544 qp: -1
2025-12-24T02:11:17.094438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309756434264:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:17.094446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.094356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241307500467692:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.094374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.094384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241307500467692:2108] [node 5] ICP22 created new session: [6:7587241307500467732:2108]
2025-12-24T02:11:17.094390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241307500467732:2108] [node 5] ICS09 handshake done sender: [6:7587241307500467723:2121] self: [6:7587241307500467727:0] peer: [5:7587241309756434240:0] socket: 563 qp: -1
2025-12-24T02:11:17.094396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241307500467732:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:17.094403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.101596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:17.101732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241310736369614:2048] [node 1] ICS01 socket: 543 reason# EndOfStream
2025-12-24T02:11:17.101746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241306441402234:2048] [node 1] ICP30 unregister session Session# [2:7587241310736369614:2048] VirtualId# [2:7587241310736369609:0]
2025-12-24T02:11:17.101753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241310736369614:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.101811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.101856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.101967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:17.102053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241307091788164:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:17.102058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241302796820769:2048] [node 1] ICP30 unregister session Session# [3:7587241307091788164:2048] VirtualId# [3:7587241307091788156:0]
2025-12-24T02:11:17.102063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241307091788164:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.102102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.102139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.103657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:17.103731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241307500467719:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:17.103743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241307500467630:2048] [node 1] ICP30 unregister session Session# [6:7587241307500467719:2048] VirtualId# [6:7587241307500467710:0]
2025-12-24T02:11:17.103749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241307500467719:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.103805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.103842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.105456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:17.105548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241309756434250:2048] [node 1] ICS07 socket disconnect 542 reason# EndOfStream
2025-12-24T02:11:17.105553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309756434250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.105601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.105626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241309756434250:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:17.105918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241305461466865:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.105937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309756434250:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.105942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305461466865:2048] [node 1] ICP30 unregister session Session# [5:7587241309756434250:2048] VirtualId# [5:7587241309756434236:0]
2025-12-24T02:11:17.105968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.105983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241305461466865:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.105986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.103078Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:17.103173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241310073687170:2048] [node 1] ICS07 socket disconnect 540 reason# EndOfStream
2025-12-24T02:11:17.103179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241310073687170:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.103221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.103243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241310073687170:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:17.103591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305778719784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.103601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241310073687170:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.103608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305778719784:2048] [node 1] ICP30 unregister session Session# [4:7587241310073687170:2048] VirtualId# [4:7587241310073687158:0]
2025-12-24T02:11:17.103631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.103638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241305778719784:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.103641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6983) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.113491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241305778719784:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:17.114744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:17.114849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241307091788152:2049] [node 2] ICS07 socket disconnect 559 reason# EndOfStream
2025-12-24T02:11:17.114856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241307091788152:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.114909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.114929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241307091788152:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:17.115516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241305461466865:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:17.116414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:17.116504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309756434261:2049] [node 2] ICS01 socket: 574 reason# EndOfStream
2025-12-24T02:11:17.116511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305461466866:2049] [node 2] ICP30 unregister session Session# [5:7587241309756434261:2049] VirtualId# [5:7587241309756434249:0]
2025-12-24T02:11:17.116517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309756434261:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.116576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.116597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.117952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:17.118026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241310073687179:2049] [node 2] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:17.118032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305778719785:2049] [node 2] ICP30 unregister session Session# [4:7587241310073687179:2049] VirtualId# [4:7587241310073687172:0]
2025-12-24T02:11:17.118037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241310073687179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.118090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.118110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.118708Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:17.118780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241307500467725:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-12-24T02:11:17.118786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241307500467631:2049] [node 2] ICP30 unregister session Session# [6:7587241307500467725:2049] VirtualId# [6:7587241307500467715:0]
2025-12-24T02:11:17.118791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241307500467725:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.118848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.118873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.126146Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:17.126217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309756434259:2050] [node 3] ICS01 socket: 565 reason# EndOfStream
2025-12-24T02:11:17.126226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241305461466867:2050] [node 3] ICP30 unregister session Session# [5:7587241309756434259:2050] VirtualId# [5:7587241309756434248:0]
2025-12-24T02:11:17.126231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309756434259:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.126270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.126295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.126028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:17.126118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241310073687181:2050] [node 3] ICS07 socket disconnect 568 reason# EndOfStream
2025-12-24T02:11:17.126124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241310073687181:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.126181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.126203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241310073687181:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:17.126526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:17.126594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241307500467729:2050] [node 3] ICS01 socket: 553 reason# EndOfStream
2025-12-24T02:11:17.126604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241307500467632:2050] [node 3] ICP30 unregister session Session# [6:7587241307500467729:2050] VirtualId# [6:7587241307500467721:0]
2025-12-24T02:11:17.126608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241307500467729:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.126637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.126664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.126829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241305778719786:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9609) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.126841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241310073687181:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.126846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241305778719786:2050] [node 3] ICP30 unregister session Session# [4:7587241310073687181:2050] VirtualId# [4:7587241310073687168:0]
2025-12-24T02:11:17.126860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.126867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241305778719786:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.126869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9609) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.131734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:17.131830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241307500467717:2110] [node 4] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:17.131838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241307500467700:2110] [node 4] ICP30 unregister session Session# [6:7587241307500467717:2110] VirtualId# [6:7587241307500467711:0]
2025-12-24T02:11:17.131850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241307500467717:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.131901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.131926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:17.132073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:17.132138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309756434255:2110] [node 4] ICS01 socket: 564 reason# EndOfStream
2025-12-24T02:11:17.132142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241309756434234:2110] [node 4] ICP30 unregister session Session# [5:7587241309756434255:2110] VirtualId# [5:7587241309756434247:0]
2025-12-24T02:11:17.132146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309756434255:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.132179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.132195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:17.162478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307182958041:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:17.162485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307182958042:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.170256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309558712433:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.170268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309558712434:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.172059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241309501110783:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.172070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241309501110784:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.173739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241307963147652:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.173747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241307963147653:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.173751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241307963147654:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.192896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307182958096:2101] [node 4] ICP01 ready to work
2025-12-24T02:11:17.192984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307182958098:2102] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.193780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241307963147713:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.194866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110843:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.194091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307182958107:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:17.193966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309558712493:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:17.194268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309558712495:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.206252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307182958119:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19974
2025-12-24T02:11:17.211834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241309501110848:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:17.176850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241307850253812:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.176861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241307850253813:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.176869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241307850253814:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.193231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253872:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.194878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253873:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.207532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253873:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.210961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241307850253882:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.214881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147727:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.218505Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241309558712433:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24479796ns) {Configure:104(24479305ns) {ProcessConfigured:135(10843901ns) {StartInitialHandshake:157(10843424ns) {DropHandshakes:470(438ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:477(1136ns) {SwitchToState:179(233ns)}}}}}
2025-12-24T02:11:17.218629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309558712503:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:17.218842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958106:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.218859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307182958106:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.216016Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241309501110783:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20821040ns) {Configure:104(20820046ns) {ProcessConfigured:135(20467894ns) {StartInitialHandshake:157(20467604ns) {DropHandshakes:470(394ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(905ns) {SwitchToState:179(402ns)}}}}}
2025-12-24T02:11:17.216095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110843:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.216143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241309501110783:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.222656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307182958106:2106] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.231515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307182958042:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.231546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.231560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307182958042:2049] [node 3] ICP22 created new session: [1:7587241307182958122:2049]
2025-12-24T02:11:17.231568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241307182958122:2049] [node 3] ICS09 handshake done sender: [1:7587241307182958106:2106] self: [1:7587241307182958105:0] peer: [3:7587241309501110855:0] socket: 544 qp: -1
2025-12-24T02:11:17.231576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241307182958122:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.231608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.235027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712507:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
2025-12-24T02:11:17.235055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712508:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
2025-12-24T02:11:17.235096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712506:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
2025-12-24T02:11:17.235701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712506:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.237257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147729:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.235938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241309501110848:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24007434ns) {Configure:104(24006852ns) {ProcessConfigured:135(23688580ns) {StartInitialHandshake:157(23688323ns) {DropHandshakes:470(510ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:477(1168ns) {SwitchToState:179(593ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.237405Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147730:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.235558Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241307850253882:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24537632ns) {Configure:104(24537024ns) {ProcessConfigured:135(24230083ns) {StartInitialHandshake:157(24229544ns) {DropHandshakes:470(526ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:477(870ns) {SwitchToState:179(299ns)}}}}}
2025-12-24T02:11:17.235592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241307850253812:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.238811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958103:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.238785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241309501110851:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:17.238841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307182958103:2105] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.180743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241308405395540:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.180751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241308405395541:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.180755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241308405395542:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.192238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241308405395600:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:17.194121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241308405395606:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:17.218652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395616:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.221042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241308405395616:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.229913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241308405395606:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.234817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241308405395540:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35292348ns) {Configure:104(35291229ns) {ProcessConfigured:135(34902635ns) {StartInitialHandshake:157(34902107ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(115ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1607ns) {SwitchToState:179(775ns)}}}}}
2025-12-24T02:11:17.240387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241309501110853:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:17.238822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241309558712495:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(44492978ns) {Configure:104(44491906ns) {ProcessConfigured:135(30770211ns) {StartInitialHandshake:157(30768987ns) {DropHandshakes:470(246ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(1215ns) {SwitchToState:179(440ns)}}}}}
2025-12-24T02:11:17.238895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712507:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.238790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241307850253814:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35657703ns) {Configure:104(35655748ns) {ProcessConfigured:135(35073103ns) {StartInitialHandshake:157(35072536ns) {DropHandshakes:470(567ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:477(1330ns) {SwitchToState:179(751ns)}}}}}
2025-12-24T02:11:17.247128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241307963147652:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19917540ns) {Configure:104(19916285ns) {ProcessConfigured:135(19619880ns) {StartInitialHandshake:157(19619364ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(2112ns) {SwitchToState:179(1181ns)}}}}}
2025-12-24T02:11:17.247194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147716:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.247203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241307963147716:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.247257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241307963147726:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:17.248344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958119:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.249187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110862:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.249207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110863:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.249244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110864:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.249271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.249284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110866:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.249472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147730:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.249500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241307963147654:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.249524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958101:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.249536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307182958101:2104] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.249577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307182958126:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19974
2025-12-24T02:11:17.249356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110863:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.249445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241309501110843:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.249679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110862:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.249709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147727:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.248631Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241309558712493:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(54528891ns) {Configure:104(54526804ns) {ProcessConfigured:135(40826870ns) {StartInitialHandshake:157(40826080ns) {DropHandshakes:470(332ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(1565ns) {SwitchToState:179(1114ns)}}}}}
2025-12-24T02:11:17.248926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241309558712503:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30250346ns) {Configure:104(30249914ns) {ProcessConfigured:135(29929922ns) {StartInitialHandshake:157(29929471ns) {DropHandshakes:470(794ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(230ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(870ns) {SwitchToState:179(676ns)}}}}}
2025-12-24T02:11:17.249476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712508:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.249511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241309558712433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.250194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307182958125:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19974
2025-12-24T02:11:17.246727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241307850253884:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:17.247341Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241307850253813:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39747910ns) {Configure:104(39746748ns) {ProcessConfigured:135(39460875ns) {StartInitialHandshake:157(39460263ns) {DropHandshakes:470(456ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:477(1055ns) {SwitchToState:179(312ns)}}}}}
2025-12-24T02:11:17.248813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253872:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.257562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241309501110851:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18673295ns) {Configure:104(18672447ns) {ProcessConfigured:135(18330504ns) {StartInitialHandshake:157(18329688ns) {DropHandshakes:470(1276ns) {DropIncomingHandshake:442(496ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(2084ns) {SwitchToState:179(980ns)}}}}}
2025-12-24T02:11:17.257604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241309501110783:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.257622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.257632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241309501110783:2048] [node 1] ICP22 created new session: [3:7587241309501110868:2048]
2025-12-24T02:11:17.257638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241309501110868:2048] [node 1] ICS09 handshake done sender: [3:7587241309501110843:2107] self: [3:7587241309501110855:0] peer: [1:7587241307182958105:0] socket: 547 qp: -1
2025-12-24T02:11:17.257648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241309501110868:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.257670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.259056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958125:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.259547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147729:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.259733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110866:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.263132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241309501110853:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22657073ns) {Configure:104(22656157ns) {ProcessConfigured:135(22372008ns) {StartInitialHandshake:157(22371329ns) {DropHandshakes:470(1443ns) {DropIncomingHandshake:442(370ns) DropOutgoingHandshake:457(165ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(885ns) {SwitchToState:179(330ns)}}}}}
2025-12-24T02:11:17.263844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241307963147716:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.265283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110858:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.265307Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241309501110858:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.263983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241307963147713:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.264016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.264024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241307963147713:2107] [node 6] ICP22 created new session: [4:7587241307963147737:2107]
2025-12-24T02:11:17.264031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241307963147737:2107] [node 6] ICS09 handshake done sender: [4:7587241307963147716:2109] self: [4:7587241307963147715:0] peer: [6:7587241308405395618:0] socket: 543 qp: -1
2025-12-24T02:11:17.264041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241307963147737:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:17.264088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.263159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241307850253884:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16319020ns) {Configure:104(16317718ns) {ProcessConfigured:135(16010623ns) {StartInitialHandshake:157(16010066ns) {DropHandshakes:470(1030ns) {DropIncomingHandshake:442(127ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(414ns) PrepareNewSessionHandshake:477(1739ns) {SwitchToState:179(1036ns)}}}}}
2025-12-24T02:11:17.265337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241307182958042:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(42487536ns) {HandleHandshakeStatus:329(42486815ns) {DropIncomingHandshake:442(818ns) DropOutgoingHandshake:457(606ns) SwitchToState:179(193ns) ProcessPendingSessionEvents:495(26413049ns) {ForwardSessionEventToSession:612(26389059ns) ForwardSessionEventToSession:612(2133ns) ForwardSessionEventToSession:612(1104ns) ForwardSessionEventToSession:612(1267ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(953ns) ForwardSessionEventToSession:612(492ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(1175ns)}}}
2025-12-24T02:11:17.265454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241307182958042:2049] [node 3] ICP18 handshake (actor [1:7587241307182958125:2116]) is too old
2025-12-24T02:11:17.266832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307182958103:2105] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.267063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307182958098:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.267080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.267089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307182958098:2102] [node 5] ICP22 created new session: [1:7587241307182958130:2102]
2025-12-24T02:11:17.267094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241307182958130:2102] [node 5] ICS09 handshake done sender: [1:7587241307182958103:2105] self: [1:7587241307182958102:0] peer: [5:7587241307850253887:0] socket: 545 qp: -1
2025-12-24T02:11:17.267100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241307182958130:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:17.267107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.264814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241308405395606:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22937) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241308405395616:2114] held: no
2025-12-24T02:11:17.270786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241309558712434:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28686469ns) {Configure:104(28676651ns) {ProcessConfigured:135(14893558ns) {StartInitialHandshake:157(14892395ns) {DropHandshakes:470(405ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(1782ns) {SwitchToState:179(1011ns)}}}}}
2025-12-24T02:11:17.270987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395619:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.274555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110865:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:17.274616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241309501110784:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-12-24T02:11:17.274796Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241307850253812:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241307850253873:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.275204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241309501110858:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.275788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147741:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.275447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241309501110848:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.275470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.275479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241309501110848:2108] [node 4] ICP22 created new session: [3:7587241309501110877:2108]
2025-12-24T02:11:17.275486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241309501110877:2108] [node 4] ICS09 handshake done sender: [3:7587241309501110858:2113] self: [3:7587241309501110852:0] peer: [4:7587241307963147733:0] socket: 577 qp: -1
2025-12-24T02:11:17.275492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241309501110877:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:17.275514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.276493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241307963147654:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241307963147730:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.277995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241307963147730:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.278008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253894:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.278083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241307963147654:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.278102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.278108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241307963147654:2050] [node 3] ICP22 created new session: [4:7587241307963147743:2050]
2025-12-24T02:11:17.278120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241307963147743:2050] [node 3] ICS09 handshake done sender: [4:7587241307963147730:2116] self: [4:7587241307963147733:0] peer: [3:7587241309501110852:0] socket: 578 qp: -1
2025-12-24T02:11:17.278128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241307963147743:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.278144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.278018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958126:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958112:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307182958112:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.278230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253894:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307182958101:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.275590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241308405395619:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147740:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.275633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241308405395540:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.278144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.278272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241307850253813:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.278137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712505:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712518:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
2025-12-24T02:11:17.278155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309558712505:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.278363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712518:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147740:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241307963147652:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.278591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958113:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.278600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307182958113:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.279088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307182958112:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.279219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307182958107:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.279230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.279236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307182958107:2103] [node 6] ICP22 created new session: [1:7587241307182958137:2103]
2025-12-24T02:11:17.279241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241307182958137:2103] [node 6] ICS09 handshake done sender: [1:7587241307182958112:2107] self: [1:7587241307182958111:0] peer: [6:7587241308405395623:0] socket: 567 qp: -1
2025-12-24T02:11:17.279248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241307182958137:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:17.279255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.279562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307182958113:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.279934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110873:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.279945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241309501110873:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.280508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241309501110873:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.280663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241309501110853:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.280692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.280700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241309501110853:2110] [node 6] ICP22 created new session: [3:7587241309501110881:2110]
2025-12-24T02:11:17.280706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241309501110881:2110] [node 6] ICS09 handshake done sender: [3:7587241309501110873:2120] self: [3:7587241309501110861:0] peer: [6:7587241308405395628:0] socket: 595 qp: -1
2025-12-24T02:11:17.280715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241309501110881:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:17.280730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.279363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241308405395616:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.279459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395624:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.279514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.279685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241308405395606:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.279694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.279700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241308405395606:2109] [node 4] ICP22 created new session: [6:7587241308405395626:2109]
2025-12-24T02:11:17.279705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241308405395626:2109] [node 4] ICS09 handshake done sender: [6:7587241308405395616:2114] self: [6:7587241308405395618:0] peer: [4:7587241307963147715:0] socket: 562 qp: -1
2025-12-24T02:11:17.279711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241308405395626:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:17.279727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.279740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241308405395624:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.279790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241308405395542:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.279937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241308405395619:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.280026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241308405395540:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.280035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.280040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241308405395540:2048] [node 1] ICP22 created new session: [6:7587241308405395629:2048]
2025-12-24T02:11:17.280044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241308405395629:2048] [node 1] ICS09 handshake done sender: [6:7587241308405395619:2115] self: [6:7587241308405395623:0] peer: [1:7587241307182958111:0] socket: 568 qp: -1
2025-12-24T02:11:17.280052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241308405395629:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.280059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.282874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147746:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.283062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241307963147740:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.282900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307182958096:2101] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.282920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.282928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307182958096:2101] [node 4] ICP22 created new session: [1:7587241307182958139:2101]
2025-12-24T02:11:17.282935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241307182958139:2101] [node 4] ICS09 handshake done sender: [1:7587241307182958113:2108] self: [1:7587241307182958104:0] peer: [4:7587241307963147745:0] socket: 591 qp: -1
2025-12-24T02:11:17.282943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241307182958139:2101] [node 4] ICS10 traffic start
2025-12-24T02:11:17.282951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.283023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307182958134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19974
2025-12-24T02:11:17.283456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307182958134:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.283305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712512:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.283322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309558712512:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.286974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
2025-12-24T02:11:17.287509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309558712505:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.290761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309558712434:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.290784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.290793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309558712434:2049] [node 3] ICP22 created new session: [2:7587241309558712522:2049]
2025-12-24T02:11:17.290804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309558712522:2049] [node 3] ICS09 handshake done sender: [2:7587241309558712505:2111] self: [2:7587241309558712499:0] peer: [3:7587241309501110875:0] socket: 572 qp: -1
2025-12-24T02:11:17.290816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309558712522:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.290825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.290877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309558712506:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.291317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307182958041:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.291378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.291408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307182958041:2048] [node 2] ICP22 created new session: [1:7587241307182958141:2048]
2025-12-24T02:11:17.291416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241307182958141:2048] [node 2] ICS09 handshake done sender: [1:7587241307182958101:2104] self: [1:7587241307182958100:0] peer: [2:7587241309558712511:0] socket: 542 qp: -1
2025-12-24T02:11:17.291424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241307182958141:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:17.291438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.291357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241308405395542:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241308405395624:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.295078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253897:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.295128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253898:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.295081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241307182958096:2101] [node 4] ICP18 handshake (actor [1:7587241307182958134:2118]) is too old
2025-12-24T02:11:17.295283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241307850253873:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.295247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241309558712433:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.295268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.295275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309558712433:2048] [node 1] ICP22 created new session: [2:7587241309558712524:2048]
2025-12-24T02:11:17.295282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309558712524:2048] [node 1] ICS09 handshake done sender: [2:7587241309558712506:2112] self: [2:7587241309558712511:0] peer: [1:7587241307182958100:0] socket: 551 qp: -1
2025-12-24T02:11:17.295289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309558712524:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.295296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.295518Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241309501110784:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241309501110865:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.297221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147747:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.296995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309558712526:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13927
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.297409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.297556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309558712512:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.297723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309558712495:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.297749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.297757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309558712495:2107] [node 5] ICP22 created new session: [2:7587241309558712530:2107]
2025-12-24T02:11:17.297765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309558712530:2107] [node 5] ICS09 handshake done sender: [2:7587241309558712512:2115] self: [2:7587241309558712500:0] peer: [5:7587241307850253895:0] socket: 582 qp: -1
2025-12-24T02:11:17.297772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309558712530:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:17.297795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.298479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.298547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253899:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.298597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241307850253812:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.298610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.298621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241307850253812:2048] [node 1] ICP22 created new session: [5:7587241307850253900:2048]
2025-12-24T02:11:17.298630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241307850253900:2048] [node 1] ICS09 handshake done sender: [5:7587241307850253873:2108] self: [5:7587241307850253887:0] peer: [1:7587241307182958102:0] socket: 549 qp: -1
2025-12-24T02:11:17.298636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241307850253900:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.298655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.298731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241307963147652:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.298760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.298769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241307963147652:2048] [node 1] ICP22 created new session: [4:7587241307963147748:2048]
2025-12-24T02:11:17.298796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241307963147748:2048] [node 1] ICS09 handshake done sender: [4:7587241307963147740:2118] self: [4:7587241307963147745:0] peer: [1:7587241307182958104:0] socket: 592 qp: -1
2025-12-24T02:11:17.298805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241307963147748:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.298821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.298919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241308405395624:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.298991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241308405395542:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.299002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.299010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241308405395542:2050] [node 3] ICP22 created new session: [6:7587241308405395633:2050]
2025-12-24T02:11:17.299016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241308405395633:2050] [node 3] ICS09 handshake done sender: [6:7587241308405395624:2118] self: [6:7587241308405395628:0] peer: [3:7587241309501110861:0] socket: 558 qp: -1
2025-12-24T02:11:17.299022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241308405395633:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.299030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.307432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253897:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.307563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253899:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:17.307535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241309501110883:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14940
2025-12-24T02:11:17.307669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253902:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.307746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241309501110865:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.307955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241307850253882:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.308080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241307850253894:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.308224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241307850253813:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.308242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.308250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241307850253813:2049] [node 2] ICP22 created new session: [5:7587241307850253905:2049]
2025-12-24T02:11:17.308261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241307850253905:2049] [node 2] ICS09 handshake done sender: [5:7587241307850253894:2117] self: [5:7587241307850253895:0] peer: [2:7587241309558712500:0] socket: 585 qp: -1
2025-12-24T02:11:17.308267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241307850253905:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.308287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.308354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241307850253814:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.308036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241307182958041:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(29646969ns) {HandleHandshakeStatus:329(29646598ns) {DropIncomingHandshake:442(30077ns) DropOutgoingHandshake:457(237ns) SwitchToState:179(100ns) ProcessPendingSessionEvents:495(9665504ns) {ForwardSessionEventToSession:612(9652118ns) ForwardSessionEventToSession:612(2408ns) ForwardSessionEventToSession:612(1036ns) ForwardSessionEventToSession:612(659ns) ForwardSessionEventToSession:612(543ns) ForwardSessionEventToSession:612(1119ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(1189ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(1032ns)}}}
2025-12-24T02:11:17.308647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241309501110784:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.308663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.308669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241309501110784:2049] [node 2] ICP22 created new session: [3:7587241309501110884:2049]
2025-12-24T02:11:17.308676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241309501110884:2049] [node 2] ICS09 handshake done sender: [3:7587241309501110865:2117] self: [3:7587241309501110875:0] peer: [2:7587241309558712499:0] socket: 583 qp: -1
2025-12-24T02:11:17.308682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241309501110884:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.308698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.312422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241309501110867:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.312438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241309501110867:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.313201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241309501110867:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.313297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253908:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.313424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241307850253897:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.313674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241307850253814:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.313698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.313705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241307850253814:2050] [node 3] ICP22 created new session: [5:7587241307850253909:2050]
2025-12-24T02:11:17.313710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241307850253909:2050] [node 3] ICS09 handshake done sender: [5:7587241307850253897:2118] self: [5:7587241307850253903:0] peer: [3:7587241309501110860:0] socket: 596 qp: -1
2025-12-24T02:11:17.313717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241307850253909:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.313725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.313580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241309501110851:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.313605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.313613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241309501110851:2109] [node 5] ICP22 created new session: [3:7587241309501110887:2109]
2025-12-24T02:11:17.313620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241309501110887:2109] [node 5] ICS09 handshake done sender: [3:7587241309501110867:2119] self: [3:7587241309501110860:0] peer: [5:7587241307850253903:0] socket: 560 qp: -1
2025-12-24T02:11:17.313626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241309501110887:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:17.313641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.314825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241307963147652:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31417338ns) {HandleHandshakeStatus:329(31416726ns) {DropIncomingHandshake:442(942ns) DropOutgoingHandshake:457(2015ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(4009370ns) {ForwardSessionEventToSession:612(3997039ns) ForwardSessionEventToSession:612(2031ns) ForwardSessionEventToSession:612(1031ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(535ns) ForwardSessionEventToSession:612(973ns) ForwardSessionEventToSession:612(633ns) ForwardSessionEventToSession:612(502ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(766ns)}}}
2025-12-24T02:11:17.314862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241307963147652:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.314931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241308405395632:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.314985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241308405395541:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.315542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712514:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.315757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147747:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.315557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309558712514:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.315630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241309558712495:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.315800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241307963147653:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.318853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241307963147732:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.318874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241307963147732:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.319822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241307963147732:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.320118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309558712513:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.320135Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309558712513:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.320328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309558712514:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.320548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309558712503:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.320565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.320573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309558712503:2108] [node 6] ICP22 created new session: [2:7587241309558712533:2108]
2025-12-24T02:11:17.320580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309558712533:2108] [node 6] ICS09 handshake done sender: [2:7587241309558712514:2117] self: [2:7587241309558712504:0] peer: [6:7587241308405395635:0] socket: 607 qp: -1
2025-12-24T02:11:17.320588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309558712533:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:17.320597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.325236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395636:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.325376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241307963147726:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.325431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.325441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241307963147726:2110] [node 5] ICP22 created new session: [4:7587241307963147752:2110]
2025-12-24T02:11:17.325450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241307963147752:2110] [node 5] ICS09 handshake done sender: [4:7587241307963147732:2117] self: [4:7587241307963147731:0] peer: [5:7587241307850253904:0] socket: 557 qp: -1
2025-12-24T02:11:17.325461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241307963147752:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:17.325482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.325421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241307850253911:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51237
2025-12-24T02:11:17.325601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241307850253899:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.325693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241308405395636:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.325753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241308405395600:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.326464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309558712513:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.326637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309558712493:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.326651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.326658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309558712493:2106] [node 4] ICP22 created new session: [2:7587241309558712536:2106]
2025-12-24T02:11:17.326665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309558712536:2106] [node 4] ICS09 handshake done sender: [2:7587241309558712513:2116] self: [2:7587241309558712498:0] peer: [4:7587241307963147750:0] socket: 571 qp: -1
2025-12-24T02:11:17.326671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309558712536:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:17.326679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.325852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241307850253882:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.325874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.325882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241307850253882:2111] [node 4] ICP22 created new session: [5:7587241307850253912:2111]
2025-12-24T02:11:17.325888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241307850253912:2111] [node 4] ICS09 handshake done sender: [5:7587241307850253899:2120] self: [5:7587241307850253904:0] peer: [4:7587241307963147731:0] socket: 605 qp: -1
2025-12-24T02:11:17.325895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241307850253912:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:17.325906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.326598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241307850253892:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.326606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241307850253892:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.334934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241307963147754:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39257
2025-12-24T02:11:17.335164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241307963147747:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.339028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241309558712503:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(18451999ns) {HandleHandshakeStatus:329(18451196ns) {DropIncomingHandshake:442(3651ns) DropOutgoingHandshake:457(340ns) SwitchToState:179(171ns) ProcessPendingSessionEvents:495(12265166ns) {ForwardSessionEventToSession:612(12251742ns) ForwardSessionEventToSession:612(1835ns) ForwardSessionEventToSession:612(1048ns) ForwardSessionEventToSession:612(519ns) ForwardSessionEventToSession:612(505ns) ForwardSessionEventToSession:612(984ns) ForwardSessionEventToSession:612(537ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(849ns) ForwardSessionEventToSession:612(1028ns)}}}
2025-12-24T02:11:17.338880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395638:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.341897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241308405395541:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241308405395632:2121] held: no
2025-12-24T02:11:17.341967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241308405395632:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.342230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241307850253892:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.342361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241307850253884:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.342384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.342393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241307850253884:2112] [node 6] ICP22 created new session: [5:7587241307850253915:2112]
2025-12-24T02:11:17.342400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241307850253915:2112] [node 6] ICS09 handshake done sender: [5:7587241307850253892:2116] self: [5:7587241307850253890:0] peer: [6:7587241308405395637:0] socket: 563 qp: -1
2025-12-24T02:11:17.342406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241307850253915:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:17.342415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.342592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241308405395541:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.342614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.342622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241308405395541:2049] [node 2] ICP22 created new session: [6:7587241308405395639:2049]
2025-12-24T02:11:17.342628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241308405395639:2049] [node 2] ICS09 handshake done sender: [6:7587241308405395632:2121] self: [6:7587241308405395635:0] peer: [2:7587241309558712504:0] socket: 609 qp: -1
2025-12-24T02:11:17.342639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241308405395639:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.342654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.343410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241308405395600:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9672) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241308405395636:2122] held: no
2025-12-24T02:11:17.343040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241307963147653:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.343066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.343075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241307963147653:2049] [node 2] ICP22 created new session: [4:7587241307963147755:2049]
2025-12-24T02:11:17.343082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241307963147755:2049] [node 2] ICS09 handshake done sender: [4:7587241307963147747:2121] self: [4:7587241307963147750:0] peer: [2:7587241309558712498:0] socket: 608 qp: -1
2025-12-24T02:11:17.343089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241307963147755:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.343099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.344043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241308405395641:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48708
2025-12-24T02:11:17.344214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241308405395636:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.344270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241308405395600:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.344282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.344289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241308405395600:2106] [node 5] ICP22 created new session: [6:7587241308405395642:2106]
2025-12-24T02:11:17.344295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241308405395642:2106] [node 5] ICS09 handshake done sender: [6:7587241308405395636:2122] self: [6:7587241308405395637:0] peer: [5:7587241307850253890:0] socket: 610 qp: -1
2025-12-24T02:11:17.344301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241308405395642:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:17.344311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.345319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241307963147653:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.355165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:17.355279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241309558712524:2048] [node 1] ICS01 socket: 551 reason# EndOfStream
2025-12-24T02:11:17.355288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241309558712433:2048] [node 1] ICP30 unregister session Session# [2:7587241309558712524:2048] VirtualId# [2:7587241309558712511:0]
2025-12-24T02:11:17.355295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241309558712524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.355349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.355373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.359086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:17.359183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241309501110868:2048] [node 1] ICS01 socket: 547 reason# EndOfStream
2025-12-24T02:11:17.359191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241309501110783:2048] [node 1] ICP30 unregister session Session# [3:7587241309501110868:2048] VirtualId# [3:7587241309501110855:0]
2025-12-24T02:11:17.359199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241309501110868:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.359234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.359255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.360442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:17.360525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241307963147748:2048] [node 1] ICS07 socket disconnect 592 reason# EndOfStream
2025-12-24T02:11:17.360531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241307963147748:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.360580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.360608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241307963147748:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:17.361275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241307963147652:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.361291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241307963147748:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.361295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241307963147652:2048] [node 1] ICP30 unregister session Session# [4:7587241307963147748:2048] VirtualId# [4:7587241307963147745:0]
2025-12-24T02:11:17.361311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.361316Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241307963147652:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.361319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.361676Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:17.361770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241308405395629:2048] [node 1] ICS07 socket disconnect 568 reason# EndOfStream
2025-12-24T02:11:17.361782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241308405395629:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.361839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.361861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241308405395629:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:17.362320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241308405395540:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.362336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241308405395629:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.362346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241308405395540:2048] [node 1] ICP30 unregister session Session# [6:7587241308405395629:2048] VirtualId# [6:7587241308405395623:0]
2025-12-24T02:11:17.362410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.362415Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241308405395540:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.362417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1614) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.360937Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:17.361003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241307850253900:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-12-24T02:11:17.361008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241307850253812:2048] [node 1] ICP30 unregister session Session# [5:7587241307850253900:2048] VirtualId# [5:7587241307850253887:0]
2025-12-24T02:11:17.361012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241307850253900:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.361037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.361054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.374747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241307963147652:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:17.374761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:17.374851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241309501110884:2049] [node 2] ICS01 socket: 583 reason# EndOfStream
2025-12-24T02:11:17.374860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241309501110784:2049] [node 2] ICP30 unregister session Session# [3:7587241309501110884:2049] VirtualId# [3:7587241309501110875:0]
2025-12-24T02:11:17.374867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241309501110884:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.374919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.374943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.375209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241308405395540:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:17.375272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:17.375336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241307963147755:2049] [node 2] ICS07 socket disconnect 608 reason# EndOfStream
2025-12-24T02:11:17.375345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241307963147755:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.375380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.375419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241307963147755:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:17.375403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:17.375486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241308405395639:2049] [node 2] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:17.375493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241308405395541:2049] [node 2] ICP30 unregister session Session# [6:7587241308405395639:2049] VirtualId# [6:7587241308405395635:0]
2025-12-24T02:11:17.375498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241308405395639:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.375547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.375572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.376086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241307963147653:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.376098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241307963147755:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.376101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241307963147653:2049] [node 2] ICP30 unregister session Session# [4:7587241307963147755:2049] VirtualId# [4:7587241307963147750:0]
2025-12-24T02:11:17.376112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.376115Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241307963147653:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.376117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.374843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:17.374922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241307850253905:2049] [node 2] ICS07 socket disconnect 585 reason# EndOfStream
2025-12-24T02:11:17.374928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241307850253905:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.374978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.374999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241307850253905:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:17.375848Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241307850253813:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.375857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241307850253905:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.375861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241307850253813:2049] [node 2] ICP30 unregister session Session# [5:7587241307850253905:2049] VirtualId# [5:7587241307850253895:0]
2025-12-24T02:11:17.375874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.375879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241307850253813:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.375882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26422) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.390770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241307850253813:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:17.390834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241307963147653:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:17.396102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:17.396200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241307850253909:2050] [node 3] ICS07 socket disconnect 596 reason# EndOfStream
2025-12-24T02:11:17.396206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241307850253909:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.396264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.396283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241307850253909:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:17.396879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241307850253814:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23610) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.396889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241307850253909:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.396894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241307850253814:2050] [node 3] ICP30 unregister session Session# [5:7587241307850253909:2050] VirtualId# [5:7587241307850253903:0]
2025-12-24T02:11:17.397136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.397145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241307850253814:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.397149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23610) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.402796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:17.402895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241308405395633:2050] [node 3] ICS01 socket: 558 reason# EndOfStream
2025-12-24T02:11:17.402904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241308405395542:2050] [node 3] ICP30 unregister session Session# [6:7587241308405395633:2050] VirtualId# [6:7587241308405395628:0]
2025-12-24T02:11:17.402911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241308405395633:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.402979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.403006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.410752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241307850253814:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:17.421715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:17.421823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241308405395626:2109] [node 4] ICS01 socket: 562 reason# EndOfStream
2025-12-24T02:11:17.421832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241308405395606:2109] [node 4] ICP30 unregister session Session# [6:7587241308405395626:2109] VirtualId# [6:7587241308405395618:0]
2025-12-24T02:11:17.421839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241308405395626:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.421911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.421941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:17.473385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241310336391454:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:17.473397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241310336391455:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.474617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310340737016:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.474634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310340737017:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.480860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241308570680352:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.480872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241308570680353:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.485235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310189511769:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.485271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310189511770:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.485275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310189511771:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.482383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241309723935347:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.482391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241309723935348:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.482395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241309723935349:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.492146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241309520164569:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.492157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241309520164570:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.492162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241309520164571:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.509722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241309520164629:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:17.510343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241309520164635:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:17.510245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241310336391517:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.510337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241310336391518:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.510406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241310336391519:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.511443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511823:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.511805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935409:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.514945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310189511827:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:17.523173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310340737081:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.523341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310340737083:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.523459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241310340737084:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:17.524038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241308570680416:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.524137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241308570680417:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:17.524946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241308570680426:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:17.527213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310189511836:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:17.534791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241310336391454:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24324200ns) {Configure:104(24323569ns) {ProcessConfigured:135(23915193ns) {StartInitialHandshake:157(23914540ns) {DropHandshakes:470(403ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(1487ns) {SwitchToState:179(409ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.535958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241310336391530:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.539284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164640:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.544735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737082:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.551275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241309723935416:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:17.553942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935413:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.553961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935414:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.553978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935415:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.557884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241310336391534:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1812
2025-12-24T02:11:17.559853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241309723935426:2117] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.559993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680431:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.560055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680432:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.560098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680433:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
2025-12-24T02:11:17.559334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511823:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241309520164640:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241309520164629:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.560280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935409:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935414:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680431:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680433:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935413:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.560704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241309723935349:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.562807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391530:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.562981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391534:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.563142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935415:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.563178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241309723935347:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.566734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511838:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.566764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241310189511838:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.567020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680422:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.567035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241308570680422:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.570789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391525:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.570817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241310336391525:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.571492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241310336391525:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.571589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241310336391517:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.571611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.571621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241310336391517:2107] [node 4] ICP22 created new session: [1:7587241310336391538:2107]
2025-12-24T02:11:17.571630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241310336391538:2107] [node 4] ICS09 handshake done sender: [1:7587241310336391525:2112] self: [1:7587241310336391524:0] peer: [4:7587241309723935432:0] socket: 565 qp: -1
2025-12-24T02:11:17.571638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241310336391538:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:17.571649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.579264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680432:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.579473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241308570680352:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.579976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737092:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
2025-12-24T02:11:17.580027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737093:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
2025-12-24T02:11:17.582840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241308570680422:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.594827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241309520164569:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17875678ns) {Configure:104(17874748ns) {ProcessConfigured:135(17589256ns) {StartInitialHandshake:157(17587890ns) {DropHandshakes:470(844ns) {DropIncomingHandshake:442(102ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1514ns) {SwitchToState:179(282ns)}}}}}
2025-12-24T02:11:17.598733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391523:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.598765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241310336391523:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.599261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241308570680416:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.599293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.599301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241308570680416:2107] [node 4] ICP22 created new session: [3:7587241308570680439:2107]
2025-12-24T02:11:17.599310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241308570680439:2107] [node 4] ICS09 handshake done sender: [3:7587241308570680422:2112] self: [3:7587241308570680421:0] peer: [4:7587241309723935431:0] socket: 556 qp: -1
2025-12-24T02:11:17.599318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241308570680439:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:17.599329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.606892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241309723935347:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5127) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241309723935415:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.611422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241310336391541:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1812
2025-12-24T02:11:17.611460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241310336391542:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1812
2025-12-24T02:11:17.612812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511849:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.614908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241310336391523:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.614899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241310189511838:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.613020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
2025-12-24T02:11:17.621639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241310336391455:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.621675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.621686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241310336391455:2049] [node 3] ICP22 created new session: [1:7587241310336391544:2049]
2025-12-24T02:11:17.621695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241310336391544:2049] [node 3] ICS09 handshake done sender: [1:7587241310336391523:2111] self: [1:7587241310336391522:0] peer: [3:7587241308570680437:0] socket: 564 qp: -1
2025-12-24T02:11:17.621704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241310336391544:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.621713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.622941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241309723935349:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241309723935413:2111] held: no
2025-12-24T02:11:17.623084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935433:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.623529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.622128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737092:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:17.622163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737082:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:17.622185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737093:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.624091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241310189511836:2106] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.624124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.624132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310189511836:2106] [node 6] ICP22 created new session: [5:7587241310189511853:2106]
2025-12-24T02:11:17.624140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241310189511853:2106] [node 6] ICS09 handshake done sender: [5:7587241310189511838:2111] self: [5:7587241310189511837:0] peer: [6:7587241309520164641:0] socket: 560 qp: -1
2025-12-24T02:11:17.624148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241310189511853:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:17.624160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.624341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511849:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.624815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310189511769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.624851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511851:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.624865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511852:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.630007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164649:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.630792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935435:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.630870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241309723935415:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.631005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241309723935347:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.631024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.631032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241309723935347:2048] [node 1] ICP22 created new session: [4:7587241309723935438:2048]
2025-12-24T02:11:17.631040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241309723935438:2048] [node 1] ICS09 handshake done sender: [4:7587241309723935415:2113] self: [4:7587241309723935432:0] peer: [1:7587241310336391524:0] socket: 571 qp: -1
2025-12-24T02:11:17.631047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241309723935438:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.631056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.632421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241309723935413:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.632548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241309723935349:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.632567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.632574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241309723935349:2050] [node 3] ICP22 created new session: [4:7587241309723935440:2050]
2025-12-24T02:11:17.632581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241309723935440:2050] [node 3] ICS09 handshake done sender: [4:7587241309723935413:2111] self: [4:7587241309723935431:0] peer: [3:7587241308570680421:0] socket: 569 qp: -1
2025-12-24T02:11:17.632587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241309723935440:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.632595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.634963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391542:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.635078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391527:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.635082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241310336391527:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.635722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241310336391547:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1812
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.636171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241310336391527:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.636332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391547:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.636349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241310336391518:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.636359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.636366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241310336391518:2108] [node 5] ICP22 created new session: [1:7587241310336391550:2108]
2025-12-24T02:11:17.636372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241310336391550:2108] [node 5] ICS09 handshake done sender: [1:7587241310336391527:2113] self: [1:7587241310336391526:0] peer: [5:7587241310189511856:0] socket: 574 qp: -1
2025-12-24T02:11:17.636378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241310336391550:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:17.636389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.636532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241310336391518:2108] [node 5] ICP18 handshake (actor [1:7587241310336391547:2119]) is too old
2025-12-24T02:11:17.637372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511857:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.637873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164650:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.637951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164651:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.638811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241309520164629:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19757) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241309520164640:2112] held: no
2025-12-24T02:11:17.638904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241309520164640:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.638963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241308570680352:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5127) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241308570680432:2116] held: no
2025-12-24T02:11:17.639610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241310340737016:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.640470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680438:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.640534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680442:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.640925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935435:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.640956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241309723935348:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.642128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241309723935443:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58679
2025-12-24T02:11:17.642376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241309723935435:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.641054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737088:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.641066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310340737088:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.641604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310340737088:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.641690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310340737081:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.641706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.641713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310340737081:2107] [node 4] ICP22 created new session: [2:7587241310340737102:2107]
2025-12-24T02:11:17.641720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310340737102:2107] [node 4] ICS09 handshake done sender: [2:7587241310340737088:2113] self: [2:7587241310340737087:0] peer: [4:7587241309723935442:0] socket: 584 qp: -1
2025-12-24T02:11:17.641726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310340737102:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:17.641736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.642994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391541:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.642638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241309520164629:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.642663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.642671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241309520164629:2106] [node 5] ICP22 created new session: [6:7587241309520164652:2106]
2025-12-24T02:11:17.642678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241309520164652:2106] [node 5] ICS09 handshake done sender: [6:7587241309520164640:2112] self: [6:7587241309520164641:0] peer: [5:7587241310189511837:0] socket: 567 qp: -1
2025-12-24T02:11:17.642686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241309520164652:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:17.642707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.643005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241309520164651:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.643037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241309520164635:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.643041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391521:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.643045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241310336391521:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.643183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241309520164650:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.644686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737105:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
2025-12-24T02:11:17.645001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241308570680353:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.645033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241308570680432:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.644435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241309520164569:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.646847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737086:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.645168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241308570680352:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.646864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310340737086:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.645184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.645191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241308570680352:2048] [node 1] ICP22 created new session: [3:7587241308570680445:2048]
2025-12-24T02:11:17.645198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241308570680445:2048] [node 1] ICS09 handshake done sender: [3:7587241308570680432:2116] self: [3:7587241308570680437:0] peer: [1:7587241310336391522:0] socket: 576 qp: -1
2025-12-24T02:11:17.645204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241308570680445:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.645211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.647690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310340737086:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.647779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310340737017:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.647803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.647814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310340737017:2049] [node 3] ICP22 created new session: [2:7587241310340737107:2049]
2025-12-24T02:11:17.647825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310340737107:2049] [node 3] ICS09 handshake done sender: [2:7587241310340737086:2112] self: [2:7587241310340737085:0] peer: [3:7587241308570680444:0] socket: 583 qp: -1
2025-12-24T02:11:17.647833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310340737107:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.647841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.651962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511852:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.652126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241310189511769:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5127) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241310189511849:2116] held: no
2025-12-24T02:11:17.652158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241310189511849:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.652295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310189511769:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.652312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.652320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310189511769:2048] [node 1] ICP22 created new session: [5:7587241310189511859:2048]
2025-12-24T02:11:17.652326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241310189511859:2048] [node 1] ICS09 handshake done sender: [5:7587241310189511849:2116] self: [5:7587241310189511856:0] peer: [1:7587241310336391526:0] socket: 586 qp: -1
2025-12-24T02:11:17.652333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241310189511859:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.652339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.652465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310189511827:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.652497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511858:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.652790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737105:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.652940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737110:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
2025-12-24T02:11:17.653006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511858:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.653026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310189511770:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.654225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737090:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.654237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310340737090:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.654850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935422:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.654868Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241309723935422:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.656670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241310336391529:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.656686Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241310336391529:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.657207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310340737090:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.657849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164657:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.657921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164658:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.657958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164659:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.658112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241309520164651:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.655338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241309723935348:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.655357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.655366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241309723935348:2049] [node 2] ICP22 created new session: [4:7587241309723935445:2049]
2025-12-24T02:11:17.655375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241309723935445:2049] [node 2] ICS09 handshake done sender: [4:7587241309723935435:2121] self: [4:7587241309723935442:0] peer: [2:7587241310340737087:0] socket: 577 qp: -1
2025-12-24T02:11:17.655382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241309723935445:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.655393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.655505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241309723935348:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.655645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241309723935428:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.655649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241309723935428:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.656566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241309723935422:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.656702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241309723935428:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.658170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241309520164635:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.658185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.658192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241309520164635:2108] [node 4] ICP22 created new session: [6:7587241309520164660:2108]
2025-12-24T02:11:17.658199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241309520164660:2108] [node 4] ICS09 handshake done sender: [6:7587241309520164651:2118] self: [6:7587241309520164655:0] peer: [4:7587241309723935427:0] socket: 600 qp: -1
2025-12-24T02:11:17.658207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241309520164660:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:17.658217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.662845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241309723935416:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.662879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.662892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241309723935416:2110] [node 5] ICP22 created new session: [4:7587241309723935448:2110]
2025-12-24T02:11:17.662905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241309723935448:2110] [node 5] ICS09 handshake done sender: [4:7587241309723935422:2116] self: [4:7587241309723935421:0] peer: [5:7587241310189511862:0] socket: 573 qp: -1
2025-12-24T02:11:17.662919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241309723935448:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:17.662932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.663119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241309723935426:2117] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.663129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.663133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241309723935426:2117] [node 6] ICP22 created new session: [4:7587241309723935450:2117]
2025-12-24T02:11:17.663138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241309723935450:2117] [node 6] ICS09 handshake done sender: [4:7587241309723935428:2118] self: [4:7587241309723935427:0] peer: [6:7587241309520164655:0] socket: 596 qp: -1
2025-12-24T02:11:17.663143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241309723935450:2117] [node 6] ICS10 traffic start
2025-12-24T02:11:17.663150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.663515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310340737083:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.663533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.663542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310340737083:2108] [node 5] ICP22 created new session: [2:7587241310340737113:2108]
2025-12-24T02:11:17.663548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310340737113:2108] [node 5] ICS09 handshake done sender: [2:7587241310340737090:2114] self: [2:7587241310340737089:0] peer: [5:7587241310189511863:0] socket: 553 qp: -1
2025-12-24T02:11:17.663556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310340737113:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:17.663564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.663817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241310340737115:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22896
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.664145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241308570680353:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28761) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241308570680438:2118] held: no
2025-12-24T02:11:17.664368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.665240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680428:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.665252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241308570680428:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.665954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241308570680448:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50481
2025-12-24T02:11:17.666004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241308570680438:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.663738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164663:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.663834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241309520164658:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.663860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241309520164570:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.664376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241309520164657:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.664400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241309520164571:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.666111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737095:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.666123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241310340737095:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.666851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310340737095:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.666982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241310336391521:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.667147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241310336391529:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.667305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241310189511851:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.667333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241310189511827:2104] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14309) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241310189511852:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.667481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310189511771:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.667516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511864:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.667623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241310336391454:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.667643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.667652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241310336391454:2048] [node 2] ICP22 created new session: [1:7587241310336391555:2048]
2025-12-24T02:11:17.667659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241310336391555:2048] [node 2] ICS09 handshake done sender: [1:7587241310336391521:2110] self: [1:7587241310336391520:0] peer: [2:7587241310340737100:0] socket: 563 qp: -1
2025-12-24T02:11:17.667666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241310336391555:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:17.667674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.667812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241310336391519:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.667819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.667824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241310336391519:2109] [node 6] ICP22 created new session: [1:7587241310336391557:2109]
2025-12-24T02:11:17.667828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241310336391557:2109] [node 6] ICS09 handshake done sender: [1:7587241310336391529:2114] self: [1:7587241310336391528:0] peer: [6:7587241309520164656:0] socket: 562 qp: -1
2025-12-24T02:11:17.667832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241310336391557:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:17.667838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.668225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511865:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.668428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241310189511858:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.668767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680425:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.668776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241308570680425:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.669460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241308570680353:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.669473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.669480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241308570680353:2049] [node 2] ICP22 created new session: [3:7587241308570680450:2049]
2025-12-24T02:11:17.669486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241308570680450:2049] [node 2] ICS09 handshake done sender: [3:7587241308570680438:2118] self: [3:7587241308570680444:0] peer: [2:7587241310340737085:0] socket: 587 qp: -1
2025-12-24T02:11:17.669493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241308570680450:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.669500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.669821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241308570680428:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.670002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241310340737110:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.670940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310189511770:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.670959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.670965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310189511770:2049] [node 2] ICP22 created new session: [5:7587241310189511867:2049]
2025-12-24T02:11:17.670972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241310189511867:2049] [node 2] ICS09 handshake done sender: [5:7587241310189511858:2120] self: [5:7587241310189511863:0] peer: [2:7587241310340737089:0] socket: 548 qp: -1
2025-12-24T02:11:17.670979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241310189511867:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.670987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.679195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241310340737084:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.679225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.679239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310340737084:2110] [node 6] ICP22 created new session: [2:7587241310340737117:2110]
2025-12-24T02:11:17.679246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310340737117:2110] [node 6] ICS09 handshake done sender: [2:7587241310340737095:2117] self: [2:7587241310340737094:0] peer: [6:7587241309520164664:0] socket: 568 qp: -1
2025-12-24T02:11:17.679254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310340737117:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:17.679269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.679412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241310340737016:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5127) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241310340737093:2116] held: no
2025-12-24T02:11:17.679954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241308570680426:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.679976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.679986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241308570680426:2110] [node 6] ICP22 created new session: [3:7587241308570680453:2110]
2025-12-24T02:11:17.679993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241308570680453:2110] [node 6] ICS09 handshake done sender: [3:7587241308570680428:2114] self: [3:7587241308570680427:0] peer: [6:7587241309520164665:0] socket: 605 qp: -1
2025-12-24T02:11:17.680001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241308570680453:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:17.680010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.680064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241310189511852:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.680418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241308570680448:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.680483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241308570680426:2110] [node 6] ICP18 handshake (actor [3:7587241308570680448:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.680731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310189511827:2104] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.680745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.680750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310189511827:2104] [node 4] ICP22 created new session: [5:7587241310189511870:2104]
2025-12-24T02:11:17.680756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241310189511870:2104] [node 4] ICS09 handshake done sender: [5:7587241310189511852:2118] self: [5:7587241310189511862:0] peer: [4:7587241309723935421:0] socket: 598 qp: -1
2025-12-24T02:11:17.680761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241310189511870:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:17.680766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.680903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241310189511869:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11597
2025-12-24T02:11:17.681193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241308570680425:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.681766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241308570680417:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.681783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.681789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241308570680417:2109] [node 5] ICP22 created new session: [3:7587241308570680455:2109]
2025-12-24T02:11:17.681794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241308570680455:2109] [node 5] ICS09 handshake done sender: [3:7587241308570680425:2113] self: [3:7587241308570680424:0] peer: [5:7587241310189511866:0] socket: 593 qp: -1
2025-12-24T02:11:17.681800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241308570680455:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:17.681807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.682077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241310189511851:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.680269Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241309520164569:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5127) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241309520164650:2117] held: no
2025-12-24T02:11:17.680409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164666:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.682149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310189511771:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.682163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.682168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310189511771:2050] [node 3] ICP22 created new session: [5:7587241310189511872:2050]
2025-12-24T02:11:17.682173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241310189511872:2050] [node 3] ICS09 handshake done sender: [5:7587241310189511851:2117] self: [5:7587241310189511866:0] peer: [3:7587241308570680424:0] socket: 597 qp: -1
2025-12-24T02:11:17.682178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241310189511872:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.682185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.686969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241309520164657:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.687122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241309520164667:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5216
2025-12-24T02:11:17.687270Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241309520164650:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.695008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241310340737083:2108] [node 5] ICP18 handshake (actor [2:7587241310340737110:2119]) is too old
2025-12-24T02:11:17.695464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241310340737093:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.695546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241310340737016:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.695563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.695575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241310340737016:2048] [node 1] ICP22 created new session: [2:7587241310340737119:2048]
2025-12-24T02:11:17.695582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241310340737119:2048] [node 1] ICS09 handshake done sender: [2:7587241310340737093:2116] self: [2:7587241310340737100:0] peer: [1:7587241310336391520:0] socket: 578 qp: -1
2025-12-24T02:11:17.695596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241310340737119:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.695609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.700777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241309520164571:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.700805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.700816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241309520164571:2050] [node 3] ICP22 created new session: [6:7587241309520164668:2050]
2025-12-24T02:11:17.700823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241309520164668:2050] [node 3] ICS09 handshake done sender: [6:7587241309520164657:2119] self: [6:7587241309520164665:0] peer: [3:7587241308570680427:0] socket: 607 qp: -1
2025-12-24T02:11:17.700833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241309520164668:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.700845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.700943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241309520164571:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.701051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241309520164570:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28761) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241309520164658:2120] held: no
2025-12-24T02:11:17.701088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241309520164569:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.701097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.701102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241309520164569:2048] [node 1] ICP22 created new session: [6:7587241309520164670:2048]
2025-12-24T02:11:17.701106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241309520164670:2048] [node 1] ICS09 handshake done sender: [6:7587241309520164650:2117] self: [6:7587241309520164656:0] peer: [1:7587241310336391528:0] socket: 599 qp: -1
2025-12-24T02:11:17.701112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241309520164670:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.701116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.701294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241309520164658:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.701535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241309520164570:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.701546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.701554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241309520164570:2049] [node 2] ICP22 created new session: [6:7587241309520164672:2049]
2025-12-24T02:11:17.701560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241309520164672:2049] [node 2] ICS09 handshake done sender: [6:7587241309520164658:2120] self: [6:7587241309520164664:0] peer: [2:7587241310340737094:0] socket: 614 qp: -1
2025-12-24T02:11:17.701569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241309520164672:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.701577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.710781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:17.710877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241308570680445:2048] [node 1] ICS01 socket: 576 reason# EndOfStream
2025-12-24T02:11:17.710884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241308570680352:2048] [node 1] ICP30 unregister session Session# [3:7587241308570680445:2048] VirtualId# [3:7587241308570680437:0]
2025-12-24T02:11:17.710891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241308570680445:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.710941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.710970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.712169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:17.712254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241310340737119:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-12-24T02:11:17.712261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241310340737016:2048] [node 1] ICP30 unregister session Session# [2:7587241310340737119:2048] VirtualId# [2:7587241310340737100:0]
2025-12-24T02:11:17.712267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241310340737119:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.712326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.712352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.713393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:17.713475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241310189511859:2048] [node 1] ICS01 socket: 586 reason# EndOfStream
2025-12-24T02:11:17.713482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310189511769:2048] [node 1] ICP30 unregister session Session# [5:7587241310189511859:2048] VirtualId# [5:7587241310189511856:0]
2025-12-24T02:11:17.713489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241310189511859:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.713540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.713568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.713404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:17.713474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241309520164670:2048] [node 1] ICS01 socket: 599 reason# EndOfStream
2025-12-24T02:11:17.713479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241309520164569:2048] [node 1] ICP30 unregister session Session# [6:7587241309520164670:2048] VirtualId# [6:7587241309520164656:0]
2025-12-24T02:11:17.713483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241309520164670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.713493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.713529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.716292Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:17.716389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241309723935438:2048] [node 1] ICS01 socket: 571 reason# EndOfStream
2025-12-24T02:11:17.716396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241309723935347:2048] [node 1] ICP30 unregister session Session# [4:7587241309723935438:2048] VirtualId# [4:7587241309723935432:0]
2025-12-24T02:11:17.716403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241309723935438:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.716464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.716492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.719027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:17.719100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241308570680450:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-12-24T02:11:17.719105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241308570680353:2049] [node 2] ICP30 unregister session Session# [3:7587241308570680450:2049] VirtualId# [3:7587241308570680444:0]
2025-12-24T02:11:17.719110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241308570680450:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.719144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.719163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.725843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:17.725967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241309520164672:2049] [node 2] ICS01 socket: 614 reason# EndOfStream
2025-12-24T02:11:17.725977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241309520164570:2049] [node 2] ICP30 unregister session Session# [6:7587241309520164672:2049] VirtualId# [6:7587241309520164664:0]
2025-12-24T02:11:17.725983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241309520164672:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.726043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.726065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.726950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:17.727038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241310189511867:2049] [node 2] ICS01 socket: 548 reason# EndOfStream
2025-12-24T02:11:17.727046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310189511770:2049] [node 2] ICP30 unregister session Session# [5:7587241310189511867:2049] VirtualId# [5:7587241310189511863:0]
2025-12-24T02:11:17.727053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241310189511867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.727109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.727140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.727660Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:17.727748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241309723935445:2049] [node 2] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:17.727755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241309723935348:2049] [node 2] ICP30 unregister session Session# [4:7587241309723935445:2049] VirtualId# [4:7587241309723935442:0]
2025-12-24T02:11:17.727762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241309723935445:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.727831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.727861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.733871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:17.733987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241309723935440:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:17.733997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241309723935349:2050] [node 3] ICP30 unregister session Session# [4:7587241309723935440:2050] VirtualId# [4:7587241309723935431:0]
2025-12-24T02:11:17.734003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241309723935440:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.734082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.734115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.733942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:17.734048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241310189511872:2050] [node 3] ICS01 socket: 597 reason# EndOfStream
2025-12-24T02:11:17.734056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310189511771:2050] [node 3] ICP30 unregister session Session# [5:7587241310189511872:2050] VirtualId# [5:7587241310189511866:0]
2025-12-24T02:11:17.734062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241310189511872:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.734161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.734188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.734812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:17.734899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241309520164668:2050] [node 3] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:17.734905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241309520164571:2050] [node 3] ICP30 unregister session Session# [6:7587241309520164668:2050] VirtualId# [6:7587241309520164665:0]
2025-12-24T02:11:17.734911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241309520164668:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.734960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.735000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.742125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:17.742231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241310189511870:2104] [node 4] ICS01 socket: 598 reason# EndOfStream
2025-12-24T02:11:17.742240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310189511827:2104] [node 4] ICP30 unregister session Session# [5:7587241310189511870:2104] VirtualId# [5:7587241310189511862:0]
2025-12-24T02:11:17.742247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241310189511870:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.742305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.742328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:17.742639Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:17.742733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241309520164660:2108] [node 4] ICS01 socket: 600 reason# EndOfStream
2025-12-24T02:11:17.742741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241309520164635:2108] [node 4] ICP30 unregister session Session# [6:7587241309520164660:2108] VirtualId# [6:7587241309520164655:0]
2025-12-24T02:11:17.742747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241309520164660:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.742795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.742819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:17.749125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:17.749218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241309520164652:2106] [node 5] ICS01 socket: 567 reason# EndOfStream
2025-12-24T02:11:17.749227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241309520164629:2106] [node 5] ICP30 unregister session Session# [6:7587241309520164652:2106] VirtualId# [6:7587241309520164641:0]
2025-12-24T02:11:17.749235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241309520164652:2106] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.749296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:17.749322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:17.765517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309272044334:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.765536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309272044335:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.768121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310032581117:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.768135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310032581118:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.770289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241308517106766:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.770297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241308517106767:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.770301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241308517106768:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.772282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309915084537:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.772290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309915084538:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.772295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309915084539:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.776382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241310082224045:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.776390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241310082224046:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.776397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241310082224047:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.763052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241308663872492:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:17.763065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241308663872493:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.794026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241310082224106:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.795277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044394:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
2025-12-24T02:11:17.795951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084598:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.796784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241308663872560:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:17.796888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241308663872561:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:17.800224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309915084598:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.800390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309915084600:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:17.800977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309915084600:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.801212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241310082224109:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.801230Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241310082224109:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.801804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241310082224109:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.802100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241310082224106:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.802135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.802144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241310082224106:2107] [node 5] ICP22 created new session: [6:7587241310082224112:2107]
2025-12-24T02:11:17.802150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241310082224112:2107] [node 5] ICS09 handshake done sender: [6:7587241310082224109:2109] self: [6:7587241310082224108:0] peer: [5:7587241309915084601:0] socket: 552 qp: -1
2025-12-24T02:11:17.802167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241310082224112:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:17.802181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.798881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241308663872563:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.802864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310032581178:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.803321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241308663872571:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.803757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084602:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.803471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224114:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.803591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224115:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.804103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241310082224114:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.804212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309915084598:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.804538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241310082224045:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.803550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309272044402:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:17.805395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872565:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.805419Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241308663872565:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.806252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309915084600:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.806278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.806287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309915084600:2109] [node 6] ICP22 created new session: [5:7587241309915084603:2109]
2025-12-24T02:11:17.806294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309915084603:2109] [node 6] ICS09 handshake done sender: [5:7587241309915084598:2108] self: [5:7587241309915084601:0] peer: [6:7587241310082224108:0] socket: 557 qp: -1
2025-12-24T02:11:17.806302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309915084603:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:17.806330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.806559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310032581187:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:17.806658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310032581188:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:17.806565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309272044406:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:17.806681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241309272044407:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.810977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241308663872565:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.811042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044413:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
2025-12-24T02:11:17.815615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241308517106832:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.815725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241308517106833:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.815895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581185:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.816477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241309915084615:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.817020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106834:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.818545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084618:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.819025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581193:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.823029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241308517106766:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16236793ns) {Configure:104(16235681ns) {ProcessConfigured:135(15863728ns) {StartInitialHandshake:157(15863052ns) {DropHandshakes:470(568ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(445ns) PrepareNewSessionHandshake:477(1396ns) {SwitchToState:179(748ns)}}}}}
2025-12-24T02:11:17.823123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044394:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.823206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241310082224120:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.823615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581194:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.823672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044415:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
2025-12-24T02:11:17.823718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581185:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.823754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241310032581117:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.823857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224119:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.823303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241308663872563:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.823325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.823332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241308663872563:2111] [node 6] ICP22 created new session: [1:7587241308663872574:2111]
2025-12-24T02:11:17.823338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241308663872574:2111] [node 6] ICS09 handshake done sender: [1:7587241308663872565:2112] self: [1:7587241308663872564:0] peer: [6:7587241310082224116:0] socket: 560 qp: -1
2025-12-24T02:11:17.823345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241308663872574:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:17.823352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.827137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241309272044334:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.827183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044416:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
2025-12-24T02:11:17.827201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044413:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.827567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106834:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.827613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241308517106767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.827757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044408:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.827769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309272044408:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.829312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106846:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.830579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309272044408:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.829556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581193:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.829623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581194:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.830972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872553:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.830989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241308663872553:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.830959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309272044402:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.830980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.830989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309272044402:2110] [node 4] ICP22 created new session: [2:7587241309272044420:2110]
2025-12-24T02:11:17.830997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309272044420:2110] [node 4] ICS09 handshake done sender: [2:7587241309272044408:2113] self: [2:7587241309272044403:0] peer: [4:7587241308517106845:0] socket: 576 qp: -1
2025-12-24T02:11:17.831005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309272044420:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:17.831015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.831061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309915084618:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.831101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309915084615:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.831641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872556:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.831654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241308663872556:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.832131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241308663872576:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.833628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872576:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.833640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241308663872553:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.833709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241308663872556:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.834291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084620:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.834411Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044416:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.834488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241310032581118:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.838461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241308517106834:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.838586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241309272044402:2110] [node 4] ICP18 handshake (actor [2:7587241309272044416:2118]) is too old
2025-12-24T02:11:17.838815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872571:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.838840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241308663872493:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.838858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.838868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241308663872493:2049] [node 3] ICP22 created new session: [1:7587241308663872579:2049]
2025-12-24T02:11:17.838876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241308663872579:2049] [node 3] ICS09 handshake done sender: [1:7587241308663872553:2107] self: [1:7587241308663872552:0] peer: [3:7587241310032581198:0] socket: 553 qp: -1
2025-12-24T02:11:17.838883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241308663872579:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.838892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.838950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241308663872492:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.838963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.838971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241308663872492:2048] [node 2] ICP22 created new session: [1:7587241308663872581:2048]
2025-12-24T02:11:17.838977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241308663872581:2048] [node 2] ICS09 handshake done sender: [1:7587241308663872556:2108] self: [1:7587241308663872555:0] peer: [2:7587241309272044417:0] socket: 555 qp: -1
2025-12-24T02:11:17.838984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241308663872581:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:17.838991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.839066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241308663872492:2048] [node 2] ICP18 handshake (actor [1:7587241308663872571:2115]) is too old
2025-12-24T02:11:17.839350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309915084620:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.839411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309915084537:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.839271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106838:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.839288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241308517106838:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.839339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241310082224114:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.839797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581201:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.839861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581202:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
2025-12-24T02:11:17.840007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310032581185:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.840656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241308517106767:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.840676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.840683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241308517106767:2049] [node 2] ICP22 created new session: [4:7587241308517106849:2049]
2025-12-24T02:11:17.840690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241308517106849:2049] [node 2] ICS09 handshake done sender: [4:7587241308517106834:2111] self: [4:7587241308517106845:0] peer: [2:7587241309272044403:0] socket: 582 qp: -1
2025-12-24T02:11:17.840696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241308517106849:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.840705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.840792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241308517106767:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:17.840981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.840995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.841306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241308517106838:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.841273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241310082224045:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.841288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.841297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241310082224045:2048] [node 1] ICP22 created new session: [6:7587241310082224126:2048]
2025-12-24T02:11:17.841303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241310082224126:2048] [node 1] ICS09 handshake done sender: [6:7587241310082224114:2110] self: [6:7587241310082224116:0] peer: [1:7587241308663872564:0] socket: 561 qp: -1
2025-12-24T02:11:17.841309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241310082224126:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.841332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.841504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241310082224115:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.841549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241308517106832:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.841561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.841566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241308517106832:2108] [node 5] ICP22 created new session: [4:7587241308517106853:2108]
2025-12-24T02:11:17.841571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241308517106853:2108] [node 5] ICS09 handshake done sender: [4:7587241308517106838:2113] self: [4:7587241308517106837:0] peer: [5:7587241309915084619:0] socket: 585 qp: -1
2025-12-24T02:11:17.841577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241308517106853:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:17.841583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.843110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106856:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.844544Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106856:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.844442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241310082224047:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.845716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872567:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.845740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241308663872567:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.846510Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241309272044334:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241309272044394:2107] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.846987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241310032581117:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.847018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.847027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310032581117:2048] [node 1] ICP22 created new session: [3:7587241310032581204:2048]
2025-12-24T02:11:17.847035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241310032581204:2048] [node 1] ICS09 handshake done sender: [3:7587241310032581185:2111] self: [3:7587241310032581198:0] peer: [1:7587241308663872552:0] socket: 559 qp: -1
2025-12-24T02:11:17.847042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241310032581204:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.847062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.847260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581201:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.847380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581180:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.847390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241310032581180:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.847687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044399:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.847701Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309272044399:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.848047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044423:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.848346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084622:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.848485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106847:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:17.848523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241308517106768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-12-24T02:11:17.848548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.849478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.848814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581190:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.848824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241310032581190:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.850505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310032581190:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.850603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241310032581187:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.850619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.850626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310032581187:2109] [node 4] ICP22 created new session: [3:7587241310032581210:2109]
2025-12-24T02:11:17.850633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241310032581210:2109] [node 4] ICS09 handshake done sender: [3:7587241310032581190:2113] self: [3:7587241310032581189:0] peer: [4:7587241308517106858:0] socket: 600 qp: -1
2025-12-24T02:11:17.850639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241310032581210:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:17.850646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.850841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241308517106847:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.851001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241308517106768:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.851025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.851029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241308517106768:2050] [node 3] ICP22 created new session: [4:7587241308517106860:2050]
2025-12-24T02:11:17.851034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241308517106860:2050] [node 3] ICS09 handshake done sender: [4:7587241308517106847:2117] self: [4:7587241308517106858:0] peer: [3:7587241310032581189:0] socket: 565 qp: -1
2025-12-24T02:11:17.851039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241308517106860:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.851046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.855417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310032581180:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.855494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241310032581178:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.855513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.855529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310032581178:2107] [node 6] ICP22 created new session: [3:7587241310032581213:2107]
2025-12-24T02:11:17.855541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241310032581213:2107] [node 6] ICS09 handshake done sender: [3:7587241310032581180:2108] self: [3:7587241310032581179:0] peer: [6:7587241310082224131:0] socket: 568 qp: -1
2025-12-24T02:11:17.855548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241310032581213:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:17.855563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.855676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044415:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.855815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309272044394:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.855589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224132:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.855652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224133:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.855828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241310082224133:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.855854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241310082224120:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.856186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241309272044424:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13095
2025-12-24T02:11:17.856318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241309272044334:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.856332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.856338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309272044334:2048] [node 1] ICP22 created new session: [2:7587241309272044427:2048]
2025-12-24T02:11:17.856346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309272044427:2048] [node 1] ICS09 handshake done sender: [2:7587241309272044394:2107] self: [2:7587241309272044417:0] peer: [1:7587241308663872555:0] socket: 556 qp: -1
2025-12-24T02:11:17.856356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309272044427:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.856351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241310082224132:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.856362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.856391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241310082224046:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.856485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044424:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.856535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309272044399:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.856704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044410:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.856713Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309272044410:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.857822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309272044335:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.857839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.857844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309272044335:2049] [node 3] ICP22 created new session: [2:7587241309272044430:2049]
2025-12-24T02:11:17.857848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309272044430:2049] [node 3] ICS09 handshake done sender: [2:7587241309272044399:2109] self: [2:7587241309272044398:0] peer: [3:7587241310032581200:0] socket: 567 qp: -1
2025-12-24T02:11:17.857854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309272044430:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:17.857860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.856020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241308663872567:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.858849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581206:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.859101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241310032581178:2107] [node 6] ICP18 handshake (actor [3:7587241310032581206:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.859076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309915084618:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.859340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106848:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.859377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241308517106766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.859798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241310032581118:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241310032581193:2115] held: no
2025-12-24T02:11:17.860199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309272044410:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.860236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310032581215:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49509
2025-12-24T02:11:17.860289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309272044407:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.860301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.860307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309272044407:2112] [node 6] ICP22 created new session: [2:7587241309272044434:2112]
2025-12-24T02:11:17.860312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309272044434:2112] [node 6] ICS09 handshake done sender: [2:7587241309272044410:2114] self: [2:7587241309272044409:0] peer: [6:7587241310082224135:0] socket: 596 qp: -1
2025-12-24T02:11:17.860316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309272044434:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:17.860322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.860313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241308517106840:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.860325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241308517106840:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.860498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.858996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241308663872561:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.859017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.859024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241308663872561:2110] [node 5] ICP22 created new session: [1:7587241308663872587:2110]
2025-12-24T02:11:17.859030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241308663872587:2110] [node 5] ICS09 handshake done sender: [1:7587241308663872567:2113] self: [1:7587241308663872566:0] peer: [5:7587241309915084621:0] socket: 589 qp: -1
2025-12-24T02:11:17.859035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241308663872587:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:17.859041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.863158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309915084615:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.863179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.863187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309915084615:2114] [node 4] ICP22 created new session: [5:7587241309915084623:2114]
2025-12-24T02:11:17.863194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309915084623:2114] [node 4] ICS09 handshake done sender: [5:7587241309915084618:2116] self: [5:7587241309915084619:0] peer: [4:7587241308517106837:0] socket: 588 qp: -1
2025-12-24T02:11:17.863202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309915084623:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:17.863211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.863896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241310082224047:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26049) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241310082224115:2111] held: no
2025-12-24T02:11:17.863942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241310082224115:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.864471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310032581193:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.864782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241308517106840:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:17.864886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241310032581118:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.864904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.864909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310032581118:2049] [node 2] ICP22 created new session: [3:7587241310032581216:2049]
2025-12-24T02:11:17.864916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241310032581216:2049] [node 2] ICS09 handshake done sender: [3:7587241310032581193:2115] self: [3:7587241310032581200:0] peer: [2:7587241309272044398:0] socket: 570 qp: -1
2025-12-24T02:11:17.864859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084626:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.864922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241310032581216:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.864936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.864924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241308517106833:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.864942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:17.864949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241308517106833:2109] [node 6] ICP22 created new session: [4:7587241308517106864:2109]
2025-12-24T02:11:17.864955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241308517106864:2109] [node 6] ICS09 handshake done sender: [4:7587241308517106840:2114] self: [4:7587241308517106839:0] peer: [6:7587241310082224134:0] socket: 584 qp: -1
2025-12-24T02:11:17.864962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241308517106864:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:17.864968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:17.865005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084625:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.865790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084627:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.865774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241310082224047:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.865798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.865807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241310082224047:2050] [node 3] ICP22 created new session: [6:7587241310082224138:2050]
2025-12-24T02:11:17.865813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241310082224138:2050] [node 3] ICS09 handshake done sender: [6:7587241310082224115:2111] self: [6:7587241310082224131:0] peer: [3:7587241310032581179:0] socket: 569 qp: -1
2025-12-24T02:11:17.865819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241310082224138:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.865827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.866093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241310082224046:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241310082224132:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.866332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224137:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.862842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241308663872586:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10603
2025-12-24T02:11:17.862943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872569:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.862946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241308663872569:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.863065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241308663872586:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.865514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241308663872569:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.865601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241308663872560:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.865612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.865617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241308663872560:2109] [node 4] ICP22 created new session: [1:7587241308663872591:2109]
2025-12-24T02:11:17.865623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241308663872591:2109] [node 4] ICS09 handshake done sender: [1:7587241308663872569:2114] self: [1:7587241308663872568:0] peer: [4:7587241308517106862:0] socket: 601 qp: -1
2025-12-24T02:11:17.865628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241308663872591:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:17.865634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.866827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241308517106766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241308517106848:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.867332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241308517106866:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23647
2025-12-24T02:11:17.867425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309915084625:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.867460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241309915084626:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.867434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241308517106848:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.867486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241309915084537:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27433) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241309915084620:2117] held: no
2025-12-24T02:11:17.867511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241308517106766:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.867526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.867521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309915084620:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:17.867531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241308517106766:2048] [node 1] ICP22 created new session: [4:7587241308517106867:2048]
2025-12-24T02:11:17.867541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241308517106867:2048] [node 1] ICS09 handshake done sender: [4:7587241308517106848:2118] self: [4:7587241308517106862:0] peer: [1:7587241308663872568:0] socket: 591 qp: -1
2025-12-24T02:11:17.867547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241308517106867:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.867554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.871035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241310082224140:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9306
2025-12-24T02:11:17.871270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241310082224120:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24412) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241310082224133:2118] held: no
2025-12-24T02:11:17.871323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241310082224133:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:17.871414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310032581192:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.871427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241310032581192:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.871168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309915084537:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.871183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309915084539:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.871197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241309915084538:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:17.871202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:17.871210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309915084537:2048] [node 1] ICP22 created new session: [5:7587241309915084630:2048]
2025-12-24T02:11:17.871216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309915084630:2048] [node 1] ICS09 handshake done sender: [5:7587241309915084620:2117] self: [5:7587241309915084621:0] peer: [1:7587241308663872566:0] socket: 592 qp: -1
2025-12-24T02:11:17.871223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309915084630:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:17.871237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:17.871855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241309272044412:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:17.871871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241309272044412:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:17.871870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241310082224120:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.871884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:17.871891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241310082224120:2112] [node 4] ICP22 created new session: [6:7587241310082224141:2112]
2025-12-24T02:11:17.871897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241310082224141:2112] [node 4] ICS09 handshake done sender: [6:7587241310082224133:2118] self: [6:7587241310082224134:0] peer: [4:7587241308517106839:0] socket: 612 qp: -1
2025-12-24T02:11:17.871914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241310082224141:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:17.871921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:17.872060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241310082224132:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.873169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241310082224046:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.873188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.873194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241310082224046:2049] [node 2] ICP22 created new session: [6:7587241310082224143:2049]
2025-12-24T02:11:17.873199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241310082224143:2049] [node 2] ICS09 handshake done sender: [6:7587241310082224132:2117] self: [6:7587241310082224135:0] peer: [2:7587241309272044409:0] socket: 602 qp: -1
2025-12-24T02:11:17.873204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241310082224143:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.873215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.873315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084632:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.872794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241309272044412:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.872873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241309272044406:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.872883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.872896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241309272044406:2111] [node 5] ICP22 created new session: [2:7587241309272044437:2111]
2025-12-24T02:11:17.872902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241309272044437:2111] [node 5] ICS09 handshake done sender: [2:7587241309272044412:2115] self: [2:7587241309272044411:0] peer: [5:7587241309915084629:0] socket: 616 qp: -1
2025-12-24T02:11:17.872907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241309272044437:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:17.872914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.873930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241309915084538:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241309915084626:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.873996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241309915084633:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53821
2025-12-24T02:11:17.874111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309915084626:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:17.874205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309915084538:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.874223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:17.874232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309915084538:2049] [node 2] ICP22 created new session: [5:7587241309915084634:2049]
2025-12-24T02:11:17.874238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309915084634:2049] [node 2] ICS09 handshake done sender: [5:7587241309915084626:2120] self: [5:7587241309915084629:0] peer: [2:7587241309272044411:0] socket: 580 qp: -1
2025-12-24T02:11:17.874319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309915084634:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:17.874336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:17.874811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310032581192:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:17.874873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241309915084625:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:17.874955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241309915084539:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:17.874966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:17.874972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241309915084539:2050] [node 3] ICP22 created new session: [5:7587241309915084636:2050]
2025-12-24T02:11:17.874977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241309915084636:2050] [node 3] ICS09 handshake done sender: [5:7587241309915084625:2119] self: [5:7587241309915084628:0] peer: [3:7587241310032581191:0] socket: 620 qp: -1
2025-12-24T02:11:17.874982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241309915084636:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:17.874988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:17.878848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241310032581188:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:17.878872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:17.878884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310032581188:2110] [node 5] ICP22 created new session: [3:7587241310032581219:2110]
2025-12-24T02:11:17.878891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241310032581219:2110] [node 5] ICS09 handshake done sender: [3:7587241310032581192:2114] self: [3:7587241310032581191:0] peer: [5:7587241309915084628:0] socket: 613 qp: -1
2025-12-24T02:11:17.878897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241310032581219:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:17.878904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:17.888278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:17.888376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241310032581204:2048] [node 1] ICS01 socket: 559 reason# EndOfStream
2025-12-24T02:11:17.888389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241310032581117:2048] [node 1] ICP30 unregister session Session# [3:7587241310032581204:2048] VirtualId# [3:7587241310032581198:0]
2025-12-24T02:11:17.888396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241310032581204:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.888489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:17.888450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.888489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.888569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241309272044427:2048] [node 1] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:17.888580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241309272044334:2048] [node 1] ICP30 unregister session Session# [2:7587241309272044427:2048] VirtualId# [2:7587241309272044417:0]
2025-12-24T02:11:17.888586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241309272044427:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.888649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.888682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.891379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:17.891500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241308517106867:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-12-24T02:11:17.891508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241308517106766:2048] [node 1] ICP30 unregister session Session# [4:7587241308517106867:2048] VirtualId# [4:7587241308517106862:0]
2025-12-24T02:11:17.891514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241308517106867:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.891581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.891605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.891588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:17.891701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309915084630:2048] [node 1] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:17.891708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241309915084537:2048] [node 1] ICP30 unregister session Session# [5:7587241309915084630:2048] VirtualId# [5:7587241309915084621:0]
2025-12-24T02:11:17.891715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309915084630:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.892104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.892133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.893234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:17.893313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241310082224126:2048] [node 1] ICS01 socket: 561 reason# EndOfStream
2025-12-24T02:11:17.893321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241310082224045:2048] [node 1] ICP30 unregister session Session# [6:7587241310082224126:2048] VirtualId# [6:7587241310082224116:0]
2025-12-24T02:11:17.893326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241310082224126:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.893389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:17.893439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:17.900159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:17.900260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241310032581216:2049] [node 2] ICS01 socket: 570 reason# EndOfStream
2025-12-24T02:11:17.900271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241310032581118:2049] [node 2] ICP30 unregister session Session# [3:7587241310032581216:2049] VirtualId# [3:7587241310032581200:0]
2025-12-24T02:11:17.900278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241310032581216:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.900333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.900356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.901277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:17.901360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241308517106849:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:17.901368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241308517106767:2049] [node 2] ICP30 unregister session Session# [4:7587241308517106849:2049] VirtualId# [4:7587241308517106845:0]
2025-12-24T02:11:17.901375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241308517106849:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.901436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.901480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.901497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:17.901578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309915084634:2049] [node 2] ICS01 socket: 580 reason# EndOfStream
2025-12-24T02:11:17.901585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241309915084538:2049] [node 2] ICP30 unregister session Session# [5:7587241309915084634:2049] VirtualId# [5:7587241309915084629:0]
2025-12-24T02:11:17.901594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309915084634:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.901660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.901688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.902179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:17.902265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241310082224143:2049] [node 2] ICS01 socket: 602 reason# EndOfStream
2025-12-24T02:11:17.902272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241310082224046:2049] [node 2] ICP30 unregister session Session# [6:7587241310082224143:2049] VirtualId# [6:7587241310082224135:0]
2025-12-24T02:11:17.902277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241310082224143:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.902330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:17.902359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:17.909098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:17.909235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241310082224138:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:17.909245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241310082224047:2050] [node 3] ICP30 unregister session Session# [6:7587241310082224138:2050] VirtualId# [6:7587241310082224131:0]
2025-12-24T02:11:17.909251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241310082224138:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.909320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.909357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.909366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:17.909448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241308517106860:2050] [node 3] ICS01 socket: 565 reason# EndOfStream
2025-12-24T02:11:17.909455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241308517106768:2050] [node 3] ICP30 unregister session Session# [4:7587241308517106860:2050] VirtualId# [4:7587241308517106858:0]
2025-12-24T02:11:17.909461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241308517106860:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.909497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.909518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.909920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:17.910026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309915084636:2050] [node 3] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:17.910032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241309915084539:2050] [node 3] ICP30 unregister session Session# [5:7587241309915084636:2050] VirtualId# [5:7587241309915084628:0]
2025-12-24T02:11:17.910037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309915084636:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.910096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:17.910121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:17.922765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:17.922874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241309915084623:2114] [node 4] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:17.922882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241309915084615:2114] [node 4] ICP30 unregister session Session# [5:7587241309915084623:2114] VirtualId# [5:7587241309915084619:0]
2025-12-24T02:11:17.922890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241309915084623:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.922948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.922978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:17.924055Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:17.924156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241310082224141:2112] [node 4] ICS01 socket: 612 reason# EndOfStream
2025-12-24T02:11:17.924164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241310082224120:2112] [node 4] ICP30 unregister session Session# [6:7587241310082224141:2112] VirtualId# [6:7587241310082224134:0]
2025-12-24T02:11:17.924170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241310082224141:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.924223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:17.924247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:17.939699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:17.939800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241310082224112:2107] [node 5] ICS07 socket disconnect 552 reason# EndOfStream
2025-12-24T02:11:17.939808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241310082224112:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:17.939854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:17.939872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241310082224112:2107] [node 5] ICS15 start handshake
2025-12-24T02:11:17.940356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241310082224106:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15826) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:17.940371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241310082224112:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:17.940375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241310082224106:2107] [node 5] ICP30 unregister session Session# [6:7587241310082224112:2107] VirtualId# [6:7587241310082224108:0]
2025-12-24T02:11:17.940393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:17.940403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241310082224106:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15826) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:17.940405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15826) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:17.962163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241307022300692:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.962178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241307022300693:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.963989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310829927491:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.963998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310829927492:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.965417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310070390324:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.965425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310070390325:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.965432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310070390326:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.967266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310955858294:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.967273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310955858295:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.967277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310955858296:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.969181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241306887416193:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:17.969189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241306887416194:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:17.969193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241306887416195:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:17.960445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307570955698:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:17.960458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307570955699:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:17.983327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310070390386:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:17.983196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241306887416255:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.984427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241307022300751:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:17.984551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241310829927548:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.984774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241306887416259:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.985067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241310070390390:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:17.984343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307570955763:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.984800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307570955766:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.985103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241307570955771:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:17.986791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241307022300755:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:17.991123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241307022300761:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:17.991212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241307022300762:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.998620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241306887416260:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:17.999372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241310070390394:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:17.999253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307570955777:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-12-24T02:11:17.999278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241307570955778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-12-24T02:11:18.005150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955778:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.007265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241310829927553:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:18.007090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241315250825656:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:18.007419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315124894854:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.007502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315124894855:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:18.010865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241307022300755:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23993768ns) {Configure:104(23993020ns) {ProcessConfigured:135(23761492ns) {StartInitialHandshake:157(23761055ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(39ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1212ns) {SwitchToState:179(749ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.011192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311317268068:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:18.011162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241306887416259:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.012049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311182383563:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:18.011968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241310829927491:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22642423ns) {Configure:104(22641134ns) {ProcessConfigured:135(22307292ns) {StartInitialHandshake:157(22306762ns) {DropHandshakes:470(321ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(645ns)}}}}}
2025-12-24T02:11:18.011956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241310955858358:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.019575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894858:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-12-24T02:11:18.019597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894859:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.019977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241311865923076:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-12-24T02:11:18.020425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241310070390325:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24544173ns) {Configure:104(24543465ns) {ProcessConfigured:135(24312558ns) {StartInitialHandshake:157(24312023ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1775ns) {SwitchToState:179(1380ns)}}}}}
2025-12-24T02:11:18.020515Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241310070390324:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25206433ns) {Configure:104(25205438ns) {ProcessConfigured:135(24841981ns) {StartInitialHandshake:157(24840834ns) {DropHandshakes:470(701ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(109ns)} GenerateSessionVirtualId:988(349ns) PrepareNewSessionHandshake:477(1395ns) {SwitchToState:179(961ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.031115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311182383563:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.031174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241311182383563:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19048013ns) {Configure:104(19047644ns) {ProcessConfigured:135(18661079ns) {StartInitialHandshake:157(18616736ns) {DropHandshakes:470(359ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(803ns) PrepareNewSessionHandshake:477(1265ns) {SwitchToState:179(151ns)}} IncomingHandshake:261(32368ns) {DropIncomingHandshake:442(183ns) PrepareNewSessionHandshake:477(129ns) {SwitchToState:179(83ns)} GenerateSessionVirtualId:988(361ns) IssueIncomingHandshakeReply:193(23597ns) {SwitchToState:179(132ns)}}}}}
2025-12-24T02:11:18.031291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241310829927492:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23658756ns) {Configure:104(23657786ns) {ProcessConfigured:135(23431633ns) {StartInitialHandshake:157(23431379ns) {DropHandshakes:470(400ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(742ns) {SwitchToState:179(398ns)}}}}}
2025-12-24T02:11:18.034800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314365357699:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:18.034949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310070390388:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.034960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241310070390388:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.035522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241310955858295:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28361110ns) {Configure:104(28360459ns) {ProcessConfigured:135(28043953ns) {StartInitialHandshake:157(28043421ns) {DropHandshakes:470(509ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(1113ns) {SwitchToState:179(226ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.035829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310070390388:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.035906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825663:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.035941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825655:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.035946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825664:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.036205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241310070390386:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.036221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.036230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310070390386:2108] [node 6] ICP22 created new session: [4:7587241314365357701:2108]
2025-12-24T02:11:18.036234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314365357701:2108] [node 6] ICS09 handshake done sender: [4:7587241310070390388:2109] self: [4:7587241310070390387:0] peer: [6:7587241311182383570:0] socket: 561 qp: -1
2025-12-24T02:11:18.036241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314365357701:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:18.036248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.036246Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241310955858294:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23428582ns) {Configure:104(23428189ns) {ProcessConfigured:135(23057052ns) {StartInitialHandshake:157(23056719ns) {DropHandshakes:470(548ns) {DropIncomingHandshake:442(118ns) DropOutgoingHandshake:457(130ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(1010ns) {SwitchToState:179(402ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.036428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241311865923077:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-12-24T02:11:18.036835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894859:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:18.037065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
::ffff:127.1.0.02025-12-24T02:11:18.037085Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955757:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.037100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307570955757:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.037131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241310829927548:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.037155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241310829927491:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.037292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894858:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.037313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241310829927492:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.035878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.036614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241307022300751:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.037068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241307022300692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.037142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311317268069:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:18.038416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241306887416259:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.038497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311182383563:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.038510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.038517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311182383563:2113] [node 4] ICP22 created new session: [6:7587241311182383572:2113]
2025-12-24T02:11:18.038522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311182383572:2113] [node 4] ICS09 handshake done sender: [6:7587241306887416259:2110] self: [6:7587241311182383570:0] peer: [4:7587241310070390387:0] socket: 564 qp: -1
2025-12-24T02:11:18.038528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311182383572:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:18.038535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.038111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825663:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.038166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310955858358:2107] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.038626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825655:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.046894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955760:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.046916Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307570955760:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.047000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314365357692:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.047021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241314365357692:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.047049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311317268068:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.050555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241307022300764:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.050581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241307022300764:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.050813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314365357699:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.050735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311317268070:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:18.051004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241310070390326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.051159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311317268070:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.051441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311317268074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:18.051541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307570955757:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.051740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241307022300751:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.051811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241307022300692:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.051882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.051890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241307022300692:2048] [node 1] ICP22 created new session: [2:7587241311317268075:2048]
2025-12-24T02:11:18.051896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311317268075:2048] [node 1] ICS09 handshake done sender: [2:7587241307022300751:2106] self: [2:7587241311317268071:0] peer: [1:7587241307570955756:0] socket: 563 qp: -1
2025-12-24T02:11:18.051902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311317268075:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.051920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.052344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955777:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.052356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307570955698:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.052365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.052369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307570955698:2048] [node 2] ICP22 created new session: [1:7587241311865923079:2048]
2025-12-24T02:11:18.052376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241311865923079:2048] [node 2] ICS09 handshake done sender: [1:7587241307570955757:2105] self: [1:7587241307570955756:0] peer: [2:7587241311317268071:0] socket: 559 qp: -1
2025-12-24T02:11:18.052380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241311865923079:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:18.052387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.052539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241307570955698:2048] [node 2] ICP18 handshake (actor [1:7587241307570955777:2115]) is too old
2025-12-24T02:11:18.054046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241306887416260:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.054080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241306887416194:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.054368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825673:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.054453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.054777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241310070390390:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.054820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241310070390324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.054850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825664:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.054881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310955858296:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.055639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314365357692:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.055164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307570955760:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.055881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241310070390394:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.055899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.055907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310070390394:2111] [node 5] ICP22 created new session: [4:7587241314365357707:2111]
2025-12-24T02:11:18.055913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314365357707:2111] [node 5] ICS09 handshake done sender: [4:7587241314365357692:2112] self: [4:7587241310070390395:0] peer: [5:7587241315250825671:0] socket: 582 qp: -1
2025-12-24T02:11:18.055921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314365357707:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:18.055931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.056146Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241311182383563:2113] [node 4] ICP35 event processing took too much time PendingConnection:279(17615512ns) {HandleHandshakeStatus:329(17614575ns) {DropIncomingHandshake:442(142ns) DropOutgoingHandshake:457(1545ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(8350980ns) {ForwardSessionEventToSession:612(8338898ns) ForwardSessionEventToSession:612(1294ns) ForwardSessionEventToSession:612(906ns) ForwardSessionEventToSession:612(1072ns) ForwardSessionEventToSession:612(622ns) ForwardSessionEventToSession:612(965ns) ForwardSessionEventToSession:612(997ns) ForwardSessionEventToSession:612(711ns) ForwardSessionEventToSession:612(584ns) ForwardSessionEventToSession:612(995ns)}}}
2025-12-24T02:11:18.058831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894863:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.058852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315124894863:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.058821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955767:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.058839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307570955767:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.062371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-12-24T02:11:18.062455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241307022300766:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.062474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241307022300766:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.062914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894861:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.062928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315124894861:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.065567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241310829927548:2104] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.065947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825673:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.066000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825676:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.066085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307570955699:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.066105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.066112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307570955699:2049] [node 3] ICP22 created new session: [1:7587241311865923083:2049]
2025-12-24T02:11:18.066118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241311865923083:2049] [node 3] ICS09 handshake done sender: [1:7587241307570955760:2106] self: [1:7587241307570955758:0] peer: [3:7587241315124894871:0] socket: 560 qp: -1
2025-12-24T02:11:18.066125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241311865923083:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.066132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.066729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310955858295:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.066788Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241315250825663:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.066998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241311865923085:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11065
2025-12-24T02:11:18.067489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307570955767:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.068109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241307022300764:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.068117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311317268066:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.068124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311317268066:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.068164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894874:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-12-24T02:11:18.068188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241310829927491:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.068201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.068208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310829927491:2048] [node 1] ICP22 created new session: [3:7587241315124894876:2048]
2025-12-24T02:11:18.068215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315124894876:2048] [node 1] ICS09 handshake done sender: [3:7587241310829927548:2104] self: [3:7587241315124894871:0] peer: [1:7587241307570955758:0] socket: 565 qp: -1
2025-12-24T02:11:18.068220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315124894876:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.068229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.069034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241307022300693:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.069050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.069056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241307022300693:2049] [node 3] ICP22 created new session: [2:7587241311317268079:2049]
2025-12-24T02:11:18.069063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311317268079:2049] [node 3] ICS09 handshake done sender: [2:7587241307022300764:2112] self: [2:7587241307022300753:0] peer: [3:7587241315124894872:0] socket: 573 qp: -1
2025-12-24T02:11:18.069070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311317268079:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.069077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.069503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241307022300766:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.069752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241307022300762:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.069772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.069779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241307022300762:2109] [node 6] ICP22 created new session: [2:7587241311317268083:2109]
2025-12-24T02:11:18.069785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311317268083:2109] [node 6] ICS09 handshake done sender: [2:7587241307022300766:2113] self: [2:7587241307022300765:0] peer: [6:7587241311182383574:0] socket: 572 qp: -1
2025-12-24T02:11:18.069790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311317268083:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.069796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.069857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310955858358:2107] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.069875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.069882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310955858358:2107] [node 4] ICP22 created new session: [5:7587241315250825678:2107]
2025-12-24T02:11:18.069888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241315250825678:2107] [node 4] ICS09 handshake done sender: [5:7587241315250825663:2114] self: [5:7587241315250825671:0] peer: [4:7587241310070390395:0] socket: 591 qp: -1
2025-12-24T02:11:18.069893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241315250825678:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.069900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.070460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241306887416194:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12093) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241306887416260:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.071139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315124894879:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28177
2025-12-24T02:11:18.070925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307570955763:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.070940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.070949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307570955763:2107] [node 4] ICP22 created new session: [1:7587241311865923088:2107]
2025-12-24T02:11:18.070955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241311865923088:2107] [node 4] ICS09 handshake done sender: [1:7587241307570955767:2109] self: [1:7587241307570955764:0] peer: [4:7587241314365357705:0] socket: 567 qp: -1
2025-12-24T02:11:18.070961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241311865923088:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.070968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.071110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241311865923077:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.071126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241311865923076:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.071312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383576:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.071942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315124894861:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.072182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825680:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.072236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825674:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.072279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314365357711:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:18.072353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241310955858294:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.072694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314365357710:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:18.072260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311317268069:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.072802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315124894863:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.072918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241310070390390:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.072932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241315250825664:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:18.073006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241310070390324:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.073036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.073042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310070390324:2048] [node 1] ICP22 created new session: [4:7587241314365357712:2048]
2025-12-24T02:11:18.073046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314365357712:2048] [node 1] ICS09 handshake done sender: [4:7587241310070390390:2110] self: [4:7587241314365357705:0] peer: [1:7587241307570955764:0] socket: 568 qp: -1
2025-12-24T02:11:18.073051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314365357712:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.073056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.072239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383577:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.072831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241306887416260:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.072972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241306887416194:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.072981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.072993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241306887416194:2049] [node 2] ICP22 created new session: [6:7587241311182383578:2049]
2025-12-24T02:11:18.072999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311182383578:2049] [node 2] ICS09 handshake done sender: [6:7587241306887416260:2111] self: [6:7587241311182383574:0] peer: [2:7587241307022300765:0] socket: 574 qp: -1
2025-12-24T02:11:18.073011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311182383578:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.073036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.073958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894869:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.073999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894874:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.074007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241315124894854:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.074019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.074028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315124894854:2108] [node 4] ICP22 created new session: [3:7587241315124894881:2108]
2025-12-24T02:11:18.074034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315124894881:2108] [node 4] ICS09 handshake done sender: [3:7587241315124894861:2114] self: [3:7587241315124894860:0] peer: [4:7587241314365357704:0] socket: 586 qp: -1
2025-12-24T02:11:18.074040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315124894881:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:18.074073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.073039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310955858296:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.073051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.073057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310955858296:2050] [node 3] ICP22 created new session: [5:7587241315250825684:2050]
2025-12-24T02:11:18.073064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241315250825684:2050] [node 3] ICS09 handshake done sender: [5:7587241315250825664:2115] self: [5:7587241315250825675:0] peer: [3:7587241315124894862:0] socket: 592 qp: -1
2025-12-24T02:11:18.073070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241315250825684:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.074090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241307570955699:2049] [node 3] ICP18 handshake (actor [1:7587241311865923076:2117]) is too old
2025-12-24T02:11:18.073076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.074326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315124894858:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.074316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311317268066:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.074951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955772:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.074960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307570955772:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.074964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825682:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.075017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383580:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.075554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241307570955774:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.075563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241307570955774:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.075430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311182383576:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.075466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241306887416193:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.076075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241307570955763:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.076089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241307022300761:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.076118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.076125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241307022300761:2108] [node 5] ICP22 created new session: [2:7587241311317268087:2108]
2025-12-24T02:11:18.076130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311317268087:2108] [node 5] ICS09 handshake done sender: [2:7587241311317268066:2114] self: [2:7587241307022300763:0] peer: [5:7587241315250825677:0] socket: 571 qp: -1
2025-12-24T02:11:18.076136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311317268087:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.076142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.076256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314365357714:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:18.076328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307570955774:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.076040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241315124894855:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.076054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.076065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315124894855:2109] [node 5] ICP22 created new session: [3:7587241315124894883:2109]
2025-12-24T02:11:18.076073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315124894883:2109] [node 5] ICS09 handshake done sender: [3:7587241315124894863:2115] self: [3:7587241315124894862:0] peer: [5:7587241315250825675:0] socket: 587 qp: -1
2025-12-24T02:11:18.076079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315124894883:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:18.076087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.076324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241310829927492:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.076332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.076336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310829927492:2049] [node 2] ICP22 created new session: [3:7587241315124894885:2049]
2025-12-24T02:11:18.076341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315124894885:2049] [node 2] ICS09 handshake done sender: [3:7587241315124894858:2112] self: [3:7587241315124894872:0] peer: [2:7587241307022300753:0] socket: 579 qp: -1
2025-12-24T02:11:18.076346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315124894885:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.076351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.076408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314365357699:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.076845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241315124894855:2109] [node 5] ICP18 handshake (actor [3:7587241315124894869:2118]) is too old
2025-12-24T02:11:18.076922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241315250825673:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.077090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310955858295:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.077104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.077109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310955858295:2049] [node 2] ICP22 created new session: [5:7587241315250825686:2049]
2025-12-24T02:11:18.077114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241315250825686:2049] [node 2] ICS09 handshake done sender: [5:7587241315250825673:2118] self: [5:7587241315250825677:0] peer: [2:7587241307022300763:0] socket: 606 qp: -1
2025-12-24T02:11:18.077132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241315250825686:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.077137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.077646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307570955771:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.077661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.077666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307570955771:2110] [node 6] ICP22 created new session: [1:7587241311865923092:2110]
2025-12-24T02:11:18.077670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241311865923092:2110] [node 6] ICS09 handshake done sender: [1:7587241307570955774:2112] self: [1:7587241307570955773:0] peer: [6:7587241311182383581:0] socket: 616 qp: -1
2025-12-24T02:11:18.077674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241311865923092:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:18.077679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.076963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241310070390326:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.076976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.076983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310070390326:2050] [node 3] ICP22 created new session: [4:7587241314365357715:2050]
2025-12-24T02:11:18.076988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314365357715:2050] [node 3] ICS09 handshake done sender: [4:7587241314365357699:2116] self: [4:7587241314365357704:0] peer: [3:7587241315124894860:0] socket: 590 qp: -1
2025-12-24T02:11:18.076993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314365357715:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.076999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.077754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314365357711:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.077791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241310070390326:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.077818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241310070390325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.076842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311317268089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15663
2025-12-24T02:11:18.078526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241307570955772:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.078590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241307570955766:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.078606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.078615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241307570955766:2108] [node 5] ICP22 created new session: [1:7587241311865923095:2108]
2025-12-24T02:11:18.078625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241311865923095:2108] [node 5] ICS09 handshake done sender: [1:7587241307570955772:2111] self: [1:7587241307570955770:0] peer: [5:7587241315250825681:0] socket: 577 qp: -1
2025-12-24T02:11:18.078630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241311865923095:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.078640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.078747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311317268067:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.078757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311317268067:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.080440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241306887416193:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241311182383576:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.086852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383583:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.086920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311317268067:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.086988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314365357718:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62838
2025-12-24T02:11:18.087076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314365357711:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.087188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241307022300755:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.087211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.087219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241307022300755:2107] [node 4] ICP22 created new session: [2:7587241311317268091:2107]
2025-12-24T02:11:18.087226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311317268091:2107] [node 4] ICS09 handshake done sender: [2:7587241311317268067:2115] self: [2:7587241307022300756:0] peer: [4:7587241314365357717:0] socket: 622 qp: -1
2025-12-24T02:11:18.087232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311317268091:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.087240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.087152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241310070390325:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.087182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.087189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241310070390325:2049] [node 2] ICP22 created new session: [4:7587241314365357719:2049]
2025-12-24T02:11:18.087195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314365357719:2049] [node 2] ICS09 handshake done sender: [4:7587241314365357711:2118] self: [4:7587241314365357717:0] peer: [2:7587241307022300756:0] socket: 624 qp: -1
2025-12-24T02:11:18.087202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314365357719:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.087210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.089635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383582:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.089753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311182383576:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.089810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241306887416193:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.089699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241310955858294:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241315250825674:2119] held: no
2025-12-24T02:11:18.089842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.089854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241306887416193:2048] [node 1] ICP22 created new session: [6:7587241311182383584:2048]
2025-12-24T02:11:18.089861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311182383584:2048] [node 1] ICS09 handshake done sender: [6:7587241311182383576:2119] self: [6:7587241311182383581:0] peer: [1:7587241307570955773:0] socket: 619 qp: -1
2025-12-24T02:11:18.089868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311182383584:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.089877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.090522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311182383580:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.090580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241306887416255:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.090739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241315250825688:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25946
2025-12-24T02:11:18.090749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241315250825662:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.090755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241315250825662:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.091000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241315250825674:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.090996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311182383583:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.091039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241306887416195:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.091083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241310955858294:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.091098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.091105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241310955858294:2048] [node 1] ICP22 created new session: [5:7587241315250825690:2048]
2025-12-24T02:11:18.091111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241315250825690:2048] [node 1] ICS09 handshake done sender: [5:7587241315250825674:2119] self: [5:7587241315250825681:0] peer: [1:7587241307570955770:0] socket: 607 qp: -1
2025-12-24T02:11:18.091122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241315250825690:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.091129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.091125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315124894868:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.091140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315124894868:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.091301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241315250825662:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.091423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241315250825656:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.091443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.091452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241315250825656:2108] [node 6] ICP22 created new session: [5:7587241315250825692:2108]
2025-12-24T02:11:18.091472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241315250825692:2108] [node 6] ICS09 handshake done sender: [5:7587241315250825662:2113] self: [5:7587241315250825661:0] peer: [6:7587241311182383586:0] socket: 632 qp: -1
2025-12-24T02:11:18.091478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241315250825692:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:18.091484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.091931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315124894868:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.092188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241306887416255:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23141) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241311182383580:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.092031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241310829927553:2107] [node 6] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:18.092056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.02025-12-24T02:11:18.092064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241310829927553:2107] [node 6] ICP22 created new session: [3:7587241315124894890:2107]
2025-12-24T02:11:18.092070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315124894890:2107] [node 6] ICS09 handshake done sender: [3:7587241315124894868:2117] self: [3:7587241315124894857:0] peer: [6:7587241311182383587:0] socket: 583 qp: -1
2025-12-24T02:11:18.092075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315124894890:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:18.092096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.092264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383589:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.092503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311182383588:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19029
2025-12-24T02:11:18.092868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241306887416195:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4462) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241311182383583:2123] held: no
2025-12-24T02:11:18.092897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311182383583:2123] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.092913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311182383580:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.092975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241306887416195:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.092982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241306887416255:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.092990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.092990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.092995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241306887416195:2050] [node 3] ICP22 created new session: [6:7587241311182383590:2050]
2025-12-24T02:11:18.092997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241306887416255:2108] [node 5] ICP22 created new session: [6:7587241311182383591:2108]
2025-12-24T02:11:18.093000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311182383590:2050] [node 3] ICS09 handshake done sender: [6:7587241311182383583:2123] self: [6:7587241311182383587:0] peer: [3:7587241315124894857:0] socket: 584 qp: -1
2025-12-24T02:11:18.093003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311182383591:2108] [node 5] ICS09 handshake done sender: [6:7587241311182383580:2121] self: [6:7587241311182383586:0] peer: [5:7587241315250825661:0] socket: 633 qp: -1
2025-12-24T02:11:18.093006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311182383590:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.093010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311182383591:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.093012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.093016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.102245Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:18.102340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241311317268075:2048] [node 1] ICS07 socket disconnect 563 reason# EndOfStream
2025-12-24T02:11:18.102351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241311317268075:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.102409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.102446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241311317268075:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.102452Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:18.102534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241315124894876:2048] [node 1] ICS07 socket disconnect 565 reason# EndOfStream
2025-12-24T02:11:18.102539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241315124894876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.102583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.102604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241315124894876:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.103129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241310829927491:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.103141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241315124894876:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.103146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241310829927491:2048] [node 1] ICP30 unregister session Session# [3:7587241315124894876:2048] VirtualId# [3:7587241315124894871:0]
2025-12-24T02:11:18.103166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.103171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241310829927491:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.103173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.103174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241307022300692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.103188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241311317268075:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.103193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241307022300692:2048] [node 1] ICP30 unregister session Session# [2:7587241311317268075:2048] VirtualId# [2:7587241311317268071:0]
2025-12-24T02:11:18.103215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.103221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241307022300692:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.103223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14635) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.103276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:18.103399Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:18.103358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241314365357712:2048] [node 1] ICS01 socket: 568 reason# EndOfStream
2025-12-24T02:11:18.103370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241310070390324:2048] [node 1] ICP30 unregister session Session# [4:7587241314365357712:2048] VirtualId# [4:7587241314365357705:0]
2025-12-24T02:11:18.103379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241314365357712:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.103438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.103481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.103490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241315250825690:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:18.103501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310955858294:2048] [node 1] ICP30 unregister session Session# [5:7587241315250825690:2048] VirtualId# [5:7587241315250825681:0]
2025-12-24T02:11:18.103506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241315250825690:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.103578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.103617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.103620Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:18.103711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311182383584:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-12-24T02:11:18.103717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241306887416193:2048] [node 1] ICP30 unregister session Session# [6:7587241311182383584:2048] VirtualId# [6:7587241311182383581:0]
2025-12-24T02:11:18.103722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311182383584:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.103801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.103826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.112909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241310829927491:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:18.115156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:18.115239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241315124894885:2049] [node 2] ICS07 socket disconnect 579 reason# EndOfStream
2025-12-24T02:11:18.115248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241315124894885:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.115302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.115322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241315124894885:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:18.115672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241310829927492:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12093) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.115685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241315124894885:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.115689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241310829927492:2049] [node 2] ICP30 unregister session Session# [3:7587241315124894885:2049] VirtualId# [3:7587241315124894872:0]
2025-12-24T02:11:18.115702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.115711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241310829927492:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.115716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12093) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.116262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:18.116403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:18.116466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311182383578:2049] [node 2] ICS01 socket: 574 reason# EndOfStream
2025-12-24T02:11:18.116482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241306887416194:2049] [node 2] ICP30 unregister session Session# [6:7587241311182383578:2049] VirtualId# [6:7587241311182383574:0]
2025-12-24T02:11:18.116486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311182383578:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.116527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.116547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.116345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241315250825686:2049] [node 2] ICS01 socket: 606 reason# EndOfStream
2025-12-24T02:11:18.116358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310955858295:2049] [node 2] ICP30 unregister session Session# [5:7587241315250825686:2049] VirtualId# [5:7587241315250825677:0]
2025-12-24T02:11:18.116364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241315250825686:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.116421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.116455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.117495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:18.117567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241314365357719:2049] [node 2] ICS01 socket: 624 reason# EndOfStream
2025-12-24T02:11:18.117573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241310070390325:2049] [node 2] ICP30 unregister session Session# [4:7587241314365357719:2049] VirtualId# [4:7587241314365357717:0]
2025-12-24T02:11:18.117578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241314365357719:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.117614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.117630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.129120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:18.129088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:18.129181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241314365357715:2050] [node 3] ICS01 socket: 590 reason# EndOfStream
2025-12-24T02:11:18.129189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241310070390326:2050] [node 3] ICP30 unregister session Session# [4:7587241314365357715:2050] VirtualId# [4:7587241314365357704:0]
2025-12-24T02:11:18.129196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241314365357715:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.129246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.129292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.129209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241315250825684:2050] [node 3] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:18.129645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310955858296:2050] [node 3] ICP30 unregister session Session# [5:7587241315250825684:2050] VirtualId# [5:7587241315250825675:0]
2025-12-24T02:11:18.129652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241315250825684:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.129868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.129906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.131186Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:18.131280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311182383590:2050] [node 3] ICS01 socket: 584 reason# EndOfStream
2025-12-24T02:11:18.131292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241306887416195:2050] [node 3] ICP30 unregister session Session# [6:7587241311182383590:2050] VirtualId# [6:7587241311182383587:0]
2025-12-24T02:11:18.131298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311182383590:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.131347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.131385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.139088Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:18.139184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241311182383572:2113] [node 4] ICS07 socket disconnect 564 reason# EndOfStream
2025-12-24T02:11:18.139190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311182383572:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.139252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:18.139251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:18.139287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241311182383572:2113] [node 4] ICS15 start handshake
2025-12-24T02:11:18.139346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241315250825678:2107] [node 4] ICS07 socket disconnect 591 reason# EndOfStream
2025-12-24T02:11:18.139357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241315250825678:2107] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.139425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:18.139445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241315250825678:2107] [node 4] ICS15 start handshake
2025-12-24T02:11:18.139568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241311182383563:2113] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.139612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311182383572:2113] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.139617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311182383563:2113] [node 4] ICP30 unregister session Session# [6:7587241311182383572:2113] VirtualId# [6:7587241311182383570:0]
2025-12-24T02:11:18.139644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:18.139650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241311182383563:2113] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.139652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.139820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241310955858358:2107] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.139832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241315250825678:2107] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.139836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241310955858358:2107] [node 4] ICP30 unregister session Session# [5:7587241315250825678:2107] VirtualId# [5:7587241315250825671:0]
2025-12-24T02:11:18.139847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:18.139852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241310955858358:2107] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.139854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30453) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.150783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241311182383563:2113] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:18.166825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241312394191887:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:18.166839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241312394191888:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.169753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312402008639:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.169759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312402008640:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.172923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241314770778753:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.172934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241314770778755:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.172946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241314770778754:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.174333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311303308499:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.174340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311303308500:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.174344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311303308501:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.171443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311501298116:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.171452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311501298117:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.171459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311501298118:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.168358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241311061969023:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.168371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241311061969024:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.186569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241312394191948:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:18.185883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311303308560:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.186988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778814:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.189720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241312394191954:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.189827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241312394191955:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.192982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241312394191959:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23122
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.199433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308564:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.199867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312402008699:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:18.199329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969093:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
2025-12-24T02:11:18.199452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241311061969086:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:18.202965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241311061969096:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.203844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312402008707:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.205466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969092:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
2025-12-24T02:11:18.205597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312402008704:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:18.205619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969101:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
2025-12-24T02:11:18.205629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298190:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
2025-12-24T02:11:18.205805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191959:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.206746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241311061969089:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.207061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969093:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.207102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241311061969023:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.207095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241312394191948:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20466834ns) {Configure:104(20466328ns) {ProcessConfigured:135(18805407ns) {StartInitialHandshake:157(18804703ns) {DropHandshakes:470(179ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(140ns) PrepareNewSessionHandshake:477(794ns) {SwitchToState:179(293ns)}}}}}
2025-12-24T02:11:18.207285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191957:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.207289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241312394191957:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.208013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241312394191957:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.208176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241312394191887:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.208189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.208196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241312394191887:2048] [node 2] ICP22 created new session: [1:7587241312394191969:2048]
2025-12-24T02:11:18.208203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241312394191969:2048] [node 2] ICS09 handshake done sender: [1:7587241312394191957:2110] self: [1:7587241312394191951:0] peer: [2:7587241311061969103:0] socket: 572 qp: -1
2025-12-24T02:11:18.208218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241312394191969:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:18.208226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.209049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311303308564:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.209825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778814:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.210373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008716:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.210122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311501298189:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:18.210712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311501298188:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:18.210429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241312394191972:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23122
2025-12-24T02:11:18.210917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969092:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.211422Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241314770778754:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16113157ns) {Configure:104(16110859ns) {ProcessConfigured:135(13939564ns) {StartInitialHandshake:157(13939092ns) {DropHandshakes:470(962ns) {DropIncomingHandshake:442(158ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:477(1123ns) {SwitchToState:179(341ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.211483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298194:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.211845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298194:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.211873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311501298118:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.211478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241312402008640:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21839642ns) {Configure:104(21838901ns) {ProcessConfigured:135(20583600ns) {StartInitialHandshake:157(20583103ns) {DropHandshakes:470(155ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(785ns) {SwitchToState:179(357ns)}}}}}
2025-12-24T02:11:18.211526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008718:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.211574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008716:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.211775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008722:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.212153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298190:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.212169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311501298117:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.212243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969098:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.212252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311061969098:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.212614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191972:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.212647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241312394191887:2048] [node 2] ICP18 handshake (actor [1:7587241312394191972:2116]) is too old
2025-12-24T02:11:18.212730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008718:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.212762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241312402008640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.213061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241311303308572:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.213512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969091:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.213522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311061969091:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.213556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008709:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.213566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312402008709:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.213706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241314770778822:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:18.213784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311061969098:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.213862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241314770778820:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:18.214116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008717:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.214317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008717:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.214356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241312402008639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.214853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311303308499:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.214934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191963:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.214943Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241312394191963:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.215606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241312394191963:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.215690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241312394191955:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.215701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.215706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241312394191955:2109] [node 6] ICP22 created new session: [1:7587241312394191974:2109]
2025-12-24T02:11:18.215711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241312394191974:2109] [node 6] ICS09 handshake done sender: [1:7587241312394191963:2113] self: [1:7587241312394191962:0] peer: [6:7587241311303308576:0] socket: 575 qp: -1
2025-12-24T02:11:18.215716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241312394191974:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.215722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.218773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969101:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.218837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241311061969086:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.219008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308578:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.219191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311303308578:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.219235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311303308501:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.218852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.218864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241311061969086:2107] [node 4] ICP22 created new session: [2:7587241311061969112:2107]
2025-12-24T02:11:18.218871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311061969112:2107] [node 4] ICS09 handshake done sender: [2:7587241311061969098:2114] self: [2:7587241311061969097:0] peer: [4:7587241311501298196:0] socket: 586 qp: -1
2025-12-24T02:11:18.218878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311061969112:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.218895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.219050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241311061969023:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241311061969093:2113] held: no
2025-12-24T02:11:18.219120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969109:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.219383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008711:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.219396Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312402008711:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.220326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312402008711:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.220422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312402008707:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.220444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.220451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312402008707:2108] [node 6] ICP22 created new session: [3:7587241312402008726:2108]
2025-12-24T02:11:18.220459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312402008726:2108] [node 6] ICS09 handshake done sender: [3:7587241312402008711:2112] self: [3:7587241312402008710:0] peer: [6:7587241311303308579:0] socket: 590 qp: -1
2025-12-24T02:11:18.220523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312402008726:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:18.220546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.222766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241311501298117:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241311501298190:2113] held: no
2025-12-24T02:11:18.222768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191958:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.222790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241312394191958:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.222751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778828:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.223044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778828:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.223077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241314770778753:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.226752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969108:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.226948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298198:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
2025-12-24T02:11:18.229790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241311303308572:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16652165ns) {Configure:104(16651456ns) {ProcessConfigured:135(16255297ns) {StartInitialHandshake:157(16254797ns) {DropHandshakes:470(674ns) {DropIncomingHandshake:442(319ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(86ns) PrepareNewSessionHandshake:477(973ns) {SwitchToState:179(227ns)}}}}}
2025-12-24T02:11:18.230611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311061969093:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.230780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241314770778820:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16768563ns) {Configure:104(16768172ns) {ProcessConfigured:135(16593390ns) {StartInitialHandshake:157(16592631ns) {DropHandshakes:470(271ns) {DropIncomingHandshake:442(37ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:477(1120ns) {SwitchToState:179(751ns)}}}}}
2025-12-24T02:11:18.230911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241311061969023:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.230923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312402008709:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.230932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.230940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241311061969023:2048] [node 1] ICP22 created new session: [2:7587241311061969115:2048]
2025-12-24T02:11:18.230947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311061969115:2048] [node 1] ICS09 handshake done sender: [2:7587241311061969093:2113] self: [2:7587241311061969103:0] peer: [1:7587241312394191951:0] socket: 578 qp: -1
2025-12-24T02:11:18.230955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311061969115:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.230975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.231099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241312394191977:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23122
2025-12-24T02:11:18.231126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008730:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.231240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191977:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.231309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312402008699:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.231338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.231344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312402008699:2106] [node 4] ICP22 created new session: [3:7587241312402008731:2106]
2025-12-24T02:11:18.231350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312402008731:2106] [node 4] ICS09 handshake done sender: [3:7587241312402008709:2111] self: [3:7587241312402008708:0] peer: [4:7587241311501298195:0] socket: 589 qp: -1
2025-12-24T02:11:18.231355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312402008731:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:18.231362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.231732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778833:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.233123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778833:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.233168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241314770778755:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.233264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312402008721:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.233277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312402008721:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.233942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312402008721:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.233944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778835:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.234049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241314770778833:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.234037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312402008704:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.234058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.234064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312402008704:2107] [node 5] ICP22 created new session: [3:7587241312402008735:2107]
2025-12-24T02:11:18.234071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312402008735:2107] [node 5] ICS09 handshake done sender: [3:7587241312402008721:2117] self: [3:7587241312402008715:0] peer: [5:7587241314770778834:0] socket: 621 qp: -1
2025-12-24T02:11:18.234076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312402008735:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:18.234084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.234147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241314770778755:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.234166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.234172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241314770778755:2050] [node 3] ICP22 created new session: [5:7587241314770778836:2050]
2025-12-24T02:11:18.234179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241314770778836:2050] [node 3] ICS09 handshake done sender: [5:7587241314770778833:2116] self: [5:7587241314770778834:0] peer: [3:7587241312402008715:0] socket: 622 qp: -1
2025-12-24T02:11:18.234185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241314770778836:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.234223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.234314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308581:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.234340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308582:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.234416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311303308564:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.234495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311303308499:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.234512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.234519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311303308499:2048] [node 1] ICP22 created new session: [6:7587241311303308583:2048]
2025-12-24T02:11:18.234525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311303308583:2048] [node 1] ICS09 handshake done sender: [6:7587241311303308564:2109] self: [6:7587241311303308576:0] peer: [1:7587241312394191962:0] socket: 576 qp: -1
2025-12-24T02:11:18.234531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311303308583:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.234537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.230813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298199:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
2025-12-24T02:11:18.230899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311501298190:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.244309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311061969091:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.244593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241312394191958:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.244519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241311061969024:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.244537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.244545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241311061969024:2049] [node 3] ICP22 created new session: [2:7587241311061969117:2049]
2025-12-24T02:11:18.244551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311061969117:2049] [node 3] ICS09 handshake done sender: [2:7587241311061969091:2111] self: [2:7587241311061969088:0] peer: [3:7587241312402008723:0] socket: 580 qp: -1
2025-12-24T02:11:18.244559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311061969117:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.244582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.247021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191965:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.247043Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241312394191965:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.250903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241311061969120:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4109
2025-12-24T02:11:18.251017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241312394191888:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.251045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.251051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241312394191888:2049] [node 3] ICP22 created new session: [1:7587241312394191980:2049]
2025-12-24T02:11:18.251058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241312394191980:2049] [node 3] ICS09 handshake done sender: [1:7587241312394191958:2111] self: [1:7587241312394191952:0] peer: [3:7587241312402008724:0] socket: 573 qp: -1
2025-12-24T02:11:18.251064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241312394191980:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.251071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.250857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241314770778822:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37073270ns) {Configure:104(37072857ns) {ProcessConfigured:135(36774046ns) {StartInitialHandshake:157(36752879ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(1831ns) {SwitchToState:179(949ns)}} IncomingHandshake:261(9663ns) {DropIncomingHandshake:442(117ns) PrepareNewSessionHandshake:477(99ns) {SwitchToState:179(66ns)} GenerateSessionVirtualId:988(513ns) IssueIncomingHandshakeReply:193(314ns)}}}}
2025-12-24T02:11:18.255658Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241312402008704:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(21586314ns) {HandleHandshakeStatus:329(21585491ns) {DropIncomingHandshake:442(107ns) DropOutgoingHandshake:457(385ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(21501491ns) {ForwardSessionEventToSession:612(21479709ns) ForwardSessionEventToSession:612(6914ns) ForwardSessionEventToSession:612(823ns) ForwardSessionEventToSession:612(2720ns) ForwardSessionEventToSession:612(642ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(497ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(1149ns)}}}
2025-12-24T02:11:18.254902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298199:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.256504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312402008718:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.256604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241312402008639:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241312402008717:2114] held: no
2025-12-24T02:11:18.256630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241312402008640:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.256645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.256654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312402008640:2049] [node 2] ICP22 created new session: [3:7587241312402008737:2049]
2025-12-24T02:11:18.256658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312402008737:2049] [node 2] ICS09 handshake done sender: [3:7587241312402008718:2115] self: [3:7587241312402008723:0] peer: [2:7587241311061969088:0] socket: 595 qp: -1
2025-12-24T02:11:18.256666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312402008737:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.256675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.258796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969120:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.258971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241311061969024:2049] [node 3] ICP18 handshake (actor [2:7587241311061969120:2120]) is too old
2025-12-24T02:11:18.259838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191966:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.259852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241312394191966:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.260552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241312394191966:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.260635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241312394191948:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.260653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.260660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241312394191948:2107] [node 4] ICP22 created new session: [1:7587241312394191983:2107]
2025-12-24T02:11:18.260666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241312394191983:2107] [node 4] ICS09 handshake done sender: [1:7587241312394191966:2115] self: [1:7587241312394191953:0] peer: [4:7587241311501298202:0] socket: 611 qp: -1
2025-12-24T02:11:18.260673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241312394191983:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.260681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.258794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241311501298118:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26921) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241311501298194:2115] held: no
2025-12-24T02:11:18.258897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311501298116:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.259008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298200:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.263426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.263627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311303308581:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.263667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311303308500:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.263785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969100:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.263799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311061969100:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.263834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.264181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312402008717:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.264239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241312402008639:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.264255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.264262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312402008639:2048] [node 1] ICP22 created new session: [3:7587241312402008741:2048]
2025-12-24T02:11:18.264271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312402008741:2048] [node 1] ICS09 handshake done sender: [3:7587241312402008717:2114] self: [3:7587241312402008724:0] peer: [1:7587241312394191952:0] socket: 594 qp: -1
2025-12-24T02:11:18.264277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312402008741:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.264284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.264414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312402008740:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10601
2025-12-24T02:11:18.264451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311061969100:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.264541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241311061969096:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.264562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.264570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241311061969096:2109] [node 6] ICP22 created new session: [2:7587241311061969122:2109]
2025-12-24T02:11:18.264581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311061969122:2109] [node 6] ICS09 handshake done sender: [2:7587241311061969100:2115] self: [2:7587241311061969099:0] peer: [6:7587241311303308586:0] socket: 609 qp: -1
2025-12-24T02:11:18.264587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311061969122:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.264594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.265023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241311303308500:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3344) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241311303308581:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.267325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778842:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.267051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298201:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.268781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308587:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.268879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778841:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.268912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241314770778754:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.268703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241312394191985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23122
2025-12-24T02:11:18.268809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241312394191965:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.270815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241312394191985:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.270831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241312394191954:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.270842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.270850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241312394191954:2108] [node 5] ICP22 created new session: [1:7587241312394191987:2108]
2025-12-24T02:11:18.270856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241312394191987:2108] [node 5] ICS09 handshake done sender: [1:7587241312394191965:2114] self: [1:7587241312394191956:0] peer: [5:7587241314770778831:0] socket: 598 qp: -1
2025-12-24T02:11:18.270863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241312394191987:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.270872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.274903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311303308578:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.274923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241311501298116:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241311501298199:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.275024Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311501298194:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.275678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778844:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.275573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241312394191989:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23122
2025-12-24T02:11:18.276304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241312394191955:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.278845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241314770778828:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.278790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311303308501:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.278824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.278836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311303308501:2050] [node 3] ICP22 created new session: [6:7587241311303308590:2050]
2025-12-24T02:11:18.278843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311303308590:2050] [node 3] ICS09 handshake done sender: [6:7587241311303308578:2115] self: [6:7587241311303308579:0] peer: [3:7587241312402008710:0] socket: 593 qp: -1
2025-12-24T02:11:18.278856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311303308590:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.278880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.278979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241314770778753:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.279002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.279009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241314770778753:2048] [node 1] ICP22 created new session: [5:7587241314770778845:2048]
2025-12-24T02:11:18.279015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241314770778845:2048] [node 1] ICS09 handshake done sender: [5:7587241314770778828:2114] self: [5:7587241314770778831:0] peer: [1:7587241312394191956:0] socket: 600 qp: -1
2025-12-24T02:11:18.278840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311501298118:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.278859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.278866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311501298118:2050] [node 3] ICP22 created new session: [4:7587241311501298205:2050]
2025-12-24T02:11:18.279024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241314770778845:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.278874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241311501298205:2050] [node 3] ICS09 handshake done sender: [4:7587241311501298194:2115] self: [4:7587241311501298195:0] peer: [3:7587241312402008708:0] socket: 591 qp: -1
2025-12-24T02:11:18.279032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.278881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241311501298205:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.278888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.279216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308588:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.279739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969109:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.282733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241311061969105:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.282762Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241311061969105:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.283204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311303308581:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.283327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311303308500:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.283344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.283353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311303308500:2049] [node 2] ICP22 created new session: [6:7587241311303308592:2049]
2025-12-24T02:11:18.283359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311303308592:2049] [node 2] ICS09 handshake done sender: [6:7587241311303308581:2117] self: [6:7587241311303308586:0] peer: [2:7587241311061969099:0] socket: 612 qp: -1
2025-12-24T02:11:18.283364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311303308592:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.283371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.282728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311501298117:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.282759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.282771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311501298117:2049] [node 2] ICP22 created new session: [4:7587241311501298207:2049]
2025-12-24T02:11:18.282778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241311501298207:2049] [node 2] ICS09 handshake done sender: [4:7587241311501298190:2113] self: [4:7587241311501298196:0] peer: [2:7587241311061969097:0] socket: 587 qp: -1
2025-12-24T02:11:18.282786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241311501298207:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.282794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.282874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298201:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.286965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298204:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
2025-12-24T02:11:18.287376Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311501298199:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.287495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311501298116:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.287506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.287513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311501298116:2048] [node 1] ICP22 created new session: [4:7587241311501298212:2048]
2025-12-24T02:11:18.287520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241311501298212:2048] [node 1] ICS09 handshake done sender: [4:7587241311501298199:2118] self: [4:7587241311501298202:0] peer: [1:7587241312394191953:0] socket: 614 qp: -1
2025-12-24T02:11:18.287526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241311501298212:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.287545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.287698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311501298210:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64082
2025-12-24T02:11:18.290812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241312394191954:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19938495ns) {HandleHandshakeStatus:329(19937683ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(267ns) SwitchToState:179(459ns) ProcessPendingSessionEvents:495(19848425ns) {ForwardSessionEventToSession:612(19834397ns) ForwardSessionEventToSession:612(2516ns) ForwardSessionEventToSession:612(2169ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(650ns) ForwardSessionEventToSession:612(683ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(737ns) ForwardSessionEventToSession:612(965ns)}}}
2025-12-24T02:11:18.290889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241312394191954:2108] [node 5] ICP18 handshake (actor [1:7587241312394191985:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.292138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308594:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.292556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311303308594:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.292595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311303308572:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.292753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241312402008707:2108] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.293431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308596:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.294012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311303308594:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.294081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311303308572:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.294098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.294104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311303308572:2112] [node 4] ICP22 created new session: [6:7587241311303308597:2112]
2025-12-24T02:11:18.294110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311303308597:2112] [node 4] ICS09 handshake done sender: [6:7587241311303308594:2121] self: [6:7587241311303308595:0] peer: [4:7587241311501298191:0] socket: 620 qp: -1
2025-12-24T02:11:18.294116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311303308597:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:18.294125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.292841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298197:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.292853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241311501298197:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.293838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311501298197:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.294384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241311501298189:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.294393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.294398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311501298189:2112] [node 6] ICP22 created new session: [4:7587241311501298215:2112]
2025-12-24T02:11:18.294403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241311501298215:2112] [node 6] ICS09 handshake done sender: [4:7587241311501298197:2116] self: [4:7587241311501298191:0] peer: [6:7587241311303308595:0] socket: 569 qp: -1
2025-12-24T02:11:18.294409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241311501298215:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:18.294416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.294881Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241311501298117:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(38394303ns) {HandleHandshakeStatus:329(38393642ns) {DropIncomingHandshake:442(1130ns) DropOutgoingHandshake:457(91ns) SwitchToState:179(170ns) ProcessPendingSessionEvents:495(95ns)}}
2025-12-24T02:11:18.295604Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241314770778753:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16599646ns) {HandleHandshakeStatus:329(16587664ns) {DropIncomingHandshake:442(376ns) DropOutgoingHandshake:457(2782ns) SwitchToState:179(135ns) ProcessPendingSessionEvents:495(15895539ns) {ForwardSessionEventToSession:612(15884041ns) ForwardSessionEventToSession:612(2380ns) ForwardSessionEventToSession:612(1053ns) ForwardSessionEventToSession:612(574ns) ForwardSessionEventToSession:612(558ns) ForwardSessionEventToSession:612(933ns) ForwardSessionEventToSession:612(658ns) ForwardSessionEventToSession:612(565ns) ForwardSessionEventToSession:612(559ns) ForwardSessionEventToSession:612(705ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.296219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308599:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.297626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241311501298189:2112] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: 2025-12-24T02:11:18.302830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241311061969105:2117] [node 5] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:18.302946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778848:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.303626Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241311303308599:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.303679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241311303308560:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.304156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778839:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.304168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241314770778839:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.304820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241311303308601:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1567
2025-12-24T02:11:18.305019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241314770778839:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.305249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241311303308599:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.305591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241311303308560:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.305614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.305624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241311303308560:2107] [node 5] ICP22 created new session: [6:7587241311303308602:2107]
2025-12-24T02:11:18.305631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241311303308602:2107] [node 5] ICS09 handshake done sender: [6:7587241311303308599:2123] self: [6:7587241311303308600:0] peer: [5:7587241314770778829:0] socket: 632 qp: -1
2025-12-24T02:11:18.305639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241311303308602:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:18.305649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.306763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241314770778841:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.306855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241314770778754:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.306870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.306877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241314770778754:2049] [node 2] ICP22 created new session: [5:7587241314770778850:2049]
2025-12-24T02:11:18.306885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241314770778850:2049] [node 2] ICS09 handshake done sender: [5:7587241314770778841:2119] self: [5:7587241314770778843:0] peer: [2:7587241311061969104:0] socket: 604 qp: -1
2025-12-24T02:11:18.306892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241314770778850:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.306900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.306965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241311061969089:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.306999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.307006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241311061969089:2108] [node 5] ICP22 created new session: [2:7587241311061969126:2108]
2025-12-24T02:11:18.307011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241311061969126:2108] [node 5] ICS09 handshake done sender: [2:7587241311061969105:2117] self: [2:7587241311061969104:0] peer: [5:7587241314770778843:0] socket: 579 qp: -1
2025-12-24T02:11:18.307016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241311061969126:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.307025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.307039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241314770778842:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.307068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241314770778822:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.307101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.307108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241314770778822:2110] [node 6] ICP22 created new session: [5:7587241314770778852:2110]
2025-12-24T02:11:18.307118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241314770778852:2110] [node 6] ICS09 handshake done sender: [5:7587241314770778839:2118] self: [5:7587241314770778829:0] peer: [6:7587241311303308600:0] socket: 628 qp: -1
2025-12-24T02:11:18.307124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241314770778852:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:18.307132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.307893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241314770778820:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.314795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311501298193:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.314814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241311501298193:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.315618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241314770778855:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15173
2025-12-24T02:11:18.322856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311501298193:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.323245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241314770778842:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.323365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241314770778820:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.323385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.323396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241314770778820:2109] [node 4] ICP22 created new session: [5:7587241314770778856:2109]
2025-12-24T02:11:18.323403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241314770778856:2109] [node 4] ICS09 handshake done sender: [5:7587241314770778842:2120] self: [5:7587241314770778854:0] peer: [4:7587241311501298192:0] socket: 615 qp: -1
2025-12-24T02:11:18.323424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241314770778856:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:18.323439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.330870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241311501298188:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.330903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.330914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311501298188:2111] [node 5] ICP22 created new session: [4:7587241311501298218:2111]
2025-12-24T02:11:18.330920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241311501298218:2111] [node 5] ICS09 handshake done sender: [4:7587241311501298193:2114] self: [4:7587241311501298192:0] peer: [5:7587241314770778854:0] socket: 585 qp: -1
2025-12-24T02:11:18.330930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241311501298218:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:18.330952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.344091Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:18.344187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241311061969115:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-12-24T02:11:18.344196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241311061969023:2048] [node 1] ICP30 unregister session Session# [2:7587241311061969115:2048] VirtualId# [2:7587241311061969103:0]
2025-12-24T02:11:18.344204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241311061969115:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.344251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.344275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.344650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:18.344722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241312402008741:2048] [node 1] ICS01 socket: 594 reason# EndOfStream
2025-12-24T02:11:18.344727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241312402008639:2048] [node 1] ICP30 unregister session Session# [3:7587241312402008741:2048] VirtualId# [3:7587241312402008724:0]
2025-12-24T02:11:18.344733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241312402008741:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.344760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.344786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.349094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:18.349189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241311501298212:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-12-24T02:11:18.349198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241311501298116:2048] [node 1] ICP30 unregister session Session# [4:7587241311501298212:2048] VirtualId# [4:7587241311501298202:0]
2025-12-24T02:11:18.349204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241311501298212:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.349264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.349290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.349329Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:18.349394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241314770778845:2048] [node 1] ICS07 socket disconnect 600 reason# EndOfStream
2025-12-24T02:11:18.349398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241314770778845:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.349425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.349446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241314770778845:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.349889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241314770778753:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.349915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241314770778845:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.349920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241314770778753:2048] [node 1] ICP30 unregister session Session# [5:7587241314770778845:2048] VirtualId# [5:7587241314770778831:0]
2025-12-24T02:11:18.349939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.349946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241314770778753:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.349949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.350028Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:18.350105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241311303308583:2048] [node 1] ICS07 socket disconnect 576 reason# EndOfStream
2025-12-24T02:11:18.350115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311303308583:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.350170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.350203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241311303308583:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.350834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241311303308499:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.350853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311303308583:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.350858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311303308499:2048] [node 1] ICP30 unregister session Session# [6:7587241311303308583:2048] VirtualId# [6:7587241311303308576:0]
2025-12-24T02:11:18.350873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.350878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241311303308499:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.350880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21082) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.356860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:18.356959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241312402008737:2049] [node 2] ICS01 socket: 595 reason# EndOfStream
2025-12-24T02:11:18.356974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241312402008640:2049] [node 2] ICP30 unregister session Session# [3:7587241312402008737:2049] VirtualId# [3:7587241312402008723:0]
2025-12-24T02:11:18.356981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241312402008737:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.357029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.357058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.359154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241311303308499:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:18.360004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:18.360100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311303308592:2049] [node 2] ICS01 socket: 612 reason# EndOfStream
2025-12-24T02:11:18.360108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311303308500:2049] [node 2] ICP30 unregister session Session# [6:7587241311303308592:2049] VirtualId# [6:7587241311303308586:0]
2025-12-24T02:11:18.360116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311303308592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.360178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.360216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.362748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241314770778753:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:18.362786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:18.362867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241314770778850:2049] [node 2] ICS01 socket: 604 reason# EndOfStream
2025-12-24T02:11:18.362874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241314770778754:2049] [node 2] ICP30 unregister session Session# [5:7587241314770778850:2049] VirtualId# [5:7587241314770778843:0]
2025-12-24T02:11:18.362880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241314770778850:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.362939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.362965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.365493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:18.365591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241311501298207:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-12-24T02:11:18.365600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241311501298117:2049] [node 2] ICP30 unregister session Session# [4:7587241311501298207:2049] VirtualId# [4:7587241311501298196:0]
2025-12-24T02:11:18.365606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241311501298207:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.365670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.365696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.375884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:18.375960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241311303308590:2050] [node 3] ICS07 socket disconnect 593 reason# EndOfStream
2025-12-24T02:11:18.375965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311303308590:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.376010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.376030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241311303308590:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:18.376574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241311303308501:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26921) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.376586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311303308590:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.376592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311303308501:2050] [node 3] ICP30 unregister session Session# [6:7587241311303308590:2050] VirtualId# [6:7587241311303308579:0]
2025-12-24T02:11:18.376606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.376614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241311303308501:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26921) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.376615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26921) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.386301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241311303308501:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:18.382782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:18.382884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241314770778836:2050] [node 3] ICS01 socket: 622 reason# EndOfStream
2025-12-24T02:11:18.382893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241314770778755:2050] [node 3] ICP30 unregister session Session# [5:7587241314770778836:2050] VirtualId# [5:7587241314770778834:0]
2025-12-24T02:11:18.382900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241314770778836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.382959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.382990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.397137Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:18.397224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311303308597:2112] [node 4] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:18.397230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311303308572:2112] [node 4] ICP30 unregister session Session# [6:7587241311303308597:2112] VirtualId# [6:7587241311303308595:0]
2025-12-24T02:11:18.397241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311303308597:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.397293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:18.397317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:18.406241Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:18.406334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241311303308602:2107] [node 5] ICS01 socket: 632 reason# EndOfStream
2025-12-24T02:11:18.406346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241311303308560:2107] [node 5] ICP30 unregister session Session# [6:7587241311303308602:2107] VirtualId# [6:7587241311303308600:0]
2025-12-24T02:11:18.406354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241311303308602:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.406404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:18.406434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:18.421596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241313208298704:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:18.421697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241313208298705:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.430088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312300527408:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.430098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312300527409:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.430103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312300527410:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.423671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241312456822708:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.425108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312611047860:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.423682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241312456822709:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.425116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312611047861:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.428507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312509454238:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.428520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312509454239:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.428524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312509454240:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.426823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241314716736118:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.426831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241314716736119:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.426835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241314716736120:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.456736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241313208298765:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:18.458525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241313208298770:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.458617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241313208298771:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.459010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241313208298772:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3823
::ffff:127.1.0.0
2025-12-24T02:11:18.459438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527449:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.459261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047920:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
2025-12-24T02:11:18.464070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241312456822777:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.467810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822774:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
2025-12-24T02:11:18.470847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241312456822776:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:18.471399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312611047926:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.474757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298772:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.479547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241312456822779:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.482521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312300527449:2088] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.482561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312300527408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.479974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241314716736182:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.482838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312300527452:2090] [node 5] ICP01 ready to work
2025-12-24T02:11:18.483293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822774:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.483324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241312456822708:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.483434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298775:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.483447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241313208298775:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.484026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241313208298775:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.484136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241313208298704:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.484186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.484196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241313208298704:2048] [node 2] ICP22 created new session: [1:7587241313208298787:2048]
2025-12-24T02:11:18.484203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241313208298787:2048] [node 2] ICS09 handshake done sender: [1:7587241313208298775:2112] self: [1:7587241313208298774:0] peer: [2:7587241312456822786:0] socket: 573 qp: -1
2025-12-24T02:11:18.484215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241313208298787:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:18.484226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.484407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822790:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
2025-12-24T02:11:18.483348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312611047929:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.486897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454312:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.488126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241312456822708:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241312456822774:2111] held: no
2025-12-24T02:11:18.488167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241312456822774:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.488285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241312456822708:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.488304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.488316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241312456822708:2048] [node 1] ICP22 created new session: [2:7587241312456822791:2048]
2025-12-24T02:11:18.488323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241312456822791:2048] [node 1] ICS09 handshake done sender: [2:7587241312456822774:2111] self: [2:7587241312456822786:0] peer: [1:7587241313208298774:0] socket: 578 qp: -1
2025-12-24T02:11:18.488335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241312456822791:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.488356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.488492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822787:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.488538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822788:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
2025-12-24T02:11:18.488561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822789:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
2025-12-24T02:11:18.488742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822787:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.487372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241312611047860:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26710631ns) {Configure:104(26709372ns) {ProcessConfigured:135(25319844ns) {StartInitialHandshake:157(25319249ns) {DropHandshakes:470(813ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(639ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(418ns)}}}}}
2025-12-24T02:11:18.493619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454312:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.493682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312509454238:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.494332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298778:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.494349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241313208298778:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.494472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822788:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.494742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241312611047930:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:18.494228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241314716736184:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.494491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312509454302:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.494542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312509454301:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.494138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312300527465:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:18.499609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241312456822779:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19990218ns) {Configure:104(19989280ns) {ProcessConfigured:135(19644449ns) {StartInitialHandshake:157(19644118ns) {DropHandshakes:470(428ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(1266ns) {SwitchToState:179(545ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.503225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454316:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.504156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822789:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.504344Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241312300527452:2090] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(21425973ns) {Configure:104(21424923ns) {ProcessConfigured:135(21089338ns) {StartInitialHandshake:157(21089099ns) {DropHandshakes:470(694ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(513ns) {SwitchToState:179(270ns)}}}}}
2025-12-24T02:11:18.504394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527487:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.499836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241314716736182:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19803296ns) {Configure:104(19802810ns) {ProcessConfigured:135(19474322ns) {StartInitialHandshake:157(19474098ns) {DropHandshakes:470(529ns) {DropIncomingHandshake:442(146ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(1414ns) {SwitchToState:179(777ns)}}}}}
2025-12-24T02:11:18.503905Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241314716736119:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(40966638ns) {Configure:104(40965859ns) {ProcessConfigured:135(40418887ns) {StartInitialHandshake:157(40418123ns) {DropHandshakes:470(391ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(96ns)} GenerateSessionVirtualId:988(170ns) PrepareNewSessionHandshake:477(820ns) {SwitchToState:179(281ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.503384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047920:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.503475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241312611047860:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.504718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298781:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.504736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241313208298781:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.505996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298769:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.506003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241313208298769:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.506968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454316:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.514192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736196:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.514041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241313208298778:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.514276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241313208298771:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.514289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.514036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241312300527465:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19811216ns) {Configure:104(19810411ns) {ProcessConfigured:135(19462871ns) {StartInitialHandshake:157(19462319ns) {DropHandshakes:470(685ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(783ns)}}}}}
2025-12-24T02:11:18.514833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241312509454301:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20232278ns) {Configure:104(20231565ns) {ProcessConfigured:135(19914838ns) {StartInitialHandshake:157(19914560ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:477(933ns) {SwitchToState:179(901ns)}}}}}
2025-12-24T02:11:18.514883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312509454301:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.514772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312300527487:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.514461Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241312456822776:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43429008ns) {Configure:104(43428431ns) {ProcessConfigured:135(43114832ns) {StartInitialHandshake:157(43114640ns) {DropHandshakes:470(372ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1107ns) {SwitchToState:179(926ns)}}}}}
2025-12-24T02:11:18.514831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312300527410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.514981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736196:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.515053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241312509454302:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20488194ns) {Configure:104(20487600ns) {ProcessConfigured:135(20034443ns) {StartInitialHandshake:157(20034087ns) {DropHandshakes:470(475ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(912ns) {SwitchToState:179(433ns)}}}}}
2025-12-24T02:11:18.514296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241313208298771:2109] [node 6] ICP22 created new session: [1:7587241313208298792:2109]
2025-12-24T02:11:18.514304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241313208298792:2109] [node 6] ICS09 handshake done sender: [1:7587241313208298778:2113] self: [1:7587241313208298777:0] peer: [6:7587241312300527481:0] socket: 576 qp: -1
2025-12-24T02:11:18.514310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241313208298792:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.514331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.515046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047934:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.515095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736190:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.515062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312611047934:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.516070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312611047934:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.515105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241314716736190:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.516160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312611047929:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.516173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.516181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312611047929:2110] [node 6] ICP22 created new session: [3:7587241312611047941:2110]
2025-12-24T02:11:18.516187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312611047941:2110] [node 6] ICS09 handshake done sender: [3:7587241312611047934:2113] self: [3:7587241312611047933:0] peer: [6:7587241312300527490:0] socket: 601 qp: -1
2025-12-24T02:11:18.516194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312611047941:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:18.516201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.519872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241313208298789:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3823
2025-12-24T02:11:18.520312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298789:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.520350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241313208298771:2109] [node 6] ICP18 handshake (actor [1:7587241313208298789:2116]) is too old
2025-12-24T02:11:18.518824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241312611047930:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23970068ns) {Configure:104(23969386ns) {ProcessConfigured:135(23546461ns) {StartInitialHandshake:157(23545986ns) {DropHandshakes:470(434ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:477(1048ns) {SwitchToState:179(354ns)}}}}}
2025-12-24T02:11:18.519612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
2025-12-24T02:11:18.519634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047946:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
2025-12-24T02:11:18.519664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
No RDMA context found for GID: 2025-12-24T02:11:18.519705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047948:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
::ffff:127.1.0.0
2025-12-24T02:11:18.520024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047946:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.520073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241312611047929:2110] [node 6] ICP18 handshake (actor [3:7587241312611047946:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.523613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241313208298781:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.523915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241313208298770:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.523940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.523950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241313208298770:2108] [node 5] ICP22 created new session: [1:7587241313208298795:2108]
2025-12-24T02:11:18.523958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241313208298795:2108] [node 5] ICS09 handshake done sender: [1:7587241313208298781:2114] self: [1:7587241313208298780:0] peer: [5:7587241312509454313:0] socket: 600 qp: -1
2025-12-24T02:11:18.523970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241313208298795:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.523979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.524347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241313208298769:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.525687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527492:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.525146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736197:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.525635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736198:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.525743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314716736190:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.525850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241314716736182:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.525864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.525873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241314716736182:2108] [node 5] ICP22 created new session: [4:7587241314716736200:2108]
2025-12-24T02:11:18.525881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314716736200:2108] [node 5] ICS09 handshake done sender: [4:7587241314716736190:2112] self: [4:7587241314716736188:0] peer: [5:7587241312509454318:0] socket: 607 qp: -1
2025-12-24T02:11:18.525888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314716736200:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.525897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.524389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047947:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.524467Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047948:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.526895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527486:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.526979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527491:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.530805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736197:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.530832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047945:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.530898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241312611047861:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.535517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241313208298705:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.535548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.535560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241313208298705:2049] [node 3] ICP22 created new session: [1:7587241313208298798:2049]
2025-12-24T02:11:18.535569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241313208298798:2049] [node 3] ICS09 handshake done sender: [1:7587241313208298769:2110] self: [1:7587241313208298768:0] peer: [3:7587241312611047939:0] socket: 572 qp: -1
2025-12-24T02:11:18.535580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241313208298798:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.535591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.535806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822780:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.535829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241312456822780:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.535875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
2025-12-24T02:11:18.536847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312611047920:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.536953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241312611047860:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.536968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.536975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312611047860:2048] [node 1] ICP22 created new session: [3:7587241312611047953:2048]
2025-12-24T02:11:18.536981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312611047953:2048] [node 1] ICS09 handshake done sender: [3:7587241312611047920:2107] self: [3:7587241312611047939:0] peer: [1:7587241313208298768:0] socket: 574 qp: -1
2025-12-24T02:11:18.536992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312611047953:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.537019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.539590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312300527408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241312300527449:2088] held: no
2025-12-24T02:11:18.539660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312300527486:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.539679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312300527449:2088] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.540241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822795:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.540255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241312456822795:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.540583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822783:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.540586Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241312456822783:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.539773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312300527408:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.539792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.539798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312300527408:2048] [node 1] ICP22 created new session: [6:7587241312300527493:2048]
2025-12-24T02:11:18.539805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241312300527493:2048] [node 1] ICS09 handshake done sender: [6:7587241312300527449:2088] self: [6:7587241312300527481:0] peer: [1:7587241313208298777:0] socket: 577 qp: -1
2025-12-24T02:11:18.539812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241312300527493:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.539820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.539995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312300527409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.541260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241312456822795:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.541420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241312456822783:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.541491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241312456822779:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.541505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.541511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241312456822779:2108] [node 4] ICP22 created new session: [2:7587241312456822801:2108]
2025-12-24T02:11:18.541517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241312456822801:2108] [node 4] ICS09 handshake done sender: [2:7587241312456822795:2118] self: [2:7587241312456822785:0] peer: [4:7587241314716736203:0] socket: 605 qp: -1
2025-12-24T02:11:18.541521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241312456822801:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:18.541559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.541685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241312456822777:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.541697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.541702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241312456822777:2110] [node 6] ICP22 created new session: [2:7587241312456822803:2110]
2025-12-24T02:11:18.541706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241312456822803:2110] [node 6] ICS09 handshake done sender: [2:7587241312456822783:2113] self: [2:7587241312456822778:0] peer: [6:7587241312300527496:0] socket: 589 qp: -1
2025-12-24T02:11:18.541709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241312456822803:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:18.541713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.537342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241314716736120:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23350370ns) {Configure:104(23348621ns) {ProcessConfigured:135(23031448ns) {StartInitialHandshake:157(23030809ns) {DropHandshakes:470(568ns) {DropIncomingHandshake:442(97ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(1126ns) {SwitchToState:179(513ns)}}}}}
2025-12-24T02:11:18.538769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241314716736120:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.542966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241313208298800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3823
2025-12-24T02:11:18.542998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241313208298801:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3823
2025-12-24T02:11:18.543021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241313208298802:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3823
2025-12-24T02:11:18.544311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298800:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.544350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298783:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.544357Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241313208298783:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.543162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047932:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.543178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312611047932:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.544505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312611047932:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.547829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241313208298783:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.547899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241313208298801:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.547947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241313208298770:2108] [node 5] ICP18 handshake (actor [1:7587241313208298801:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.548125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241313208298765:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:18.548139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.548145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241313208298765:2107] [node 4] ICP22 created new session: [1:7587241313208298805:2107]
2025-12-24T02:11:18.548151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241313208298805:2107] [node 4] ICS09 handshake done sender: [1:7587241313208298783:2115] self: [1:7587241313208298782:0] peer: [4:7587241314716736205:0] socket: 613 qp: -1
2025-12-24T02:11:18.548155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241313208298805:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.548163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.544115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736198:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.544216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241314716736118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.545806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312300527409:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241312300527486:2114] held: no
2025-12-24T02:11:18.547514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527497:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.547579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.547870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312300527486:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.547983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312300527497:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.548000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312300527409:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.548016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.548023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312300527409:2049] [node 2] ICP22 created new session: [6:7587241312300527500:2049]
2025-12-24T02:11:18.548030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241312300527500:2049] [node 2] ICS09 handshake done sender: [6:7587241312300527486:2114] self: [6:7587241312300527496:0] peer: [2:7587241312456822778:0] socket: 595 qp: -1
2025-12-24T02:11:18.548036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241312300527500:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.548056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.554840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241312456822780:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.554965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241312456822709:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.554990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.554997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241312456822709:2049] [node 3] ICP22 created new session: [2:7587241312456822807:2049]
2025-12-24T02:11:18.555003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241312456822807:2049] [node 3] ICS09 handshake done sender: [2:7587241312456822780:2112] self: [2:7587241312456822775:0] peer: [3:7587241312611047951:0] socket: 585 qp: -1
2025-12-24T02:11:18.555009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241312456822807:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.555033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.555639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454320:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.555704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.555960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.555988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241312509454238:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241312509454312:2113] held: no
2025-12-24T02:11:18.558583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312509454316:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.556863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241314716736119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.556910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241314716736119:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(26035042ns) {IncomingHandshake:261(26034449ns) {DropIncomingHandshake:442(302ns) PrepareNewSessionHandshake:477(739ns) {SwitchToState:179(228ns)} GenerateSessionVirtualId:988(323ns) IssueIncomingHandshakeReply:193(26026750ns) {SwitchToState:179(682ns)}}}
2025-12-24T02:11:18.558753Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241314716736118:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241314716736198:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.560100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312300527465:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.562060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527503:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.561252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736193:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.561267Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241314716736193:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.561870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736206:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.561964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736208:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.562209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736209:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.562299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314716736193:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.562554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241314716736209:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.562563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314716736198:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.562623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241314716736184:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.562641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.562648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241314716736184:2109] [node 6] ICP22 created new session: [4:7587241314716736211:2109]
2025-12-24T02:11:18.562655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314716736211:2109] [node 6] ICS09 handshake done sender: [4:7587241314716736193:2113] self: [4:7587241314716736192:0] peer: [6:7587241312300527502:0] socket: 617 qp: -1
2025-12-24T02:11:18.562662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314716736211:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.562682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.562521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312300527497:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.560287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312611047926:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.560319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-12-24T02:11:18.560328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312611047926:2108] [node 4] ICP22 created new session: [3:7587241312611047956:2108]
::ffff:127.1.0.0
2025-12-24T02:11:18.560339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312611047956:2108] [node 4] ICS09 handshake done sender: [3:7587241312611047932:2112] self: [3:7587241312611047931:0] peer: [4:7587241314716736204:0] socket: 599 qp: -1
2025-12-24T02:11:18.560348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312611047956:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:18.560371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.562669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241312611047861:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241312611047945:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.562936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312509454312:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.562845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241314716736207:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17175
2025-12-24T02:11:18.562966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312300527465:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.562982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.562994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312300527465:2096] [node 4] ICP22 created new session: [6:7587241312300527504:2096]
2025-12-24T02:11:18.563000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241312300527504:2096] [node 4] ICS09 handshake done sender: [6:7587241312300527497:2120] self: [6:7587241312300527502:0] peer: [4:7587241314716736192:0] socket: 587 qp: -1
2025-12-24T02:11:18.563006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241312300527504:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:18.563013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.567251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241313208298765:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(19093249ns) {HandleHandshakeStatus:329(19092843ns) {DropIncomingHandshake:442(3246ns) DropOutgoingHandshake:457(181ns) SwitchToState:179(132ns) ProcessPendingSessionEvents:495(19020351ns) {ForwardSessionEventToSession:612(19008201ns) ForwardSessionEventToSession:612(1741ns) ForwardSessionEventToSession:612(958ns) ForwardSessionEventToSession:612(682ns) ForwardSessionEventToSession:612(589ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(652ns) ForwardSessionEventToSession:612(624ns) ForwardSessionEventToSession:612(755ns) ForwardSessionEventToSession:612(649ns)}}}
2025-12-24T02:11:18.566956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241314716736182:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(41066719ns) {HandleHandshakeStatus:329(41065598ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(246ns) SwitchToState:179(99ns) ProcessPendingSessionEvents:495(30198681ns) {ForwardSessionEventToSession:612(30184796ns) ForwardSessionEventToSession:612(2111ns) ForwardSessionEventToSession:612(1066ns) ForwardSessionEventToSession:612(490ns) ForwardSessionEventToSession:612(676ns) ForwardSessionEventToSession:612(1200ns) ForwardSessionEventToSession:612(569ns) ForwardSessionEventToSession:612(601ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(984ns)}}}
2025-12-24T02:11:18.566992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241314716736118:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.567007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.567015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241314716736118:2048] [node 1] ICP22 created new session: [4:7587241314716736213:2048]
2025-12-24T02:11:18.567021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314716736213:2048] [node 1] ICS09 handshake done sender: [4:7587241314716736198:2117] self: [4:7587241314716736205:0] peer: [1:7587241313208298782:0] socket: 623 qp: -1
2025-12-24T02:11:18.567029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314716736213:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.567037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.570853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241312611047958:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30468
2025-12-24T02:11:18.574943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312611047945:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.575034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241312611047861:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.575051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.575060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312611047861:2049] [node 2] ICP22 created new session: [3:7587241312611047960:2049]
2025-12-24T02:11:18.575066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312611047960:2049] [node 2] ICS09 handshake done sender: [3:7587241312611047945:2116] self: [3:7587241312611047951:0] peer: [2:7587241312456822775:0] socket: 588 qp: -1
2025-12-24T02:11:18.575072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312611047960:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.575080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.576255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241312611047860:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39258801ns) {HandleHandshakeStatus:329(39257955ns) {DropIncomingHandshake:442(427ns) DropOutgoingHandshake:457(2390ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(20732162ns) {ForwardSessionEventToSession:612(20718170ns) ForwardSessionEventToSession:612(1938ns) ForwardSessionEventToSession:612(1354ns) ForwardSessionEventToSession:612(587ns) ForwardSessionEventToSession:612(629ns) ForwardSessionEventToSession:612(766ns) ForwardSessionEventToSession:612(643ns) ForwardSessionEventToSession:612(550ns) ForwardSessionEventToSession:612(624ns) ForwardSessionEventToSession:612(802ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.578817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312509454301:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.578842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.578852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312509454301:2108] [node 4] ICP22 created new session: [5:7587241312509454324:2108]
2025-12-24T02:11:18.578860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312509454324:2108] [node 4] ICS09 handshake done sender: [5:7587241312509454316:2114] self: [5:7587241312509454318:0] peer: [4:7587241314716736188:0] socket: 608 qp: -1
2025-12-24T02:11:18.578870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312509454324:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:18.578897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.579080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312509454238:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.579088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.579094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312509454238:2048] [node 1] ICP22 created new session: [5:7587241312509454326:2048]
2025-12-24T02:11:18.579098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312509454326:2048] [node 1] ICS09 handshake done sender: [5:7587241312509454312:2113] self: [5:7587241312509454313:0] peer: [1:7587241313208298780:0] socket: 604 qp: -1
2025-12-24T02:11:18.579102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312509454326:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.579107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.579376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454323:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.579476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.579681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241314716736120:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241314716736196:2115] held: no
2025-12-24T02:11:18.579848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314716736196:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.581648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527506:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.582945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454322:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.583489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454328:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.583525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454323:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.583582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312509454240:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.583597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312509454239:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.583887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241312611047944:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.583904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241312611047944:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.583882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241314716736197:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.583794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312300527506:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.583847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312300527452:2090] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.585023Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312300527410:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1143) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241312300527487:2115] held: no
2025-12-24T02:11:18.585104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241312300527410:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(59146296ns) {HandleHandshakeStatus:395(59145291ns) {LogHandshakeFail:472(4483ns) {UpdateErrorStateLog:545(1168ns)} DropOutgoingHandshake:457(1588ns) ProcessPendingSessionEvents:495(29581ns) {DropSessionEvent:512(12488ns) DropSessionEvent:512(3555ns) DropSessionEvent:512(1613ns) DropSessionEvent:512(1345ns) DropSessionEvent:512(1458ns) DropSessionEvent:512(1959ns) DropSessionEvent:512(1333ns) DropSessionEvent:512(1352ns) DropSessionEvent:512(1118ns) DropSessionEvent:512(1676ns)}}}
2025-12-24T02:11:18.585155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312300527487:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.590807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241312456822709:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35801331ns) {HandleHandshakeStatus:329(35799913ns) {DropIncomingHandshake:442(7420ns) DropOutgoingHandshake:457(852ns) SwitchToState:179(190ns) ProcessPendingSessionEvents:495(32420259ns) {ForwardSessionEventToSession:612(32406188ns) ForwardSessionEventToSession:612(1714ns) ForwardSessionEventToSession:612(890ns) ForwardSessionEventToSession:612(568ns) ForwardSessionEventToSession:612(631ns) ForwardSessionEventToSession:612(1349ns) ForwardSessionEventToSession:612(802ns) ForwardSessionEventToSession:612(709ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(805ns)}}}
2025-12-24T02:11:18.591002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241312456822797:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.591008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241312456822797:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.590942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312509454319:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.590950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241312509454319:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.592023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454334:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.592038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312509454333:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9065
2025-12-24T02:11:18.592627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312509454328:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.592685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312509454240:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.592696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.592701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312509454240:2050] [node 3] ICP22 created new session: [5:7587241312509454335:2050]
2025-12-24T02:11:18.592706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312509454335:2050] [node 3] ICS09 handshake done sender: [5:7587241312509454328:2121] self: [5:7587241312509454331:0] peer: [3:7587241312611047938:0] socket: 593 qp: -1
2025-12-24T02:11:18.592711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312509454335:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.592718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.591636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241312611047944:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.592141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241312611047930:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.592164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.592172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241312611047930:2109] [node 5] ICP22 created new session: [3:7587241312611047963:2109]
2025-12-24T02:11:18.592181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241312611047963:2109] [node 5] ICS09 handshake done sender: [3:7587241312611047944:2115] self: [3:7587241312611047938:0] peer: [5:7587241312509454331:0] socket: 580 qp: -1
2025-12-24T02:11:18.592187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241312611047963:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:18.592196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.594849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241312456822797:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.594974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241312456822776:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.595002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.595010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241312456822776:2109] [node 5] ICP22 created new session: [2:7587241312456822810:2109]
2025-12-24T02:11:18.595016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241312456822810:2109] [node 5] ICS09 handshake done sender: [2:7587241312456822797:2119] self: [2:7587241312456822782:0] peer: [5:7587241312509454332:0] socket: 630 qp: -1
2025-12-24T02:11:18.595022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241312456822810:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:18.595034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.595275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241312456822812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54105
2025-12-24T02:11:18.592979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312300527410:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.592999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.593003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312300527410:2050] [node 3] ICP22 created new session: [6:7587241312300527508:2050]
2025-12-24T02:11:18.593008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241312300527508:2050] [node 3] ICS09 handshake done sender: [6:7587241312300527487:2115] self: [6:7587241312300527490:0] peer: [3:7587241312611047933:0] socket: 609 qp: -1
2025-12-24T02:11:18.593014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241312300527508:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.593027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.596872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241312509454239:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241312509454323:2120] held: no
2025-12-24T02:11:18.596920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312509454323:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.596998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312509454239:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.597013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.597019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312509454239:2049] [node 2] ICP22 created new session: [5:7587241312509454337:2049]
2025-12-24T02:11:18.597024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312509454337:2049] [node 2] ICS09 handshake done sender: [5:7587241312509454323:2120] self: [5:7587241312509454332:0] peer: [2:7587241312456822782:0] socket: 637 qp: -1
2025-12-24T02:11:18.597030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312509454337:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.597037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.594952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241314716736120:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.594981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.594994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241314716736120:2050] [node 3] ICP22 created new session: [4:7587241314716736216:2050]
2025-12-24T02:11:18.595003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314716736216:2050] [node 3] ICS09 handshake done sender: [4:7587241314716736196:2115] self: [4:7587241314716736204:0] peer: [3:7587241312611047931:0] socket: 570 qp: -1
2025-12-24T02:11:18.595013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314716736216:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.595021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.595163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241314716736184:2109] [node 6] ICP18 handshake (actor [4:7587241314716736209:2121]) is too old
2025-12-24T02:11:18.595289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241314716736119:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.595297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.595302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241314716736119:2049] [node 2] ICP22 created new session: [4:7587241314716736218:2049]
2025-12-24T02:11:18.595307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241314716736218:2049] [node 2] ICS09 handshake done sender: [4:7587241314716736197:2116] self: [4:7587241314716736203:0] peer: [2:7587241312456822785:0] socket: 611 qp: -1
2025-12-24T02:11:18.595312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241314716736218:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.595319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.604576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312509454319:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.604704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241312509454302:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.604732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.604741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312509454302:2109] [node 6] ICP22 created new session: [5:7587241312509454340:2109]
2025-12-24T02:11:18.604747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312509454340:2109] [node 6] ICS09 handshake done sender: [5:7587241312509454319:2116] self: [5:7587241312509454315:0] peer: [6:7587241312300527507:0] socket: 619 qp: -1
2025-12-24T02:11:18.604754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312509454340:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.604763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.608371Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241313208298705:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.607999Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312300527452:2090] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26915) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241312300527506:2123] held: no
2025-12-24T02:11:18.608100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312300527510:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41549
2025-12-24T02:11:18.608275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312300527506:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.608438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312300527452:2090] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.608450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.608461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312300527452:2090] [node 5] ICP22 created new session: [6:7587241312300527511:2090]
2025-12-24T02:11:18.608467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241312300527511:2090] [node 5] ICS09 handshake done sender: [6:7587241312300527506:2123] self: [6:7587241312300527507:0] peer: [5:7587241312509454315:0] socket: 641 qp: -1
2025-12-24T02:11:18.608472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241312300527511:2090] [node 5] ICS10 traffic start
2025-12-24T02:11:18.608479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.622432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:18.622521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241312456822791:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-12-24T02:11:18.622533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241312456822708:2048] [node 1] ICP30 unregister session Session# [2:7587241312456822791:2048] VirtualId# [2:7587241312456822786:0]
2025-12-24T02:11:18.622540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241312456822791:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.622575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.622600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.626793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:18.626884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241312611047953:2048] [node 1] ICS07 socket disconnect 574 reason# EndOfStream
2025-12-24T02:11:18.626890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241312611047953:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.626937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.626959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241312611047953:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.631061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:18.631129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312509454326:2048] [node 1] ICS01 socket: 604 reason# EndOfStream
2025-12-24T02:11:18.631135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312509454238:2048] [node 1] ICP30 unregister session Session# [5:7587241312509454326:2048] VirtualId# [5:7587241312509454313:0]
2025-12-24T02:11:18.631139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312509454326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.631172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.631193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.631228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:18.631296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241312300527493:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:18.631301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312300527408:2048] [node 1] ICP30 unregister session Session# [6:7587241312300527493:2048] VirtualId# [6:7587241312300527481:0]
2025-12-24T02:11:18.631308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241312300527493:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.631339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.631361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.630759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:18.630872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241314716736213:2048] [node 1] ICS01 socket: 623 reason# EndOfStream
2025-12-24T02:11:18.630881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241314716736118:2048] [node 1] ICP30 unregister session Session# [4:7587241314716736213:2048] VirtualId# [4:7587241314716736205:0]
2025-12-24T02:11:18.630889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241314716736213:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.630969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.631007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.631248Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241312611047860:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.631273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241312611047953:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.631280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241312611047860:2048] [node 1] ICP30 unregister session Session# [3:7587241312611047953:2048] VirtualId# [3:7587241312611047939:0]
2025-12-24T02:11:18.631300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.631306Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241312611047860:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.631309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61198) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.639749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:18.639846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241312611047960:2049] [node 2] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:18.639853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241312611047861:2049] [node 2] ICP30 unregister session Session# [3:7587241312611047960:2049] VirtualId# [3:7587241312611047951:0]
2025-12-24T02:11:18.639861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241312611047960:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.639893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.639915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.646761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241312611047860:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:18.654769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:18.654769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:18.654888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312509454337:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:18.654898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312509454239:2049] [node 2] ICP30 unregister session Session# [5:7587241312509454337:2049] VirtualId# [5:7587241312509454332:0]
2025-12-24T02:11:18.654905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312509454337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.654975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.655009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.654821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:18.654893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241312300527500:2049] [node 2] ICS01 socket: 595 reason# EndOfStream
2025-12-24T02:11:18.654898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312300527409:2049] [node 2] ICP30 unregister session Session# [6:7587241312300527500:2049] VirtualId# [6:7587241312300527496:0]
2025-12-24T02:11:18.654905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241312300527500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.654978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.655009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.655108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241314716736218:2049] [node 2] ICS07 socket disconnect 611 reason# EndOfStream
2025-12-24T02:11:18.655117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241314716736218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.655154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.655181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241314716736218:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:18.655560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241314716736119:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.655568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241314716736218:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.655573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241314716736119:2049] [node 2] ICP30 unregister session Session# [4:7587241314716736218:2049] VirtualId# [4:7587241314716736203:0]
2025-12-24T02:11:18.655589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.655595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241314716736119:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.655597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22995) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.667189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241314716736119:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:18.670788Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:18.670894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312509454335:2050] [node 3] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:18.670903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312509454240:2050] [node 3] ICP30 unregister session Session# [5:7587241312509454335:2050] VirtualId# [5:7587241312509454331:0]
2025-12-24T02:11:18.670910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312509454335:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.670985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.671012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.670961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:18.671061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241312300527508:2050] [node 3] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:18.671071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312300527410:2050] [node 3] ICP30 unregister session Session# [6:7587241312300527508:2050] VirtualId# [6:7587241312300527490:0]
2025-12-24T02:11:18.671077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241312300527508:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.671124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.671155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.690751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:18.690852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241312300527504:2096] [node 4] ICS01 socket: 587 reason# EndOfStream
2025-12-24T02:11:18.690860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312300527465:2096] [node 4] ICP30 unregister session Session# [6:7587241312300527504:2096] VirtualId# [6:7587241312300527502:0]
2025-12-24T02:11:18.690868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241312300527504:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.690920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:18.690947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:18.696832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:18.696925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241312300527511:2090] [node 5] ICS01 socket: 641 reason# EndOfStream
2025-12-24T02:11:18.696933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312300527452:2090] [node 5] ICP30 unregister session Session# [6:7587241312300527511:2090] VirtualId# [6:7587241312300527507:0]
2025-12-24T02:11:18.696941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241312300527511:2090] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.696975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:18.696995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:18.718200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241314673750056:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:18.718213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241314673750057:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.719590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314192448600:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.719607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314192448601:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.720850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313752397289:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.720867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313752397290:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.722327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241313200941260:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.722342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241313200941261:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.722347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241313200941262:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.723744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312944832350:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.723769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312944832351:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.723774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312944832352:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.725003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241313473961076:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.725010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241313473961077:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.725015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241313473961078:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.741622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832411:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.741674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448656:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
2025-12-24T02:11:18.740832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241313473961138:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.742404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241314673750124:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.751331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241314673750125:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:18.751450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241314673750126:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:18.758809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241314192448600:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16246392ns) {Configure:104(16245344ns) {ProcessConfigured:135(15873629ns) {StartInitialHandshake:157(15873194ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1148ns) {SwitchToState:179(400ns)}}}}}
2025-12-24T02:11:18.759009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314192448666:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:18.759078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314192448667:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:18.759147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314192448668:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:18.760045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313752397354:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:18.760144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313752397355:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.760220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313752397356:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.765406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241313200941325:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.765505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241313200941326:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.766725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397366:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.771390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312944832416:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:18.771484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241312944832417:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:18.775560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241314192448601:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16350628ns) {Configure:104(16349868ns) {ProcessConfigured:135(16075425ns) {StartInitialHandshake:157(16075022ns) {DropHandshakes:470(702ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(145ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(949ns) {SwitchToState:179(400ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.775719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241313473961149:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:18.777910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448678:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
2025-12-24T02:11:18.779094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832411:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.781951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397369:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.782067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448679:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.782324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397370:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.782599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397366:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:18.782630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241313752397289:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.782312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241314673750135:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13318
2025-12-24T02:11:18.782360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241314673750136:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13318
2025-12-24T02:11:18.782589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750135:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.782685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241314673750137:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13318
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.783331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397372:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.782777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241314673750138:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13318
2025-12-24T02:11:18.782794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241314673750139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13318
2025-12-24T02:11:18.782826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750121:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.782829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241314673750121:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.783276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241314673750121:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.783401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241314673750057:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.783433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.783439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241314673750057:2049] [node 3] ICP22 created new session: [1:7587241314673750142:2049]
2025-12-24T02:11:18.783444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241314673750142:2049] [node 3] ICS09 handshake done sender: [1:7587241314673750121:2109] self: [1:7587241314673750120:0] peer: [3:7587241313752397371:0] socket: 582 qp: -1
2025-12-24T02:11:18.783450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241314673750142:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.783463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.783882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313752397366:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.783973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241313752397289:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.783988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.784000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313752397289:2048] [node 1] ICP22 created new session: [3:7587241313752397373:2048]
2025-12-24T02:11:18.784007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313752397373:2048] [node 1] ICS09 handshake done sender: [3:7587241313752397366:2115] self: [3:7587241313752397371:0] peer: [1:7587241314673750120:0] socket: 584 qp: -1
2025-12-24T02:11:18.784019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313752397373:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.784027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.784086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750137:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.785836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.785968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.789244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448679:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.792221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941336:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.792290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941337:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.792351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241313200941260:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.792364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241313200941262:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.792393Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941340:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.789299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448656:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.792883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832430:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.792935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832431:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.801159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.799572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448678:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.799610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
2025-12-24T02:11:18.799655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448683:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
2025-12-24T02:11:18.798824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397360:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.798849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313752397360:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.799456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.799927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313752397360:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.800062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397369:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.800091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397362:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.800094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313752397362:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.800590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313752397354:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.800604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.800612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313752397354:2107] [node 4] ICP22 created new session: [3:7587241313752397378:2107]
2025-12-24T02:11:18.800620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313752397378:2107] [node 4] ICS09 handshake done sender: [3:7587241313752397360:2112] self: [3:7587241313752397359:0] peer: [4:7587241313200941342:0] socket: 589 qp: -1
2025-12-24T02:11:18.800627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313752397378:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:18.800636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.801360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313752397362:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.799701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832431:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.799741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312944832352:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.801197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832433:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.801590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312944832431:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.801649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312944832352:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.801666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.801673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312944832352:2050] [node 3] ICP22 created new session: [5:7587241312944832434:2050]
2025-12-24T02:11:18.801679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312944832434:2050] [node 3] ICS09 handshake done sender: [5:7587241312944832431:2117] self: [5:7587241312944832432:0] peer: [3:7587241313752397361:0] socket: 620 qp: -1
2025-12-24T02:11:18.801687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312944832434:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.801694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.802500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832430:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.802557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312944832350:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.801259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750128:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.801271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241314673750128:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.801585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750139:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.802998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241314673750128:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.803387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750130:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.803391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241314673750130:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.803865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961154:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.803694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241314673750124:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.803713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.803719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241314673750124:2108] [node 4] ICP22 created new session: [1:7587241314673750148:2108]
2025-12-24T02:11:18.803724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241314673750148:2108] [node 4] ICS09 handshake done sender: [1:7587241314673750128:2112] self: [1:7587241314673750127:0] peer: [4:7587241313200941341:0] socket: 583 qp: -1
2025-12-24T02:11:18.803730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241314673750148:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:18.803740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.805819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.805848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241313200941337:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.805963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241313200941262:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.805981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.805987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241313200941262:2050] [node 3] ICP22 created new session: [4:7587241313200941346:2050]
2025-12-24T02:11:18.805995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241313200941346:2050] [node 3] ICS09 handshake done sender: [4:7587241313200941337:2116] self: [4:7587241313200941342:0] peer: [3:7587241313752397359:0] socket: 592 qp: -1
2025-12-24T02:11:18.806003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241313200941346:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.806010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.806256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.807126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241314192448600:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.807182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448682:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.807531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313752397355:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.807558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.807567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313752397355:2108] [node 5] ICP22 created new session: [3:7587241313752397383:2108]
2025-12-24T02:11:18.807574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313752397383:2108] [node 5] ICS09 handshake done sender: [3:7587241313752397362:2113] self: [3:7587241313752397361:0] peer: [5:7587241312944832432:0] socket: 585 qp: -1
2025-12-24T02:11:18.807581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313752397383:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.807590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.808392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750136:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.808426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750138:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.808457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750117:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.808460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241314673750117:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.809660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314192448688:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53272
2025-12-24T02:11:18.809248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241314673750124:2108] [node 4] ICP18 handshake (actor [1:7587241314673750136:2116]) is too old
2025-12-24T02:11:18.809788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832438:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.811160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241313200941336:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.811353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961155:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.811278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448683:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.811752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314192448656:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.811823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241314192448600:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.811843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.811850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314192448600:2048] [node 1] ICP22 created new session: [2:7587241314192448690:2048]
2025-12-24T02:11:18.811855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314192448690:2048] [node 1] ICS09 handshake done sender: [2:7587241314192448656:2103] self: [2:7587241314192448685:0] peer: [1:7587241314673750116:0] socket: 578 qp: -1
2025-12-24T02:11:18.811861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314192448690:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.811867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.812440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941345:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.812459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241313200941260:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.812472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.812478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241313200941260:2048] [node 1] ICP22 created new session: [4:7587241313200941348:2048]
2025-12-24T02:11:18.812484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241313200941348:2048] [node 1] ICS09 handshake done sender: [4:7587241313200941336:2115] self: [4:7587241313200941341:0] peer: [1:7587241314673750127:0] socket: 586 qp: -1
2025-12-24T02:11:18.812490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241313200941348:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.812498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.812569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241313200941260:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.812865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941340:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.812945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241313200941261:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.813122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448672:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.813132Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314192448672:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.811381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397370:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.811553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241314673750117:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.811710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241314673750130:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.811799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241314673750056:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.811819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.811825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241314673750056:2048] [node 2] ICP22 created new session: [1:7587241314673750154:2048]
2025-12-24T02:11:18.811831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241314673750154:2048] [node 2] ICS09 handshake done sender: [1:7587241314673750117:2107] self: [1:7587241314673750116:0] peer: [2:7587241314192448685:0] socket: 576 qp: -1
2025-12-24T02:11:18.811843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241314673750154:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:18.811853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.811998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241314673750125:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.812007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.812011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241314673750125:2110] [node 5] ICP22 created new session: [1:7587241314673750156:2110]
2025-12-24T02:11:18.812015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241314673750156:2110] [node 5] ICS09 handshake done sender: [1:7587241314673750130:2113] self: [1:7587241314673750129:0] peer: [5:7587241312944832437:0] socket: 591 qp: -1
2025-12-24T02:11:18.812019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241314673750156:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:18.812025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.813064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312944832430:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.815195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314192448672:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.815473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314192448666:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.815492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.815498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314192448666:2105] [node 4] ICP22 created new session: [2:7587241314192448695:2105]
2025-12-24T02:11:18.815505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314192448695:2105] [node 4] ICS09 handshake done sender: [2:7587241314192448672:2113] self: [2:7587241314192448671:0] peer: [4:7587241313200941353:0] socket: 606 qp: -1
2025-12-24T02:11:18.815511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314192448695:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:18.815518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.815858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961156:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.815939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961157:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.817734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961158:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.817839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241313473961157:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.817873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241313473961077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.817957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448676:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.817969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314192448676:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.818463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314192448676:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.816195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312944832350:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.816221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.816230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312944832350:2048] [node 1] ICP22 created new session: [5:7587241312944832439:2048]
2025-12-24T02:11:18.816238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312944832439:2048] [node 1] ICS09 handshake done sender: [5:7587241312944832430:2116] self: [5:7587241312944832437:0] peer: [1:7587241314673750129:0] socket: 619 qp: -1
2025-12-24T02:11:18.816246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312944832439:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.816256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.817456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832442:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.817515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832443:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.820143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241313752397354:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.820246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241313752397290:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.820503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241313473961077:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6352) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241313473961157:2118] held: no
2025-12-24T02:11:18.820559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241313473961154:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.820653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.820855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832443:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.822192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397386:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.818658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314192448668:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.818675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.818681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314192448668:2108] [node 6] ICP22 created new session: [2:7587241314192448698:2108]
2025-12-24T02:11:18.818687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314192448698:2108] [node 6] ICS09 handshake done sender: [2:7587241314192448676:2115] self: [2:7587241314192448675:0] peer: [6:7587241313473961159:0] socket: 635 qp: -1
2025-12-24T02:11:18.818708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314192448698:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:18.818718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.820419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448670:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.820424Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314192448670:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.822284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314192448674:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.822289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314192448674:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.822775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241313473961078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.822968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241313473961157:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.823123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397364:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.823133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313752397364:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.823597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313752397387:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44314
2025-12-24T02:11:18.823917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313752397364:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.825296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313752397370:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.821981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312944832351:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.825427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241313473961077:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.825448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.825456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241313473961077:2049] [node 2] ICP22 created new session: [6:7587241313473961164:2049]
2025-12-24T02:11:18.825463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241313473961164:2049] [node 2] ICS09 handshake done sender: [6:7587241313473961157:2118] self: [6:7587241313473961159:0] peer: [2:7587241314192448675:0] socket: 636 qp: -1
2025-12-24T02:11:18.825471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241313473961164:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.825480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.825634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241313473961158:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.827222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241313473961138:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.827255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241313473961155:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.827302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961163:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.827360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241313752397290:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.827379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.827385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313752397290:2049] [node 2] ICP22 created new session: [3:7587241313752397389:2049]
2025-12-24T02:11:18.827393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313752397389:2049] [node 2] ICS09 handshake done sender: [3:7587241313752397370:2117] self: [3:7587241313752397385:0] peer: [2:7587241314192448669:0] socket: 607 qp: -1
2025-12-24T02:11:18.827403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313752397389:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.827425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.828243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241313200941261:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6352) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241313200941340:2117] held: no
2025-12-24T02:11:18.828440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941354:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.829067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241313200941340:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.829384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241313200941261:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.829402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.829408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241313200941261:2049] [node 2] ICP22 created new session: [4:7587241313200941355:2049]
2025-12-24T02:11:18.829414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241313200941355:2049] [node 2] ICS09 handshake done sender: [4:7587241313200941340:2117] self: [4:7587241313200941353:0] peer: [2:7587241314192448671:0] socket: 616 qp: -1
2025-12-24T02:11:18.829422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241313200941355:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.829431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.829858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313752397356:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.829872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.829877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313752397356:2109] [node 6] ICP22 created new session: [3:7587241313752397391:2109]
2025-12-24T02:11:18.829883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313752397391:2109] [node 6] ICS09 handshake done sender: [3:7587241313752397364:2114] self: [3:7587241313752397363:0] peer: [6:7587241313473961162:0] socket: 624 qp: -1
2025-12-24T02:11:18.829890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313752397391:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.829896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.825063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314192448670:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.828630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314192448601:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.828647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.828654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314192448601:2049] [node 3] ICP22 created new session: [2:7587241314192448702:2049]
2025-12-24T02:11:18.828661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314192448702:2049] [node 3] ICS09 handshake done sender: [2:7587241314192448670:2112] self: [2:7587241314192448669:0] peer: [3:7587241313752397385:0] socket: 605 qp: -1
2025-12-24T02:11:18.828667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314192448702:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.828674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.829308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314192448674:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.830157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314192448667:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.830170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.830175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314192448667:2107] [node 5] ICP22 created new session: [2:7587241314192448704:2107]
2025-12-24T02:11:18.830180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314192448704:2107] [node 5] ICS09 handshake done sender: [2:7587241314192448674:2114] self: [2:7587241314192448673:0] peer: [5:7587241312944832444:0] socket: 621 qp: -1
2025-12-24T02:11:18.830185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314192448704:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:18.830192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.830515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832442:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.830565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832445:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.830739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241312944832416:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.830751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312944832443:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.835744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941332:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.835760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241313200941332:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.836557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241313473961156:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.836690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241313473961154:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.836829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313752397387:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.836869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241313752397356:2109] [node 6] ICP18 handshake (actor [3:7587241313752397387:2121]) is too old
2025-12-24T02:11:18.837203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241313200941332:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.837306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241313200941325:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.837331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.837341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241313200941325:2108] [node 5] ICP22 created new session: [4:7587241313200941358:2108]
2025-12-24T02:11:18.837347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241313200941358:2108] [node 5] ICS09 handshake done sender: [4:7587241313200941332:2113] self: [4:7587241313200941331:0] peer: [5:7587241312944832446:0] socket: 593 qp: -1
2025-12-24T02:11:18.837355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241313200941358:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.837364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.839134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241313473961078:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.839154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.839161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241313473961078:2050] [node 3] ICP22 created new session: [6:7587241313473961167:2050]
2025-12-24T02:11:18.839167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241313473961167:2050] [node 3] ICS09 handshake done sender: [6:7587241313473961154:2115] self: [6:7587241313473961162:0] peer: [3:7587241313752397363:0] socket: 628 qp: -1
2025-12-24T02:11:18.839173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241313473961167:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.839180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.839245Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241313473961078:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.839556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241313473961076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.839573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241313473961149:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.839824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241314673750132:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.839838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241314673750132:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.840716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241314673750132:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.840809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241314673750126:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.840822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.840829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241314673750126:2111] [node 6] ICP22 created new session: [1:7587241314673750159:2111]
2025-12-24T02:11:18.840835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241314673750159:2111] [node 6] ICS09 handshake done sender: [1:7587241314673750132:2114] self: [1:7587241314673750131:0] peer: [6:7587241313473961170:0] socket: 632 qp: -1
2025-12-24T02:11:18.840840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241314673750159:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:18.840848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.841193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941334:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.841204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241313200941334:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.841727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241313200941360:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53029
2025-12-24T02:11:18.842101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241313200941334:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.842440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241313200941326:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.842455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.842463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241313200941326:2109] [node 6] ICP22 created new session: [4:7587241313200941362:2109]
2025-12-24T02:11:18.842469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241313200941362:2109] [node 6] ICS09 handshake done sender: [4:7587241313200941334:2114] self: [4:7587241313200941333:0] peer: [6:7587241313473961171:0] socket: 587 qp: -1
2025-12-24T02:11:18.838375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312944832351:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.838398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.838406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312944832351:2049] [node 2] ICP22 created new session: [5:7587241312944832447:2049]
2025-12-24T02:11:18.838412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312944832447:2049] [node 2] ICS09 handshake done sender: [5:7587241312944832443:2121] self: [5:7587241312944832444:0] peer: [2:7587241314192448673:0] socket: 637 qp: -1
2025-12-24T02:11:18.838419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312944832447:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:18.838429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.838519Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241312944832351:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:18.838750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241312944832449:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42822
2025-12-24T02:11:18.842603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241312944832425:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.842609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241312944832425:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.842477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241313200941362:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.842485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.843862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241313473961076:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1588) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241313473961155:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.845232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961172:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.846033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241313473961155:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.846091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241313473961076:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.846103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.846109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241313473961076:2048] [node 1] ICP22 created new session: [6:7587241313473961174:2048]
2025-12-24T02:11:18.846114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241313473961174:2048] [node 1] ICS09 handshake done sender: [6:7587241313473961155:2116] self: [6:7587241313473961170:0] peer: [1:7587241314673750131:0] socket: 615 qp: -1
2025-12-24T02:11:18.846119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241313473961174:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.846125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.846282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961173:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.846610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241313473961156:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.843166Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241312944832416:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9679) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241312944832442:2120] held: no
2025-12-24T02:11:18.843221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312944832442:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.844822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241312944832416:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.844833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.844839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312944832416:2109] [node 4] ICP22 created new session: [5:7587241312944832451:2109]
2025-12-24T02:11:18.844845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312944832451:2109] [node 4] ICS09 handshake done sender: [5:7587241312944832442:2120] self: [5:7587241312944832446:0] peer: [4:7587241313200941331:0] socket: 618 qp: -1
2025-12-24T02:11:18.844852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312944832451:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:18.844858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.845512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241312944832425:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.845603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241312944832417:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.845615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.845621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241312944832417:2110] [node 6] ICP22 created new session: [5:7587241312944832453:2110]
2025-12-24T02:11:18.845626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241312944832453:2110] [node 6] ICS09 handshake done sender: [5:7587241312944832425:2115] self: [5:7587241312944832424:0] peer: [6:7587241313473961166:0] socket: 600 qp: -1
2025-12-24T02:11:18.845631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241312944832453:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:18.845638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.846668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241313473961149:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.846680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.846685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241313473961149:2110] [node 4] ICP22 created new session: [6:7587241313473961176:2110]
2025-12-24T02:11:18.846690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241313473961176:2110] [node 4] ICS09 handshake done sender: [6:7587241313473961156:2117] self: [6:7587241313473961171:0] peer: [4:7587241313200941333:0] socket: 598 qp: -1
2025-12-24T02:11:18.846709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241313473961176:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:18.846714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.848732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241313473961138:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241313473961158:2119] held: no
2025-12-24T02:11:18.848795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241313473961178:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64020
2025-12-24T02:11:18.848942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241313473961158:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.848995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241313473961138:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.849006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.849011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241313473961138:2108] [node 5] ICP22 created new session: [6:7587241313473961179:2108]
2025-12-24T02:11:18.849016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241313473961179:2108] [node 5] ICS09 handshake done sender: [6:7587241313473961158:2119] self: [6:7587241313473961166:0] peer: [5:7587241312944832424:0] socket: 638 qp: -1
2025-12-24T02:11:18.849021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241313473961179:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.849028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.847651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241313200941360:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.847695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241313200941326:2109] [node 6] ICP18 handshake (actor [4:7587241313200941360:2122]) is too old
2025-12-24T02:11:18.853646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:18.853735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241314192448690:2048] [node 1] ICS01 socket: 578 reason# EndOfStream
2025-12-24T02:11:18.853744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241314192448600:2048] [node 1] ICP30 unregister session Session# [2:7587241314192448690:2048] VirtualId# [2:7587241314192448685:0]
2025-12-24T02:11:18.853750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241314192448690:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.853792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.853813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.854217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:18.854284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241313752397373:2048] [node 1] ICS07 socket disconnect 584 reason# EndOfStream
2025-12-24T02:11:18.854289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241313752397373:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.854319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.854335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241313752397373:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.856466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241313752397289:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1588) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:18.856480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241313752397373:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:18.856795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241313752397289:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1588) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:18.856806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241313752397373:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:18.856812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241313752397289:2048] [node 1] ICP30 unregister session Session# [3:7587241313752397373:2048] VirtualId# [3:7587241313752397371:0]
2025-12-24T02:11:18.856830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.856836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241313752397289:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1588) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.856840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1588) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:18.862043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:18.862294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:18.862356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312944832439:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-12-24T02:11:18.862361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312944832350:2048] [node 1] ICP30 unregister session Session# [5:7587241312944832439:2048] VirtualId# [5:7587241312944832437:0]
2025-12-24T02:11:18.862364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312944832439:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.858823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:18.858904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241313200941348:2048] [node 1] ICS01 socket: 586 reason# EndOfStream
2025-12-24T02:11:18.858910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241313200941260:2048] [node 1] ICP30 unregister session Session# [4:7587241313200941348:2048] VirtualId# [4:7587241313200941341:0]
2025-12-24T02:11:18.858916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241313200941348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.858959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.858988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.862395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.862419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.864465Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:18.864542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241313752397389:2049] [node 2] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:18.864550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241313752397290:2049] [node 2] ICP30 unregister session Session# [3:7587241313752397389:2049] VirtualId# [3:7587241313752397385:0]
2025-12-24T02:11:18.864555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241313752397389:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.864593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.864613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.862133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241313473961174:2048] [node 1] ICS01 socket: 615 reason# EndOfStream
2025-12-24T02:11:18.862144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241313473961076:2048] [node 1] ICP30 unregister session Session# [6:7587241313473961174:2048] VirtualId# [6:7587241313473961170:0]
2025-12-24T02:11:18.862150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241313473961174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.862233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:18.862258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:18.866979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241313752397289:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:18.869338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:18.869428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241313473961164:2049] [node 2] ICS01 socket: 636 reason# EndOfStream
2025-12-24T02:11:18.869436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241313473961077:2049] [node 2] ICP30 unregister session Session# [6:7587241313473961164:2049] VirtualId# [6:7587241313473961159:0]
2025-12-24T02:11:18.869442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241313473961164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.869489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.869517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.869556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:18.869622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312944832447:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:18.869626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312944832351:2049] [node 2] ICP30 unregister session Session# [5:7587241312944832447:2049] VirtualId# [5:7587241312944832444:0]
2025-12-24T02:11:18.869631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312944832447:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.869663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.869683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.869713Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:18.869774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241313200941355:2049] [node 2] ICS01 socket: 616 reason# EndOfStream
2025-12-24T02:11:18.869779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241313200941261:2049] [node 2] ICP30 unregister session Session# [4:7587241313200941355:2049] VirtualId# [4:7587241313200941353:0]
2025-12-24T02:11:18.869783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241313200941355:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.869807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:18.869827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:18.876643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:18.876739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241313473961167:2050] [node 3] ICS01 socket: 628 reason# EndOfStream
2025-12-24T02:11:18.876748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241313473961078:2050] [node 3] ICP30 unregister session Session# [6:7587241313473961167:2050] VirtualId# [6:7587241313473961162:0]
2025-12-24T02:11:18.876755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241313473961167:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.876814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.876848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.876890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:18.876952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241312944832434:2050] [node 3] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:18.876957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241312944832352:2050] [node 3] ICP30 unregister session Session# [5:7587241312944832434:2050] VirtualId# [5:7587241312944832432:0]
2025-12-24T02:11:18.876962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241312944832434:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.876990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:18.877012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:18.884487Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:18.884589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241313473961176:2110] [node 4] ICS01 socket: 598 reason# EndOfStream
2025-12-24T02:11:18.884599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241313473961149:2110] [node 4] ICP30 unregister session Session# [6:7587241313473961176:2110] VirtualId# [6:7587241313473961171:0]
2025-12-24T02:11:18.884606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241313473961176:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:18.884669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:18.884706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:18.912748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315013035270:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:18.912766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315013035271:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.916923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313075105410:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.916930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313075105411:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.920290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241313584720338:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.920299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241313584720339:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.920306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241313584720340:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.922328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312538692836:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.922337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312538692837:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.922341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312538692838:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.914805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314743423672:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.914816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314743423673:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:18.918705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311544214162:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:18.918714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311544214163:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:18.918720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311544214164:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:18.935849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312538692897:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.936692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720396:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.938821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315013035332:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:18.938948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315013035333:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:18.946520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241312538692903:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:18.946592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313075105477:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:18.946646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313075105478:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.946686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241313075105479:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.946474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315013035340:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:18.947691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314743423735:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:18.947856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314743423736:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:18.947953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241314743423737:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.951063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241313584720401:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:18.951235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241313584720404:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:18.951201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311544214230:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.955051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241315013035271:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16143833ns) {Configure:104(16141733ns) {ProcessConfigured:135(15784601ns) {StartInitialHandshake:157(15784139ns) {DropHandshakes:470(782ns) {DropIncomingHandshake:442(147ns) DropOutgoingHandshake:457(98ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(1259ns) {SwitchToState:179(803ns)}}}}}
2025-12-24T02:11:18.958248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315013035346:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11622
2025-12-24T02:11:18.959055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241311544214228:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.962245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105489:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.962574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.963227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423739:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
2025-12-24T02:11:18.963318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423740:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.963366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423741:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
2025-12-24T02:11:18.962117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241313584720339:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20101626ns) {Configure:104(20099945ns) {ProcessConfigured:135(19721820ns) {StartInitialHandshake:157(19720928ns) {DropHandshakes:470(619ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(499ns) PrepareNewSessionHandshake:477(1357ns) {SwitchToState:179(988ns)}}}}}
2025-12-24T02:11:18.962609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241313584720396:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.963090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720411:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.963409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423750:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.963497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315013035350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.963698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105491:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.963303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692914:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.963442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692915:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.964746Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241314743423672:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16709222ns) {Configure:104(16708014ns) {ProcessConfigured:135(16403499ns) {StartInitialHandshake:157(16403232ns) {DropHandshakes:470(340ns) {DropIncomingHandshake:442(35ns) DropOutgoingHandshake:457(35ns)} GenerateSessionVirtualId:988(94ns) PrepareNewSessionHandshake:477(579ns) {SwitchToState:179(478ns)}}}}}
2025-12-24T02:11:18.964983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423741:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.965041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423755:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
2025-12-24T02:11:18.964950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241313584720411:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.965061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.965098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423739:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.965143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241314743423672:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.965078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241313584720338:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.965326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035342:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.965344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315013035342:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.965750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035328:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.965761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315013035328:2104] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.966453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315013035342:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.966483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241314743423758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28945
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.966612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315013035328:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:18.962986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311544214232:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:18.963009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311544214233:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:18.966539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214233:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.966596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311544214242:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:18.966609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311544214243:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:18.966640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311544214162:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.967005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314743423739:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.967761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692916:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.968803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241314743423672:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.968825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.968834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314743423672:2048] [node 1] ICP22 created new session: [2:7587241314743423759:2048]
2025-12-24T02:11:18.968841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314743423759:2048] [node 1] ICS09 handshake done sender: [2:7587241314743423739:2110] self: [2:7587241314743423757:0] peer: [1:7587241315013035327:0] socket: 587 qp: -1
2025-12-24T02:11:18.968847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314743423759:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.968864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.966741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315013035333:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.966760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.966767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315013035333:2106] [node 5] ICP22 created new session: [1:7587241315013035353:2106]
2025-12-24T02:11:18.966774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315013035353:2106] [node 5] ICS09 handshake done sender: [1:7587241315013035342:2111] self: [1:7587241315013035341:0] peer: [5:7587241313584720420:0] socket: 597 qp: -1
2025-12-24T02:11:18.966781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315013035353:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:18.966790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.966939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315013035270:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.966949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:18.966960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315013035270:2048] [node 2] ICP22 created new session: [1:7587241315013035355:2048]
2025-12-24T02:11:18.966966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315013035355:2048] [node 2] ICS09 handshake done sender: [1:7587241315013035328:2104] self: [1:7587241315013035327:0] peer: [2:7587241314743423757:0] socket: 582 qp: -1
2025-12-24T02:11:18.966995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315013035355:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:18.967025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:18.967130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035339:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.967133Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315013035339:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.969213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315013035339:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:18.969379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315013035332:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.969395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:18.969401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315013035332:2105] [node 4] ICP22 created new session: [1:7587241315013035360:2105]
2025-12-24T02:11:18.969407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315013035360:2105] [node 4] ICS09 handshake done sender: [1:7587241315013035339:2110] self: [1:7587241315013035336:0] peer: [4:7587241311544214244:0] socket: 594 qp: -1
2025-12-24T02:11:18.969413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315013035360:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:18.969420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:18.969500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312538692916:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.969547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312538692836:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.969635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035349:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.969644Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315013035349:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.969797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692918:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.970280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035346:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.970357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105492:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.970506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105492:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.970542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241313075105410:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.970913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105491:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.970945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241313075105411:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.972046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312538692918:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.972090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312538692837:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.967043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720421:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.969341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241313584720411:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.969699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241313584720338:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.969709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.969715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241313584720338:2048] [node 1] ICP22 created new session: [5:7587241313584720422:2048]
2025-12-24T02:11:18.969720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241313584720422:2048] [node 1] ICS09 handshake done sender: [5:7587241313584720411:2114] self: [5:7587241313584720420:0] peer: [1:7587241315013035341:0] socket: 602 qp: -1
2025-12-24T02:11:18.969726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241313584720422:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.969744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.975249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423749:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.975271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314743423749:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.976638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423743:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.976653Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314743423743:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.976682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312538692915:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.976724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035345:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.976739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315013035345:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.976736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312538692897:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.977346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315013035345:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.977990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241313584720419:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241313584720339:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.978084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105496:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.977976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423755:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423740:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315013035271:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.978075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.978080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315013035271:2049] [node 3] ICP22 created new session: [1:7587241315013035365:2049]
2025-12-24T02:11:18.978087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315013035365:2049] [node 3] ICS09 handshake done sender: [1:7587241315013035345:2113] self: [1:7587241315013035335:0] peer: [3:7587241313075105493:0] socket: 628 qp: -1
2025-12-24T02:11:18.978093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315013035365:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:18.978101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.978419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241312538692914:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312538692903:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.978484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423747:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241313584720410:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315013035349:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.978499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314743423747:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.978190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241313584720410:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.978608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315013035350:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315013035340:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.978643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.978651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315013035340:2107] [node 6] ICP22 created new session: [1:7587241315013035367:2107]
2025-12-24T02:11:18.978657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315013035367:2107] [node 6] ICS09 handshake done sender: [1:7587241315013035349:2115] self: [1:7587241315013035343:0] peer: [6:7587241312538692917:0] socket: 629 qp: -1
2025-12-24T02:11:18.978663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315013035367:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:18.978676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.977266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241311544214245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:18.978285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214243:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311544214163:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.978358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214242:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214232:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.978417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241311544214164:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.979325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105495:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.979397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313075105492:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:18.979402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423750:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.979085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241313584720410:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.979180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241313584720404:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.979197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.979207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241313584720404:2109] [node 6] ICP22 created new session: [5:7587241313584720427:2109]
2025-12-24T02:11:18.979213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241313584720427:2109] [node 6] ICS09 handshake done sender: [5:7587241313584720410:2113] self: [5:7587241313584720409:0] peer: [6:7587241312538692920:0] socket: 611 qp: -1
2025-12-24T02:11:18.979219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241313584720427:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.979234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.982145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214235:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.982164Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241311544214235:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.982741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314743423749:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:18.983051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314743423737:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.983072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:18.983080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314743423737:2109] [node 6] ICP22 created new session: [2:7587241314743423766:2109]
2025-12-24T02:11:18.983088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314743423766:2109] [node 6] ICS09 handshake done sender: [2:7587241314743423749:2116] self: [2:7587241314743423748:0] peer: [6:7587241312538692919:0] socket: 632 qp: -1
2025-12-24T02:11:18.983099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314743423766:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:18.983106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:18.982997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241315013035332:2105] [node 4] ICP18 handshake (actor [1:7587241315013035350:2116]) is too old
2025-12-24T02:11:18.983249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105481:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.983268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313075105481:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.983326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241314743423745:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.983337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241314743423745:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:18.983925Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312538692897:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7061) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241312538692915:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.984044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.984047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692923:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.984861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241313075105410:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.984883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:18.984889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313075105410:2048] [node 1] ICP22 created new session: [3:7587241313075105497:2048]
2025-12-24T02:11:18.984896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313075105497:2048] [node 1] ICS09 handshake done sender: [3:7587241313075105492:2117] self: [3:7587241313075105493:0] peer: [1:7587241315013035335:0] socket: 634 qp: -1
2025-12-24T02:11:18.984902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313075105497:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:18.984911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:18.985004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241313075105410:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.985178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105495:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.985308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241313075105499:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:18.984120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720429:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.985568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720430:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.986884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105483:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.986896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313075105483:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:18.987402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313075105483:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:18.987630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313075105478:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:18.987650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:18.987657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313075105478:2108] [node 5] ICP22 created new session: [3:7587241313075105503:2108]
2025-12-24T02:11:18.987662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241313075105503:2108] [node 5] ICS09 handshake done sender: [3:7587241313075105483:2113] self: [3:7587241313075105482:0] peer: [5:7587241313584720431:0] socket: 610 qp: -1
2025-12-24T02:11:18.987668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241313075105503:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:18.987675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:18.986685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241313584720430:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:18.986747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241313584720340:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:18.987394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241313584720432:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:18.987571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241313584720430:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:18.987645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241313584720340:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:18.987659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:18.987665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241313584720340:2050] [node 3] ICP22 created new session: [5:7587241313584720433:2050]
2025-12-24T02:11:18.987671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241313584720433:2050] [node 3] ICS09 handshake done sender: [5:7587241313584720430:2120] self: [5:7587241313584720431:0] peer: [3:7587241313075105482:0] socket: 647 qp: -1
2025-12-24T02:11:18.987677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241313584720433:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:18.987683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:18.991041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241312538692924:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:18.999965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312538692918:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.000052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314743423743:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.000108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312538692916:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.000150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313075105481:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.001614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312538692837:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.001634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.001641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312538692837:2049] [node 2] ICP22 created new session: [6:7587241316833660223:2049]
2025-12-24T02:11:19.001647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316833660223:2049] [node 2] ICS09 handshake done sender: [6:7587241312538692918:2118] self: [6:7587241312538692919:0] peer: [2:7587241314743423748:0] socket: 633 qp: -1
2025-12-24T02:11:19.001652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316833660223:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.001659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.001759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241312538692837:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.001858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312538692836:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.001866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.001871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312538692836:2048] [node 1] ICP22 created new session: [6:7587241316833660225:2048]
2025-12-24T02:11:19.001876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316833660225:2048] [node 1] ICS09 handshake done sender: [6:7587241312538692916:2117] self: [6:7587241312538692917:0] peer: [1:7587241315013035343:0] socket: 630 qp: -1
2025-12-24T02:11:19.001882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316833660225:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.001887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.002385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312538692915:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.002551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241313584720419:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.003059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241313584720339:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.003076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.003084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241313584720339:2049] [node 2] ICP22 created new session: [5:7587241317879687731:2049]
2025-12-24T02:11:19.003090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241317879687731:2049] [node 2] ICS09 handshake done sender: [5:7587241313584720419:2117] self: [5:7587241313584720425:0] peer: [2:7587241314743423746:0] socket: 620 qp: -1
2025-12-24T02:11:19.003097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241317879687731:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.003103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.000298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314743423745:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.000392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314743423673:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.000413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.000422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314743423673:2049] [node 3] ICP22 created new session: [2:7587241314743423771:2049]
2025-12-24T02:11:19.000810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313075105477:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.000825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.000832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313075105477:2107] [node 4] ICP22 created new session: [3:7587241317370072803:2107]
2025-12-24T02:11:19.000838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317370072803:2107] [node 4] ICS09 handshake done sender: [3:7587241313075105481:2112] self: [3:7587241313075105480:0] peer: [4:7587241311544214248:0] socket: 590 qp: -1
2025-12-24T02:11:19.000843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317370072803:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.000851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.002102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313075105491:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.002219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241313075105411:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.002231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.002239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313075105411:2049] [node 2] ICP22 created new session: [3:7587241317370072805:2049]
2025-12-24T02:11:19.002246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317370072805:2049] [node 2] ICS09 handshake done sender: [3:7587241313075105491:2116] self: [3:7587241313075105494:0] peer: [2:7587241314743423742:0] socket: 617 qp: -1
2025-12-24T02:11:19.002253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317370072805:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.002260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.000429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314743423771:2049] [node 3] ICS09 handshake done sender: [2:7587241314743423743:2113] self: [2:7587241314743423742:0] peer: [3:7587241313075105494:0] socket: 614 qp: -1
2025-12-24T02:11:19.000436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314743423771:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.000442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.000569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314743423735:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.000582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.000588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314743423735:2107] [node 4] ICP22 created new session: [2:7587241314743423773:2107]
2025-12-24T02:11:19.000592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241314743423773:2107] [node 4] ICS09 handshake done sender: [2:7587241314743423745:2114] self: [2:7587241314743423744:0] peer: [4:7587241311544214246:0] socket: 616 qp: -1
2025-12-24T02:11:19.000596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241314743423773:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.000606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.002349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241314743423747:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.004666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312538692897:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.004680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.004688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312538692897:2107] [node 5] ICP22 created new session: [6:7587241316833660227:2107]
2025-12-24T02:11:19.004694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316833660227:2107] [node 5] ICS09 handshake done sender: [6:7587241312538692915:2116] self: [6:7587241312538692920:0] peer: [5:7587241313584720409:0] socket: 613 qp: -1
2025-12-24T02:11:19.004700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316833660227:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:19.004706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.005512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241314743423736:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.005534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.005540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241314743423736:2108] [node 5] ICP22 created new session: [2:7587241319038391071:2108]
2025-12-24T02:11:19.005545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241319038391071:2108] [node 5] ICS09 handshake done sender: [2:7587241314743423747:2115] self: [2:7587241314743423746:0] peer: [5:7587241313584720425:0] socket: 618 qp: -1
2025-12-24T02:11:19.005550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241319038391071:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.005558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.005658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316833660222:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:19.006114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316833660222:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.006148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241312538692838:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.003675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311544214233:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.003780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241311544214162:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241311544214233:2113] held: no
2025-12-24T02:11:19.003795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311544214162:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.003802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.003808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311544214162:2048] [node 1] ICP22 created new session: [4:7587241315839181549:2048]
2025-12-24T02:11:19.003813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241315839181549:2048] [node 1] ICS09 handshake done sender: [4:7587241311544214233:2113] self: [4:7587241311544214244:0] peer: [1:7587241315013035336:0] socket: 601 qp: -1
2025-12-24T02:11:19.003819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241315839181549:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.003871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.006892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241313075105488:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.006904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241313075105488:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.007448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241313075105488:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.007536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241313075105479:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.007549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.007555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241313075105479:2109] [node 6] ICP22 created new session: [3:7587241317370072808:2109]
2025-12-24T02:11:19.007561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317370072808:2109] [node 6] ICS09 handshake done sender: [3:7587241313075105488:2114] self: [3:7587241313075105485:0] peer: [6:7587241316833660229:0] socket: 636 qp: -1
2025-12-24T02:11:19.007565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317370072808:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.007572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.008306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316833660230:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:19.008015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241311544214163:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4465) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241311544214243:2118] held: no
2025-12-24T02:11:19.008060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241315839181545:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:19.008091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241315839181546:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
2025-12-24T02:11:19.008315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241315839181545:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.008359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311544214232:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.008507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241315839181547:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.013427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316833660222:2122] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.013450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241317879687733:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.015381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317370072810:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41515
2025-12-24T02:11:19.016894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241317879687733:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.013730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312538692838:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11170) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241316833660222:2122] held: no
2025-12-24T02:11:19.016947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241313584720401:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.013755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312538692838:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.013762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.013766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312538692838:2050] [node 3] ICP22 created new session: [6:7587241316833660232:2050]
2025-12-24T02:11:19.013773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316833660232:2050] [node 3] ICS09 handshake done sender: [6:7587241316833660222:2122] self: [6:7587241316833660229:0] peer: [3:7587241313075105485:0] socket: 595 qp: -1
2025-12-24T02:11:19.013778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316833660232:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.013784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.013871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316833660231:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40996
2025-12-24T02:11:19.014040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241312538692914:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.014130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241312538692903:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.014141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.014146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241312538692903:2109] [node 4] ICP22 created new session: [6:7587241316833660234:2109]
2025-12-24T02:11:19.014152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316833660234:2109] [node 4] ICS09 handshake done sender: [6:7587241312538692914:2115] self: [6:7587241312538692921:0] peer: [4:7587241311544214234:0] socket: 609 qp: -1
2025-12-24T02:11:19.014160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316833660234:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:19.014165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.017781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241317879687735:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38171
2025-12-24T02:11:19.017928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241317879687733:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.017998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241313584720401:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.018019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.018024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241313584720401:2108] [node 4] ICP22 created new session: [5:7587241317879687736:2108]
2025-12-24T02:11:19.018030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241317879687736:2108] [node 4] ICS09 handshake done sender: [5:7587241317879687733:2122] self: [5:7587241317879687734:0] peer: [4:7587241311544214236:0] socket: 651 qp: -1
2025-12-24T02:11:19.018037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241317879687736:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:19.018043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.013369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311544214164:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.013400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.013410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311544214164:2050] [node 3] ICP22 created new session: [4:7587241315839181553:2050]
2025-12-24T02:11:19.013418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241315839181553:2050] [node 3] ICS09 handshake done sender: [4:7587241311544214232:2112] self: [4:7587241311544214248:0] peer: [3:7587241313075105480:0] socket: 592 qp: -1
2025-12-24T02:11:19.013425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241315839181553:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.013434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.013448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311544214235:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.015207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311544214243:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.015497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241311544214230:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.015506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.015513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311544214230:2109] [node 6] ICP22 created new session: [4:7587241315839181555:2109]
2025-12-24T02:11:19.015517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241315839181555:2109] [node 6] ICS09 handshake done sender: [4:7587241311544214235:2114] self: [4:7587241311544214234:0] peer: [6:7587241312538692921:0] socket: 605 qp: -1
2025-12-24T02:11:19.015523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241315839181555:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.015530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.015580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241311544214163:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.015589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.015596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311544214163:2049] [node 2] ICP22 created new session: [4:7587241315839181557:2049]
2025-12-24T02:11:19.015600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241315839181557:2049] [node 2] ICS09 handshake done sender: [4:7587241311544214243:2118] self: [4:7587241311544214246:0] peer: [2:7587241314743423744:0] socket: 627 qp: -1
2025-12-24T02:11:19.015605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241315839181557:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.015609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.015614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241311544214230:2109] [node 6] ICP18 handshake (actor [4:7587241315839181545:2120]) is too old
2025-12-24T02:11:19.017053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241311544214237:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.017059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241311544214237:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.017721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241311544214237:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.017847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241311544214228:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.017860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.017866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241311544214228:2108] [node 5] ICP22 created new session: [4:7587241315839181560:2108]
2025-12-24T02:11:19.017871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241315839181560:2108] [node 5] ICS09 handshake done sender: [4:7587241311544214237:2115] self: [4:7587241311544214236:0] peer: [5:7587241317879687734:0] socket: 619 qp: -1
2025-12-24T02:11:19.017877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241315839181560:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.017886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.022570Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:19.022867Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:19.022680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241314743423759:2048] [node 1] ICS07 socket disconnect 587 reason# EndOfStream
2025-12-24T02:11:19.022708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241314743423759:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.022767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.022798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241314743423759:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.022949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241313075105497:2048] [node 1] ICS01 socket: 634 reason# EndOfStream
2025-12-24T02:11:19.022962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241313075105410:2048] [node 1] ICP30 unregister session Session# [3:7587241313075105497:2048] VirtualId# [3:7587241313075105493:0]
2025-12-24T02:11:19.022968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241313075105497:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.023017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.023054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.023826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:19.023917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241315839181549:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-12-24T02:11:19.023931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241311544214162:2048] [node 1] ICP30 unregister session Session# [4:7587241315839181549:2048] VirtualId# [4:7587241311544214244:0]
2025-12-24T02:11:19.023937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241315839181549:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.023947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:19.023990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.024023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.024032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241313584720422:2048] [node 1] ICS07 socket disconnect 602 reason# EndOfStream
2025-12-24T02:11:19.024044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241313584720422:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.024106Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:19.024185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241316833660225:2048] [node 1] ICS07 socket disconnect 630 reason# EndOfStream
2025-12-24T02:11:19.024196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316833660225:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.024250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.024276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241316833660225:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.024109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.024145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241313584720422:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.024336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241314743423672:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:19.024351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241314743423759:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.024622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241313584720338:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.024636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241313584720422:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.024643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241313584720338:2048] [node 1] ICP30 unregister session Session# [5:7587241313584720422:2048] VirtualId# [5:7587241313584720420:0]
2025-12-24T02:11:19.024660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.024666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241313584720338:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.024669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.024702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241314743423672:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.024718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241314743423759:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.024724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241314743423672:2048] [node 1] ICP30 unregister session Session# [2:7587241314743423759:2048] VirtualId# [2:7587241314743423757:0]
2025-12-24T02:11:19.024746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.024752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241314743423672:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.024755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.024871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241312538692836:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.024892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316833660225:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.024896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312538692836:2048] [node 1] ICP30 unregister session Session# [6:7587241316833660225:2048] VirtualId# [6:7587241312538692917:0]
2025-12-24T02:11:19.024910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.024915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241312538692836:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.024919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26157) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.033482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:19.033592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241317370072805:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-12-24T02:11:19.033609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241313075105411:2049] [node 2] ICP30 unregister session Session# [3:7587241317370072805:2049] VirtualId# [3:7587241313075105494:0]
2025-12-24T02:11:19.033617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241317370072805:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.033674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.033705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.034587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241312538692836:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.034543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241313584720338:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.035115Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:19.035217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316833660223:2049] [node 2] ICS01 socket: 633 reason# EndOfStream
2025-12-24T02:11:19.035230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312538692837:2049] [node 2] ICP30 unregister session Session# [6:7587241316833660223:2049] VirtualId# [6:7587241312538692919:0]
2025-12-24T02:11:19.035237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316833660223:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.035293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.035335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.036205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:19.036305Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:19.036305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241315839181557:2049] [node 2] ICS01 socket: 627 reason# EndOfStream
2025-12-24T02:11:19.036316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241311544214163:2049] [node 2] ICP30 unregister session Session# [4:7587241315839181557:2049] VirtualId# [4:7587241311544214246:0]
2025-12-24T02:11:19.036325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241315839181557:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.036387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.036388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241317879687731:2049] [node 2] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:19.036400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241313584720339:2049] [node 2] ICP30 unregister session Session# [5:7587241317879687731:2049] VirtualId# [5:7587241313584720425:0]
2025-12-24T02:11:19.036408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241317879687731:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.036422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.036457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.036498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.047291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:19.047377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316833660232:2050] [node 3] ICS01 socket: 595 reason# EndOfStream
2025-12-24T02:11:19.047386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312538692838:2050] [node 3] ICP30 unregister session Session# [6:7587241316833660232:2050] VirtualId# [6:7587241316833660229:0]
2025-12-24T02:11:19.047392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316833660232:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.047460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.047492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.046973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:19.047084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241315839181553:2050] [node 3] ICS07 socket disconnect 592 reason# EndOfStream
2025-12-24T02:11:19.047092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241315839181553:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.047162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.047193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241315839181553:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:19.048828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:19.048949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241313584720433:2050] [node 3] ICS01 socket: 647 reason# EndOfStream
2025-12-24T02:11:19.048965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241313584720340:2050] [node 3] ICP30 unregister session Session# [5:7587241313584720433:2050] VirtualId# [5:7587241313584720431:0]
2025-12-24T02:11:19.048972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241313584720433:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.049032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.049067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.054539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:19.054765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:19.054842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316833660234:2109] [node 4] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:19.054855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312538692903:2109] [node 4] ICP30 unregister session Session# [6:7587241316833660234:2109] VirtualId# [6:7587241312538692921:0]
2025-12-24T02:11:19.054861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316833660234:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.054904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.054934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.054634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241317879687736:2108] [node 4] ICS07 socket disconnect 651 reason# EndOfStream
2025-12-24T02:11:19.054647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241317879687736:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.054747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.054786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241317879687736:2108] [node 4] ICS15 start handshake
2025-12-24T02:11:19.055108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241313584720401:2108] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8557) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.055117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241317879687736:2108] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.055137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241313584720401:2108] [node 4] ICP30 unregister session Session# [5:7587241317879687736:2108] VirtualId# [5:7587241317879687734:0]
2025-12-24T02:11:19.055158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.055165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241313584720401:2108] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.055167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8557) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.061271Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:19.061357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316833660227:2107] [node 5] ICS01 socket: 613 reason# EndOfStream
2025-12-24T02:11:19.061374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241312538692897:2107] [node 5] ICP30 unregister session Session# [6:7587241316833660227:2107] VirtualId# [6:7587241312538692920:0]
2025-12-24T02:11:19.061382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316833660227:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.061430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:19.061460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:19.076183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315828382039:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:19.076195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315828382040:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.077690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241318990892152:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.077737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241318990892153:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.082037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316038920438:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.082060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316038920439:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.082068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316038920440:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.082240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316129426804:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.082247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316129426805:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.082251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316129426806:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.083913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317507748533:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.083928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317507748535:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.083935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317507748534:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.093716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317507748595:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.094372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426865:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.094532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315828382100:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.094605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892206:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.079144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317163571839:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.079151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317163571840:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.095052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241318990892213:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:19.095274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316038920499:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:19.095370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316038920500:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:19.095170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241318990892214:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:19.095755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241318990892218:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:19.095062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317163571895:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:19.095216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317163571900:2093] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.108184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920509:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.109599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315828382108:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.109672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315828382109:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.110154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315828382111:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49981
2025-12-24T02:11:19.110800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426871:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.116130Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241317163571900:2093] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20856113ns) {Configure:104(20855514ns) {ProcessConfigured:135(16024703ns) {StartInitialHandshake:157(16024143ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1635ns) {SwitchToState:179(929ns)}}}}}
2025-12-24T02:11:19.116341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317163571909:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:19.117327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571910:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
2025-12-24T02:11:19.117343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571911:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.119302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316129426875:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:19.119396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426865:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.119517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316129426876:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.129264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315828382110:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49981
2025-12-24T02:11:19.127605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426878:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.127649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426879:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.127681Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426880:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.131635Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241317507748534:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19481106ns) {Configure:104(19480118ns) {ProcessConfigured:135(19137297ns) {StartInitialHandshake:157(19136705ns) {DropHandshakes:470(488ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:477(1425ns) {SwitchToState:179(705ns)}}}}}
2025-12-24T02:11:19.131819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317507748604:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:19.132802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426879:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.132909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892233:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.132998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892206:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.138873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382114:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.138895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315828382114:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.139409Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315828382114:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.139605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315828382108:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.139629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.139638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315828382108:2108] [node 5] ICP22 created new session: [1:7587241315828382120:2108]
2025-12-24T02:11:19.139644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315828382120:2108] [node 5] ICS09 handshake done sender: [1:7587241315828382114:2115] self: [1:7587241315828382112:0] peer: [5:7587241316129426887:0] socket: 607 qp: -1
No RDMA context found for GID: 2025-12-24T02:11:19.139653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315828382120:2108] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-12-24T02:11:19.139669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.139877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382110:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.139958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241315828382108:2108] [node 5] ICP18 handshake (actor [1:7587241315828382110:2112]) is too old
2025-12-24T02:11:19.141445Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241315828382109:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31391897ns) {Configure:104(31390723ns) {ProcessConfigured:135(30933055ns) {StartInitialHandshake:157(30932370ns) {DropHandshakes:470(621ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1290ns) {SwitchToState:179(359ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.142161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920517:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
2025-12-24T02:11:19.142402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920517:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.142432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316038920440:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.138989Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241317163571895:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(43860551ns) {Configure:104(43859916ns) {ProcessConfigured:135(38764921ns) {StartInitialHandshake:157(38764200ns) {DropHandshakes:470(261ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(149ns) PrepareNewSessionHandshake:477(1366ns) {SwitchToState:179(213ns)}}}}}
2025-12-24T02:11:19.142300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571911:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.139682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426888:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.141700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241316129426806:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22115307ns) {Configure:104(22114636ns) {ProcessConfigured:135(21800998ns) {StartInitialHandshake:157(21800336ns) {DropHandshakes:470(744ns) {DropIncomingHandshake:442(95ns) DropOutgoingHandshake:457(156ns)} GenerateSessionVirtualId:988(259ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(662ns)}}}}}
2025-12-24T02:11:19.142810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241315828382040:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48903407ns) {Configure:104(48895768ns) {ProcessConfigured:135(45247595ns) {StartInitialHandshake:157(45246637ns) {DropHandshakes:470(219ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(1571ns) {SwitchToState:179(535ns)}}}}}
2025-12-24T02:11:19.143070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426880:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.144004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426878:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.144095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316129426875:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.144131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571908:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.144145Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317163571908:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.144658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317163571908:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.144814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317163571900:2093] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.144849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.144856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317163571900:2093] [node 5] ICP22 created new session: [3:7587241317163571922:2093]
2025-12-24T02:11:19.144862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317163571922:2093] [node 5] ICS09 handshake done sender: [3:7587241317163571908:2111] self: [3:7587241317163571905:0] peer: [5:7587241316129426890:0] socket: 612 qp: -1
2025-12-24T02:11:19.144870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317163571922:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:19.144878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.145361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426893:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.147031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920508:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.147051Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241316038920508:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.147067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241317163571839:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(52109874ns) {Configure:104(52108530ns) {ProcessConfigured:135(47114008ns) {StartInitialHandshake:157(47113258ns) {DropHandshakes:470(788ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(150ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:477(889ns) {SwitchToState:179(478ns)}}}}}
2025-12-24T02:11:19.147247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426871:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.147277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316129426805:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.148133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571915:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.148154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317163571915:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.150739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316129426804:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.150784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241316129426804:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17927061ns) {IncomingHandshake:261(17926531ns) {DropIncomingHandshake:442(49ns) PrepareNewSessionHandshake:477(476ns) {SwitchToState:179(108ns)} GenerateSessionVirtualId:988(172ns) IssueIncomingHandshakeReply:193(17922186ns) {SwitchToState:179(660ns)}}}
2025-12-24T02:11:19.150817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316129426804:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241316129426879:2115] held: no
2025-12-24T02:11:19.150858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316129426879:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.151180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892233:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.151219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892223:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.151222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241318990892223:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.151795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892235:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.151835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892236:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.151930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241318990892223:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.152255Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241317507748535:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39700525ns) {Configure:104(39699778ns) {ProcessConfigured:135(39469238ns) {StartInitialHandshake:157(39468685ns) {DropHandshakes:470(303ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(21ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(1204ns) {SwitchToState:179(604ns)}}}}}
2025-12-24T02:11:19.152347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748611:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.152389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748612:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.152247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316129426804:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.152264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.152271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316129426804:2048] [node 1] ICP22 created new session: [5:7587241316129426896:2048]
2025-12-24T02:11:19.152277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316129426896:2048] [node 1] ICS09 handshake done sender: [5:7587241316129426879:2115] self: [5:7587241316129426887:0] peer: [1:7587241315828382112:0] socket: 618 qp: -1
2025-12-24T02:11:19.152284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316129426896:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.152292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.152444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426895:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.152510Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241317507748533:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40465977ns) {Configure:104(40463867ns) {ProcessConfigured:135(40098640ns) {StartInitialHandshake:157(40098137ns) {DropHandshakes:470(461ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(358ns) PrepareNewSessionHandshake:477(534ns) {SwitchToState:179(59ns)}}}}}
2025-12-24T02:11:19.152794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316129426884:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.152804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241316129426884:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.152820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571919:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.152832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317163571919:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.152876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892235:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.152894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241318990892213:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.152656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317507748612:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.152689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317507748535:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.152913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.152692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317507748611:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.152925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241318990892213:2105] [node 5] ICP22 created new session: [2:7587241318990892239:2105]
2025-12-24T02:11:19.152721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317507748595:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.152931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241318990892239:2105] [node 5] ICS09 handshake done sender: [2:7587241318990892223:2112] self: [2:7587241318990892216:0] peer: [5:7587241316129426894:0] socket: 595 qp: -1
2025-12-24T02:11:19.152937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241318990892239:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:19.152955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.152988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316129426806:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.153019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316129426880:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.152982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748610:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.153103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316129426806:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.153120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.153126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316129426806:2050] [node 3] ICP22 created new session: [5:7587241316129426898:2050]
2025-12-24T02:11:19.153131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316129426898:2050] [node 3] ICS09 handshake done sender: [5:7587241316129426880:2116] self: [5:7587241316129426890:0] peer: [3:7587241317163571905:0] socket: 619 qp: -1
2025-12-24T02:11:19.153138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316129426898:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.153148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.153517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316038920508:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.153597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892236:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.153670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241316038920500:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.153682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.153688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316038920500:2106] [node 5] ICP22 created new session: [4:7587241316038920521:2106]
2025-12-24T02:11:19.153693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316038920521:2106] [node 5] ICS09 handshake done sender: [4:7587241316038920508:2111] self: [4:7587241316038920507:0] peer: [5:7587241316129426892:0] socket: 602 qp: -1
2025-12-24T02:11:19.153698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316038920521:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:19.153703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.154173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317163571915:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.154219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920519:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
2025-12-24T02:11:19.154231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241318990892152:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.154325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571910:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.154339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317163571909:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.154350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.154360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317163571909:2107] [node 4] ICP22 created new session: [3:7587241317163571927:2107]
2025-12-24T02:11:19.154380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920509:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.154381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317163571927:2107] [node 4] ICS09 handshake done sender: [3:7587241317163571915:2115] self: [3:7587241317163571914:0] peer: [4:7587241316038920518:0] socket: 614 qp: -1
2025-12-24T02:11:19.154393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317163571927:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.154399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.154623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920523:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.154675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920524:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.154865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920525:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
2025-12-24T02:11:19.155093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316038920517:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.155158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241317163571840:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.155223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316038920439:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.155332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316038920440:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.155348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.155353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316038920440:2050] [node 3] ICP22 created new session: [4:7587241316038920528:2050]
2025-12-24T02:11:19.155358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316038920528:2050] [node 3] ICS09 handshake done sender: [4:7587241316038920517:2116] self: [4:7587241316038920518:0] peer: [3:7587241317163571914:0] socket: 617 qp: -1
2025-12-24T02:11:19.155362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316038920528:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.155378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.155559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892225:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.155569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241318990892225:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.156075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241318990892225:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.156121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
2025-12-24T02:11:19.156496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316129426805:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241316129426871:2110] held: no
2025-12-24T02:11:19.156200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892227:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.156215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241318990892227:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.156616Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241316038920440:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.156815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317163571919:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.156755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920524:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.156782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920519:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.157661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241316038920500:2106] [node 5] ICP18 handshake (actor [4:7587241316038920519:2117]) is too old
2025-12-24T02:11:19.157893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241318990892153:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.157909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.157912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241318990892153:2049] [node 3] ICP22 created new session: [2:7587241318990892245:2049]
2025-12-24T02:11:19.157917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241318990892245:2049] [node 3] ICS09 handshake done sender: [2:7587241318990892225:2113] self: [2:7587241318990892224:0] peer: [3:7587241317163571930:0] socket: 596 qp: -1
2025-12-24T02:11:19.157921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241318990892245:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.157925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.158167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241318990892227:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.158314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316129426901:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54354
2025-12-24T02:11:19.158341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241318990892218:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.158352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.158359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241318990892218:2109] [node 4] ICP22 created new session: [2:7587241318990892249:2109]
2025-12-24T02:11:19.158364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241318990892249:2109] [node 4] ICS09 handshake done sender: [2:7587241318990892227:2114] self: [2:7587241318990892226:0] peer: [4:7587241316038920527:0] socket: 597 qp: -1
2025-12-24T02:11:19.158368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241318990892249:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:19.158374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.158635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317163571895:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.158646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.158651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317163571895:2096] [node 6] ICP22 created new session: [3:7587241317163571933:2096]
2025-12-24T02:11:19.158655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317163571933:2096] [node 6] ICS09 handshake done sender: [3:7587241317163571919:2116] self: [3:7587241317163571906:0] peer: [6:7587241317507748614:0] socket: 630 qp: -1
2025-12-24T02:11:19.158660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317163571933:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:19.158664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.158961Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316129426878:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.159085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316129426884:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.159746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316129426875:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25260) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241316129426878:2114] held: no
2025-12-24T02:11:19.159767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316129426875:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.159775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.159781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316129426875:2112] [node 4] ICP22 created new session: [5:7587241316129426902:2112]
2025-12-24T02:11:19.159786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316129426902:2112] [node 4] ICS09 handshake done sender: [5:7587241316129426878:2114] self: [5:7587241316129426892:0] peer: [4:7587241316038920507:0] socket: 609 qp: -1
2025-12-24T02:11:19.159791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316129426902:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:19.159797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.159886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316129426871:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.160006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382107:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.160021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315828382107:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.159465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317163571910:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.159547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241317163571840:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.159556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.159563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317163571840:2049] [node 2] ICP22 created new session: [3:7587241317163571935:2049]
2025-12-24T02:11:19.159569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317163571935:2049] [node 2] ICS09 handshake done sender: [3:7587241317163571910:2112] self: [3:7587241317163571930:0] peer: [2:7587241318990892224:0] socket: 610 qp: -1
2025-12-24T02:11:19.159574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317163571935:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.160537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748616:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.159697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920523:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.159580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.160061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241317163571840:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.160829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316038920438:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.160898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920532:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
2025-12-24T02:11:19.160911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892251:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.161060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316038920509:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.160992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317507748616:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.161025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317507748534:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.161125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316038920439:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.161137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.161142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316038920439:2049] [node 2] ICP22 created new session: [4:7587241316038920534:2049]
2025-12-24T02:11:19.161145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316038920534:2049] [node 2] ICS09 handshake done sender: [4:7587241316038920509:2112] self: [4:7587241316038920527:0] peer: [2:7587241318990892226:0] socket: 600 qp: -1
2025-12-24T02:11:19.161149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316038920534:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.161154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.161086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241318990892229:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.161093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241318990892229:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.161615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241318990892229:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.161735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241318990892214:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.161750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.161755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241318990892214:2107] [node 6] ICP22 created new session: [2:7587241318990892253:2107]
2025-12-24T02:11:19.161761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241318990892253:2107] [node 6] ICS09 handshake done sender: [2:7587241318990892229:2115] self: [2:7587241318990892219:0] peer: [6:7587241317507748617:0] socket: 592 qp: -1
2025-12-24T02:11:19.161767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241318990892253:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:19.161773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.162059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317507748534:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6857) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241317507748616:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.162650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.162931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317507748610:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.162963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317507748533:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.162995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748621:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.163035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748622:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.163111Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317507748612:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.163265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317507748535:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.163281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.163287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317507748535:2050] [node 3] ICP22 created new session: [6:7587241317507748624:2050]
2025-12-24T02:11:19.163292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317507748624:2050] [node 3] ICS09 handshake done sender: [6:7587241317507748612:2116] self: [6:7587241317507748614:0] peer: [3:7587241317163571906:0] socket: 637 qp: -1
2025-12-24T02:11:19.163299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317507748624:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.163307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.166970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382122:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.166985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315828382122:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.167373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382111:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.163394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316129426805:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:19.163404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.163408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316129426805:2049] [node 2] ICP22 created new session: [5:7587241316129426904:2049]
2025-12-24T02:11:19.163411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316129426904:2049] [node 2] ICS09 handshake done sender: [5:7587241316129426871:2110] self: [5:7587241316129426894:0] peer: [2:7587241318990892216:0] socket: 599 qp: -1
2025-12-24T02:11:19.163415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316129426904:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.163436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.163541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241316129426876:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.163550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.163554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316129426876:2113] [node 6] ICP22 created new session: [5:7587241316129426906:2113]
2025-12-24T02:11:19.163558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316129426906:2113] [node 6] ICS09 handshake done sender: [5:7587241316129426884:2118] self: [5:7587241316129426883:0] peer: [6:7587241317507748615:0] socket: 594 qp: -1
2025-12-24T02:11:19.163561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316129426906:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:19.163566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.167970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315828382107:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.168070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748619:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.168252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382113:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.168269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315828382113:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.168959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315828382128:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49981
2025-12-24T02:11:19.169002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315828382129:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49981
2025-12-24T02:11:19.169172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315828382113:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.169201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315828382122:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.169441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382129:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.169455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315828382100:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.169467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.169474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315828382100:2107] [node 4] ICP22 created new session: [1:7587241315828382131:2107]
2025-12-24T02:11:19.169479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315828382131:2107] [node 4] ICS09 handshake done sender: [1:7587241315828382113:2114] self: [1:7587241315828382105:0] peer: [4:7587241316038920533:0] socket: 621 qp: -1
2025-12-24T02:11:19.169484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315828382131:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.169491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.168755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748620:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.169868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315828382109:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.169880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.169885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315828382109:2109] [node 6] ICP22 created new session: [1:7587241315828382133:2109]
2025-12-24T02:11:19.169891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315828382133:2109] [node 6] ICS09 handshake done sender: [1:7587241315828382122:2116] self: [1:7587241315828382117:0] peer: [6:7587241317507748623:0] socket: 628 qp: -1
2025-12-24T02:11:19.169895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315828382133:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.169900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.170217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316038920536:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44130
2025-12-24T02:11:19.170298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317507748616:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.170329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316038920523:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.170536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241317163571895:2096] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.170648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382128:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.172027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317507748619:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.172057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317507748595:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21204) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241317507748611:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.172169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317507748534:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.172182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.172192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317507748534:2049] [node 2] ICP22 created new session: [6:7587241317507748628:2049]
2025-12-24T02:11:19.172199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317507748628:2049] [node 2] ICS09 handshake done sender: [6:7587241317507748616:2118] self: [6:7587241317507748617:0] peer: [2:7587241318990892219:0] socket: 645 qp: -1
2025-12-24T02:11:19.172211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317507748628:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.172219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.172339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317507748611:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.172709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316038920438:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.172728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.172735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316038920438:2048] [node 1] ICP22 created new session: [4:7587241316038920537:2048]
2025-12-24T02:11:19.172741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316038920537:2048] [node 1] ICS09 handshake done sender: [4:7587241316038920523:2118] self: [4:7587241316038920533:0] peer: [1:7587241315828382105:0] socket: 638 qp: -1
2025-12-24T02:11:19.172748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316038920537:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.172755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.172832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571938:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
2025-12-24T02:11:19.172900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748626:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.172999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317507748595:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.173011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.173017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317507748595:2108] [node 5] ICP22 created new session: [6:7587241317507748630:2108]
2025-12-24T02:11:19.173022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317507748630:2108] [node 5] ICS09 handshake done sender: [6:7587241317507748611:2115] self: [6:7587241317507748615:0] peer: [5:7587241316129426883:0] socket: 636 qp: -1
2025-12-24T02:11:19.173025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317507748604:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.173028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317507748630:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.173046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.173386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315828382138:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49981
2025-12-24T02:11:19.173769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317507748610:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.173286Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241316038920438:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.173455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316038920515:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.173458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241316038920515:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.174342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317507748533:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.174357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.174363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317507748533:2048] [node 1] ICP22 created new session: [6:7587241317507748634:2048]
2025-12-24T02:11:19.174369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317507748634:2048] [node 1] ICS09 handshake done sender: [6:7587241317507748610:2114] self: [6:7587241317507748623:0] peer: [1:7587241315828382117:0] socket: 635 qp: -1
2025-12-24T02:11:19.174374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317507748634:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.174386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.174468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316038920515:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.174717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241318990892218:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.174808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317507748633:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46459
2025-12-24T02:11:19.175051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317163571938:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.175091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241317163571839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.175112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317507748619:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.175237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241318990892255:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51482
2025-12-24T02:11:19.175223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315828382039:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.175239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.175249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315828382039:2048] [node 2] ICP22 created new session: [1:7587241315828382139:2048]
2025-12-24T02:11:19.175260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315828382139:2048] [node 2] ICS09 handshake done sender: [1:7587241315828382107:2111] self: [1:7587241315828382104:0] peer: [2:7587241318990892243:0] socket: 622 qp: -1
2025-12-24T02:11:19.175271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317507748604:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.175267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315828382139:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:19.175283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.175294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317507748604:2110] [node 4] ICP22 created new session: [6:7587241317507748636:2110]
2025-12-24T02:11:19.175301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317507748636:2110] [node 4] ICS09 handshake done sender: [6:7587241317507748619:2119] self: [6:7587241317507748631:0] peer: [4:7587241316038920502:0] socket: 603 qp: -1
2025-12-24T02:11:19.175307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317507748636:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:19.175314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.175273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.175403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241316038920499:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.175429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.175435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316038920499:2103] [node 6] ICP22 created new session: [4:7587241316038920540:2103]
2025-12-24T02:11:19.175439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316038920540:2103] [node 6] ICS09 handshake done sender: [4:7587241316038920515:2114] self: [4:7587241316038920502:0] peer: [6:7587241317507748631:0] socket: 644 qp: -1
2025-12-24T02:11:19.175443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316038920540:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:19.175448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.175554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315828382124:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.175566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315828382124:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.176326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315828382124:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.176662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315828382040:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.176682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.176687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315828382040:2049] [node 3] ICP22 created new session: [1:7587241315828382142:2049]
2025-12-24T02:11:19.176692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315828382142:2049] [node 3] ICS09 handshake done sender: [1:7587241315828382124:2117] self: [1:7587241315828382106:0] peer: [3:7587241317163571939:0] socket: 623 qp: -1
2025-12-24T02:11:19.176697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315828382142:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.176704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.177818Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241317163571839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241317163571938:2120] held: no
2025-12-24T02:11:19.175345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241318990892235:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.175383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241318990892152:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.175392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.175397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241318990892152:2048] [node 1] ICP22 created new session: [2:7587241318990892256:2048]
2025-12-24T02:11:19.175402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241318990892256:2048] [node 1] ICS09 handshake done sender: [2:7587241318990892235:2118] self: [2:7587241318990892243:0] peer: [1:7587241315828382104:0] socket: 625 qp: -1
2025-12-24T02:11:19.175407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241318990892256:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.175412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.179547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317163571940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22573
2025-12-24T02:11:19.179804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317163571938:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.179972Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241315828382109:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.179895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241317163571839:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.179914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.179920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317163571839:2048] [node 1] ICP22 created new session: [3:7587241317163571941:2048]
2025-12-24T02:11:19.179925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317163571941:2048] [node 1] ICS09 handshake done sender: [3:7587241317163571938:2120] self: [3:7587241317163571939:0] peer: [1:7587241315828382106:0] socket: 606 qp: -1
2025-12-24T02:11:19.179930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317163571941:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.179940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.189750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:19.189839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241318990892256:2048] [node 1] ICS07 socket disconnect 625 reason# EndOfStream
2025-12-24T02:11:19.189845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241318990892256:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.189911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.189933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241318990892256:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.190063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:19.190136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241317163571941:2048] [node 1] ICS01 socket: 606 reason# EndOfStream
2025-12-24T02:11:19.190142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241317163571839:2048] [node 1] ICP30 unregister session Session# [3:7587241317163571941:2048] VirtualId# [3:7587241317163571939:0]
2025-12-24T02:11:19.190148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241317163571941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.190194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.190223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.192712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241318990892152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:19.192726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241318990892256:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.193021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241318990892152:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.193074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241318990892256:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.193082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241318990892152:2048] [node 1] ICP30 unregister session Session# [2:7587241318990892256:2048] VirtualId# [2:7587241318990892243:0]
2025-12-24T02:11:19.193102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.193108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241318990892152:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.193111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.194793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:19.194871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241316038920537:2048] [node 1] ICS07 socket disconnect 638 reason# EndOfStream
2025-12-24T02:11:19.194875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241316038920537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.194921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.194944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241316038920537:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.195153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:19.195229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316129426896:2048] [node 1] ICS01 socket: 618 reason# EndOfStream
2025-12-24T02:11:19.195234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316129426804:2048] [node 1] ICP30 unregister session Session# [5:7587241316129426896:2048] VirtualId# [5:7587241316129426887:0]
2025-12-24T02:11:19.195240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316129426896:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.195269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.195296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.195408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:19.195536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241316038920438:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.195549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241316038920537:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.195553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241316038920438:2048] [node 1] ICP30 unregister session Session# [4:7587241316038920537:2048] VirtualId# [4:7587241316038920533:0]
2025-12-24T02:11:19.195567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.195572Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241316038920438:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.195574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.195497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241317507748634:2048] [node 1] ICS07 socket disconnect 635 reason# EndOfStream
2025-12-24T02:11:19.195508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317507748634:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.195576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.195612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241317507748634:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.195955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317507748533:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.195966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317507748634:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.195971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317507748533:2048] [node 1] ICP30 unregister session Session# [6:7587241317507748634:2048] VirtualId# [6:7587241317507748623:0]
2025-12-24T02:11:19.195987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.195993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317507748533:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.195999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15811) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.205755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241316038920438:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.206892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241317507748533:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.213864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:19.213944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241316038920534:2049] [node 2] ICS07 socket disconnect 600 reason# EndOfStream
2025-12-24T02:11:19.213954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241316038920534:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.213994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.214022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241316038920534:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:19.214333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241316038920439:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6857) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.214345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241316038920534:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.214353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241316038920439:2049] [node 2] ICP30 unregister session Session# [4:7587241316038920534:2049] VirtualId# [4:7587241316038920527:0]
2025-12-24T02:11:19.214368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.214382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241316038920439:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6857) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.214385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6857) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.213770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:19.213886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316129426904:2049] [node 2] ICS01 socket: 599 reason# EndOfStream
2025-12-24T02:11:19.213896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316129426805:2049] [node 2] ICP30 unregister session Session# [5:7587241316129426904:2049] VirtualId# [5:7587241316129426894:0]
2025-12-24T02:11:19.213903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316129426904:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.213949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.213973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.214742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:19.214769Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:19.214850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317507748628:2049] [node 2] ICS01 socket: 645 reason# EndOfStream
2025-12-24T02:11:19.214858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317507748534:2049] [node 2] ICP30 unregister session Session# [6:7587241317507748628:2049] VirtualId# [6:7587241317507748617:0]
2025-12-24T02:11:19.214864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317507748628:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.214904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.214935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.215064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241317163571935:2049] [node 2] ICS01 socket: 610 reason# EndOfStream
2025-12-24T02:11:19.215077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241317163571840:2049] [node 2] ICP30 unregister session Session# [3:7587241317163571935:2049] VirtualId# [3:7587241317163571930:0]
2025-12-24T02:11:19.215081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241317163571935:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.215121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.215150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.223949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241316038920439:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:19.234807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:19.234891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317507748624:2050] [node 3] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:19.234899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317507748535:2050] [node 3] ICP30 unregister session Session# [6:7587241317507748624:2050] VirtualId# [6:7587241317507748614:0]
2025-12-24T02:11:19.234906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317507748624:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.234960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.234984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.231971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:19.232060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241316129426898:2050] [node 3] ICS07 socket disconnect 619 reason# EndOfStream
2025-12-24T02:11:19.232065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316129426898:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.232096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.232110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241316129426898:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:19.232505Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316129426806:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.232514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316129426898:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.232520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316129426806:2050] [node 3] ICP30 unregister session Session# [5:7587241316129426898:2050] VirtualId# [5:7587241316129426890:0]
2025-12-24T02:11:19.232535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.232540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241316129426806:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.232543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.240417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241316129426806:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:19.243948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:19.244046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316129426902:2112] [node 4] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:19.244061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316129426875:2112] [node 4] ICP30 unregister session Session# [5:7587241316129426902:2112] VirtualId# [5:7587241316129426892:0]
2025-12-24T02:11:19.244069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316129426902:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.244128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.244166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.244533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:19.244614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317507748636:2110] [node 4] ICS01 socket: 603 reason# EndOfStream
2025-12-24T02:11:19.244621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317507748604:2110] [node 4] ICP30 unregister session Session# [6:7587241317507748636:2110] VirtualId# [6:7587241317507748631:0]
2025-12-24T02:11:19.244626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317507748636:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.244677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.244702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.254344Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:19.254435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317507748630:2108] [node 5] ICS01 socket: 636 reason# EndOfStream
2025-12-24T02:11:19.254447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317507748595:2108] [node 5] ICP30 unregister session Session# [6:7587241317507748630:2108] VirtualId# [6:7587241317507748615:0]
2025-12-24T02:11:19.254455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317507748630:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.254512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:19.254544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:19.273419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318307086887:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:19.273421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318307086888:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.275261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241315307969218:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.275276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241315307969219:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.277040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317657947736:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.277046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317657947737:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.279092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319423902945:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.279099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319423902946:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.279103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319423902947:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.280942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319325604950:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.280943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319325604951:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.280952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319325604952:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.286978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317544336127:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.286995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317544336128:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.287000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317544336129:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.300934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318307086948:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:19.301314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317544336189:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.301298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318307086950:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.302535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605009:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.304487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318307086962:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30704
2025-12-24T02:11:19.306068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318307086954:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.305188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947797:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.306007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947796:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.303063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241315307969279:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:19.305094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241315307969285:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7541
2025-12-24T02:11:19.307527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319325605014:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.307207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241315307969292:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:19.308725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241315307969293:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:19.309980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969284:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.309996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241315307969284:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.310482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241315307969294:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7541
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.308939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947797:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.310739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947799:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.310874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241315307969284:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.311138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241315307969219:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.311155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.311163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241315307969219:2049] [node 3] ICP22 created new session: [2:7587241315307969299:2049]
2025-12-24T02:11:19.311170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241315307969299:2049] [node 3] ICS09 handshake done sender: [2:7587241315307969284:2109] self: [2:7587241315307969283:0] peer: [3:7587241317657947798:0] socket: 606 qp: -1
2025-12-24T02:11:19.311092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086962:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.311177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241315307969299:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.311198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.312141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318307086968:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30704
2025-12-24T02:11:19.311604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317657947804:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:19.312339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903006:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.315238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605009:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.318796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241317657947737:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.319007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317657947811:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:19.319032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317657947797:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.319597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241317657947737:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.319611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.319618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317657947737:2049] [node 2] ICP22 created new session: [3:7587241317657947812:2049]
2025-12-24T02:11:19.319624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317657947812:2049] [node 2] ICS09 handshake done sender: [3:7587241317657947797:2108] self: [3:7587241317657947798:0] peer: [2:7587241315307969283:0] socket: 607 qp: -1
2025-12-24T02:11:19.319631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317657947812:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.319648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.322956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319325605020:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:19.323246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317544336195:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.324063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903007:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.324300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319423903014:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.324745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241315307969303:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7541
2025-12-24T02:11:19.324956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319423903015:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:19.323953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241319325605014:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16349916ns) {Configure:104(16349402ns) {ProcessConfigured:135(16073992ns) {StartInitialHandshake:157(16073727ns) {DropHandshakes:470(342ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(615ns) {SwitchToState:179(310ns)}}}}}
2025-12-24T02:11:19.325746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969294:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.326044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969303:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.326846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969285:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.326888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241315307969218:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.327335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336193:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.327127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241317657947810:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:19.327610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317544336193:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.327669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317544336128:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.329033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969297:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.329048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241315307969297:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.329316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086953:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.329334Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318307086953:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.330888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969288:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.330914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241315307969288:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.329809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241319423902945:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16069764ns) {Configure:104(16068779ns) {ProcessConfigured:135(14689599ns) {StartInitialHandshake:157(14688890ns) {DropHandshakes:470(530ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(640ns) {SwitchToState:179(148ns)}}}}}
2025-12-24T02:11:19.329900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903007:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.330029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319423902946:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.331785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903006:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.333036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319423902945:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.333134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903025:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.339509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241315307969297:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.339615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086960:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.339633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318307086960:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.339620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947816:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.339598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241315307969307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7541
2025-12-24T02:11:19.339826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947796:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.339850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318307086953:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.339856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086968:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.339860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947816:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.339992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318307086887:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.340016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.340029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318307086887:2048] [node 2] ICP22 created new session: [1:7587241318307086972:2048]
2025-12-24T02:11:19.340035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318307086972:2048] [node 2] ICS09 handshake done sender: [1:7587241318307086953:2110] self: [1:7587241318307086952:0] peer: [2:7587241315307969306:0] socket: 599 qp: -1
2025-12-24T02:11:19.340046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318307086972:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:19.340053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.343098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903028:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:19.342753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318307086969:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30704
2025-12-24T02:11:19.343132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241317657947736:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.343178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947819:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.343202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903027:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.343365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903028:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.343447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903027:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.343640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969307:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.344418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903025:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.344454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319423902947:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.343987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947819:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.344528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947806:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.344533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317657947806:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.345263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317657947806:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.345351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317657947804:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.344607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241315307969218:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241315307969285:2110] held: no
2025-12-24T02:11:19.345902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241315307969288:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:19.345996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241315307969279:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.346013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.346019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241315307969279:2107] [node 4] ICP22 created new session: [2:7587241315307969312:2107]
2025-12-24T02:11:19.346025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241315307969312:2107] [node 4] ICS09 handshake done sender: [2:7587241315307969288:2111] self: [2:7587241315307969287:0] peer: [4:7587241319423903024:0] socket: 608 qp: -1
2025-12-24T02:11:19.346031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241315307969312:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.346037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.346209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241315307969314:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7541
2025-12-24T02:11:19.346239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605030:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.345366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.345379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317657947804:2111] [node 4] ICP22 created new session: [3:7587241317657947823:2111]
2025-12-24T02:11:19.345385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317657947823:2111] [node 4] ICS09 handshake done sender: [3:7587241317657947806:2112] self: [3:7587241317657947805:0] peer: [4:7587241319423903030:0] socket: 618 qp: -1
2025-12-24T02:11:19.345391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317657947823:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:19.345399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.345843Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241317657947810:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18519171ns) {Configure:104(18516656ns) {ProcessConfigured:135(18203908ns) {StartInitialHandshake:157(18203412ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(458ns) PrepareNewSessionHandshake:477(880ns) {SwitchToState:179(546ns)}}}}}
2025-12-24T02:11:19.346739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241319423902946:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241319423903007:2109] held: no
2025-12-24T02:11:19.346408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086969:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.348307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336207:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.348435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241315307969293:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.348462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.348470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241315307969293:2113] [node 6] ICP22 created new session: [2:7587241315307969315:2113]
2025-12-24T02:11:19.348478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241315307969315:2113] [node 6] ICS09 handshake done sender: [2:7587241315307969297:2115] self: [2:7587241315307969296:0] peer: [6:7587241317544336206:0] socket: 616 qp: -1
2025-12-24T02:11:19.348484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241315307969315:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:19.348504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.348306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086956:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.348327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318307086956:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.348691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947825:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.348699Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317657947825:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.349192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317657947825:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.349417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317657947810:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.349433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.349444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317657947810:2113] [node 5] ICP22 created new session: [3:7587241317657947829:2113]
2025-12-24T02:11:19.349449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317657947829:2113] [node 5] ICS09 handshake done sender: [3:7587241317657947825:2118] self: [3:7587241317657947817:0] peer: [5:7587241319325605032:0] socket: 647 qp: -1
2025-12-24T02:11:19.349454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317657947829:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:19.349460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.350540Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318307086960:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.350688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318307086948:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.350724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.350731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318307086948:2107] [node 4] ICP22 created new session: [1:7587241318307086977:2107]
2025-12-24T02:11:19.350737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318307086977:2107] [node 4] ICS09 handshake done sender: [1:7587241318307086960:2112] self: [1:7587241318307086959:0] peer: [4:7587241319423903026:0] socket: 602 qp: -1
2025-12-24T02:11:19.350742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318307086977:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.350750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.348291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605031:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.348561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605031:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.348596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319325604952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.349728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319325604952:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319325605031:2117] held: no
2025-12-24T02:11:19.350792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605030:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.350819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319325604951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.351624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317544336207:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.351679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317544336129:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.358852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318307086956:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.358958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336208:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.359100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317544336193:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.358945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241317657947815:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.358976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241317657947815:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.359316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241315307969285:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.359473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241319423902947:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14651) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241319423903025:2116] held: no
2025-12-24T02:11:19.359466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318307086888:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.359757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903033:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.359494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.359759Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241319423903015:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19644065ns) {Configure:104(19641751ns) {ProcessConfigured:135(19349594ns) {StartInitialHandshake:157(19348995ns) {DropHandshakes:470(514ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(158ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(550ns) {SwitchToState:179(504ns)}}}}}
2025-12-24T02:11:19.359502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318307086888:2049] [node 3] ICP22 created new session: [1:7587241318307086981:2049]
2025-12-24T02:11:19.359509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318307086981:2049] [node 3] ICS09 handshake done sender: [1:7587241318307086956:2111] self: [1:7587241318307086955:0] peer: [3:7587241317657947820:0] socket: 600 qp: -1
2025-12-24T02:11:19.359517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318307086981:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.359526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.359836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319423903007:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.359920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319423902946:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.359938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.359949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319423902946:2049] [node 2] ICP22 created new session: [4:7587241319423903034:2049]
2025-12-24T02:11:19.359963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319423903034:2049] [node 2] ICS09 handshake done sender: [4:7587241319423903007:2109] self: [4:7587241319423903024:0] peer: [2:7587241315307969287:0] socket: 610 qp: -1
2025-12-24T02:11:19.359970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319423903034:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.359995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.360054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336210:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.360074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241315307969293:2113] [node 6] ICP35 event processing took too much time PendingConnection:279(20376543ns) {HandleHandshakeStatus:329(20375328ns) {DropIncomingHandshake:442(6977ns) DropOutgoingHandshake:457(107ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(1055500ns) {ForwardSessionEventToSession:612(1042435ns) ForwardSessionEventToSession:612(1780ns) ForwardSessionEventToSession:612(1178ns) ForwardSessionEventToSession:612(853ns) ForwardSessionEventToSession:612(661ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(675ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(511ns) ForwardSessionEventToSession:612(1187ns)}}}
2025-12-24T02:11:19.360100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.359023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.359942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605035:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.359986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319325604950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.361082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241319423902945:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241319423903006:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.361152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903035:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.361309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319423903036:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8299
2025-12-24T02:11:19.361458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319423903025:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.361526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319423903006:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.361698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319423902947:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.361711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.361717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319423902947:2050] [node 3] ICP22 created new session: [4:7587241319423903040:2050]
2025-12-24T02:11:19.361723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319423903040:2050] [node 3] ICS09 handshake done sender: [4:7587241319423903025:2116] self: [4:7587241319423903030:0] peer: [3:7587241317657947805:0] socket: 622 qp: -1
2025-12-24T02:11:19.361729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319423903040:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.361747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.361872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319423902945:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.361882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.361886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319423902945:2048] [node 1] ICP22 created new session: [4:7587241319423903042:2048]
2025-12-24T02:11:19.361891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319423903042:2048] [node 1] ICS09 handshake done sender: [4:7587241319423903006:2108] self: [4:7587241319423903026:0] peer: [1:7587241318307086959:0] socket: 609 qp: -1
2025-12-24T02:11:19.362091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319423903042:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.362102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.362208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336212:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.364317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903032:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.364327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241319423903032:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.364137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317544336212:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.364186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317544336195:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.365019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319423903032:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.365056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336214:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.365117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241319423903015:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.365135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.365141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319423903015:2111] [node 6] ICP22 created new session: [4:7587241319423903045:2111]
2025-12-24T02:11:19.365146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319423903045:2111] [node 6] ICS09 handshake done sender: [4:7587241319423903032:2119] self: [4:7587241319423903018:0] peer: [6:7587241317544336213:0] socket: 629 qp: -1
2025-12-24T02:11:19.365152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319423903045:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:19.365159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.365271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317544336212:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.365346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317544336195:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.365375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.365388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317544336195:2111] [node 4] ICP22 created new session: [6:7587241317544336215:2111]
2025-12-24T02:11:19.365398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317544336215:2111] [node 4] ICS09 handshake done sender: [6:7587241317544336212:2120] self: [6:7587241317544336213:0] peer: [4:7587241319423903018:0] socket: 631 qp: -1
2025-12-24T02:11:19.365404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317544336215:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:19.365424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.366248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241315307969218:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.366275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.366284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241315307969218:2048] [node 1] ICP22 created new session: [2:7587241315307969317:2048]
2025-12-24T02:11:19.366296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241315307969317:2048] [node 1] ICS09 handshake done sender: [2:7587241315307969285:2110] self: [2:7587241315307969306:0] peer: [1:7587241318307086952:0] socket: 605 qp: -1
2025-12-24T02:11:19.366304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241315307969317:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.366313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.366839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317657947815:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.366954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241317657947811:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.366970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.366977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317657947811:2114] [node 6] ICP22 created new session: [3:7587241317657947832:2114]
2025-12-24T02:11:19.366988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317657947832:2114] [node 6] ICS09 handshake done sender: [3:7587241317657947815:2115] self: [3:7587241317657947814:0] peer: [6:7587241317544336209:0] socket: 636 qp: -1
2025-12-24T02:11:19.366995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317657947832:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:19.367012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317544336128:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.367032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.367039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317544336128:2049] [node 2] ICP22 created new session: [6:7587241317544336217:2049]
2025-12-24T02:11:19.367031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.367047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317544336217:2049] [node 2] ICS09 handshake done sender: [6:7587241317544336193:2110] self: [6:7587241317544336206:0] peer: [2:7587241315307969296:0] socket: 621 qp: -1
2025-12-24T02:11:19.367058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317544336217:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.367076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.367583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086964:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.367600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318307086964:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.367637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605037:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.367867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605037:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.367918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319325605020:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.368853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241315307969302:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.368872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241315307969302:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.369289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605038:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.369417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319325605031:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.369535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319325604952:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.369553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.369560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319325604952:2050] [node 3] ICP22 created new session: [5:7587241319325605040:2050]
2025-12-24T02:11:19.369566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319325605040:2050] [node 3] ICS09 handshake done sender: [5:7587241319325605031:2117] self: [5:7587241319325605032:0] peer: [3:7587241317657947817:0] socket: 648 qp: -1
2025-12-24T02:11:19.369573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319325605040:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.369599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.369698Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241317657947736:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241317657947796:2107] held: no
2025-12-24T02:11:19.369246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319423903020:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.369259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241319423903020:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.369852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319423903020:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.370254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605042:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.370058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241319423903014:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.370080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.370086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319423903014:2110] [node 5] ICP22 created new session: [4:7587241319423903048:2110]
2025-12-24T02:11:19.370092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319423903048:2110] [node 5] ICS09 handshake done sender: [4:7587241319423903020:2115] self: [4:7587241319423903016:0] peer: [5:7587241319325605039:0] socket: 650 qp: -1
2025-12-24T02:11:19.370099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319423903048:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:19.370106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.370786Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336219:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.370879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318307086964:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.370980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318307086950:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.370990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.370997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318307086950:2108] [node 5] ICP22 created new session: [1:7587241318307086985:2108]
2025-12-24T02:11:19.371002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318307086985:2108] [node 5] ICS09 handshake done sender: [1:7587241318307086964:2114] self: [1:7587241318307086963:0] peer: [5:7587241319325605036:0] socket: 649 qp: -1
2025-12-24T02:11:19.371007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318307086985:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.371018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.375449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317544336211:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.376533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317544336210:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.376550Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317544336207:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.376651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241317544336128:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17450527ns) {HandleHandshakeStatus:329(17449772ns) {DropIncomingHandshake:442(460ns) DropOutgoingHandshake:457(1980ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(7381461ns) {ForwardSessionEventToSession:612(7368567ns) ForwardSessionEventToSession:612(1824ns) ForwardSessionEventToSession:612(988ns) ForwardSessionEventToSession:612(720ns) ForwardSessionEventToSession:612(641ns) ForwardSessionEventToSession:612(1128ns) ForwardSessionEventToSession:612(785ns) ForwardSessionEventToSession:612(634ns) ForwardSessionEventToSession:612(647ns) ForwardSessionEventToSession:612(737ns)}}}
2025-12-24T02:11:19.376672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241317544336128:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.376709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317544336129:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.376724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.376732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317544336129:2050] [node 3] ICP22 created new session: [6:7587241317544336222:2050]
2025-12-24T02:11:19.376738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317544336222:2050] [node 3] ICS09 handshake done sender: [6:7587241317544336207:2116] self: [6:7587241317544336209:0] peer: [3:7587241317657947814:0] socket: 637 qp: -1
2025-12-24T02:11:19.376747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317544336222:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.376757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.376856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317544336127:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.376872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317544336189:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.377489Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319325605037:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.377582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319325605020:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27424) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319325605037:2119] held: no
2025-12-24T02:11:19.377638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319325605020:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.377654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.377662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319325605020:2111] [node 4] ICP22 created new session: [5:7587241319325605044:2111]
2025-12-24T02:11:19.377668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319325605044:2111] [node 4] ICS09 handshake done sender: [5:7587241319325605037:2119] self: [5:7587241319325605039:0] peer: [4:7587241319423903016:0] socket: 628 qp: -1
2025-12-24T02:11:19.377675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319325605044:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:19.377682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.377744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319325605027:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.377748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241319325605027:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.378240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318307086984:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30704
2025-12-24T02:11:19.378237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947834:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
2025-12-24T02:11:19.378432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319325605027:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.378432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241317657947796:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.378517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086984:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.379035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318307086966:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.379056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318307086966:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.379105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241319325605014:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.379128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.379135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319325605014:2109] [node 6] ICP22 created new session: [5:7587241319325605048:2109]
2025-12-24T02:11:19.379141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319325605048:2109] [node 6] ICS09 handshake done sender: [5:7587241319325605027:2114] self: [5:7587241319325605015:0] peer: [6:7587241317544336225:0] socket: 623 qp: -1
2025-12-24T02:11:19.379148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319325605048:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.379160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.379704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241317657947736:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.379723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.379732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241317657947736:2048] [node 1] ICP22 created new session: [3:7587241317657947835:2048]
2025-12-24T02:11:19.379739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241317657947835:2048] [node 1] ICS09 handshake done sender: [3:7587241317657947796:2107] self: [3:7587241317657947820:0] peer: [1:7587241318307086955:0] socket: 601 qp: -1
2025-12-24T02:11:19.379746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241317657947835:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.379753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.380771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605043:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.380935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241317657947837:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15161
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.381126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336226:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.381212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318307086988:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30704
2025-12-24T02:11:19.381280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317544336210:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.381399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317544336189:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.381428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.381437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317544336189:2108] [node 5] ICP22 created new session: [6:7587241317544336227:2108]
2025-12-24T02:11:19.381447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317544336227:2108] [node 5] ICS09 handshake done sender: [6:7587241317544336210:2118] self: [6:7587241317544336225:0] peer: [5:7587241319325605015:0] socket: 633 qp: -1
2025-12-24T02:11:19.381454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317544336227:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.381461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.381926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318307086966:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.382028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318307086954:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.382047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.382035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319325605035:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.382061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318307086954:2109] [node 6] ICP22 created new session: [1:7587241318307086990:2109]
2025-12-24T02:11:19.382068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318307086990:2109] [node 6] ICS09 handshake done sender: [1:7587241318307086966:2115] self: [1:7587241318307086965:0] peer: [6:7587241317544336224:0] socket: 653 qp: -1
2025-12-24T02:11:19.382074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318307086990:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.382081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.382168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319325604951:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319325605030:2116] held: no
2025-12-24T02:11:19.382213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319325604950:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.382227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.382237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319325604950:2048] [node 1] ICP22 created new session: [5:7587241319325605051:2048]
2025-12-24T02:11:19.382242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319325605051:2048] [node 1] ICS09 handshake done sender: [5:7587241319325605035:2118] self: [5:7587241319325605036:0] peer: [1:7587241318307086963:0] socket: 630 qp: -1
2025-12-24T02:11:19.382248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319325605051:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.382258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.379630Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241315307969302:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.379812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241315307969292:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.379842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.379850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241315307969292:2112] [node 5] ICP22 created new session: [2:7587241315307969320:2112]
2025-12-24T02:11:19.379855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241315307969320:2112] [node 5] ICS09 handshake done sender: [2:7587241315307969302:2116] self: [2:7587241315307969295:0] peer: [5:7587241319325605034:0] socket: 641 qp: -1
2025-12-24T02:11:19.379860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241315307969320:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:19.379867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.384489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317544336127:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241317544336211:2119] held: no
2025-12-24T02:11:19.384567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319325605050:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54351
2025-12-24T02:11:19.384836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319325605030:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.384912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319325604951:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.384929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.384937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319325604951:2049] [node 2] ICP22 created new session: [5:7587241319325605053:2049]
2025-12-24T02:11:19.384947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319325605053:2049] [node 2] ICS09 handshake done sender: [5:7587241319325605030:2116] self: [5:7587241319325605034:0] peer: [2:7587241315307969295:0] socket: 642 qp: -1
2025-12-24T02:11:19.384956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319325605053:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.384968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.385625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317544336229:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10767
2025-12-24T02:11:19.385762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317544336211:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.385855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317544336127:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.385873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.385879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317544336127:2048] [node 1] ICP22 created new session: [6:7587241317544336230:2048]
2025-12-24T02:11:19.385884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317544336230:2048] [node 1] ICS09 handshake done sender: [6:7587241317544336211:2119] self: [6:7587241317544336224:0] peer: [1:7587241318307086965:0] socket: 597 qp: -1
2025-12-24T02:11:19.385889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317544336230:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.385901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.385911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241318307086950:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.385987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241317657947811:2114] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.397236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:19.397339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241315307969317:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-12-24T02:11:19.397356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241315307969218:2048] [node 1] ICP30 unregister session Session# [2:7587241315307969317:2048] VirtualId# [2:7587241315307969306:0]
2025-12-24T02:11:19.397363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241315307969317:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.397429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.397482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.397539Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:19.397616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241317657947835:2048] [node 1] ICS01 socket: 601 reason# EndOfStream
2025-12-24T02:11:19.397628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241317657947736:2048] [node 1] ICP30 unregister session Session# [3:7587241317657947835:2048] VirtualId# [3:7587241317657947820:0]
2025-12-24T02:11:19.397635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241317657947835:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.397686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.397709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.398275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:19.398352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319423903042:2048] [node 1] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:19.398363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319423902945:2048] [node 1] ICP30 unregister session Session# [4:7587241319423903042:2048] VirtualId# [4:7587241319423903026:0]
2025-12-24T02:11:19.398367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319423903042:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.398409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.398452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.398488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:19.398574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241319325605051:2048] [node 1] ICS07 socket disconnect 630 reason# EndOfStream
2025-12-24T02:11:19.398586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319325605051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.398647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.398677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241319325605051:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.398759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:19.398848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317544336230:2048] [node 1] ICS01 socket: 597 reason# EndOfStream
2025-12-24T02:11:19.398855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317544336127:2048] [node 1] ICP30 unregister session Session# [6:7587241317544336230:2048] VirtualId# [6:7587241317544336224:0]
2025-12-24T02:11:19.398861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317544336230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.398869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.398910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.399075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319325604950:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.399091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319325605051:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.399096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319325604950:2048] [node 1] ICP30 unregister session Session# [5:7587241319325605051:2048] VirtualId# [5:7587241319325605036:0]
2025-12-24T02:11:19.399127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.399134Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241319325604950:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.399137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61559) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.411792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:19.411880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241317657947812:2049] [node 2] ICS07 socket disconnect 607 reason# EndOfStream
2025-12-24T02:11:19.411887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241317657947812:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.411929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.411947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241317657947812:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:19.413188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:19.413270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319423903034:2049] [node 2] ICS01 socket: 610 reason# EndOfStream
2025-12-24T02:11:19.413277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319423902946:2049] [node 2] ICP30 unregister session Session# [4:7587241319423903034:2049] VirtualId# [4:7587241319423903024:0]
2025-12-24T02:11:19.413383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:19.413283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319423903034:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.413339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.413363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.413501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319325605053:2049] [node 2] ICS01 socket: 642 reason# EndOfStream
2025-12-24T02:11:19.413513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319325604951:2049] [node 2] ICP30 unregister session Session# [5:7587241319325605053:2049] VirtualId# [5:7587241319325605034:0]
2025-12-24T02:11:19.413517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319325605053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.413565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.413588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.413555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241317657947737:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:19.413570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241317657947812:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:19.413770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241319325604950:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.413780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:19.413849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241317544336217:2049] [node 2] ICS07 socket disconnect 621 reason# EndOfStream
2025-12-24T02:11:19.413854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317544336217:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.413891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.413908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241317544336217:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:19.414401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317544336128:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.414416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317544336217:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.414422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317544336128:2049] [node 2] ICP30 unregister session Session# [6:7587241317544336217:2049] VirtualId# [6:7587241317544336206:0]
2025-12-24T02:11:19.414438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.414445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317544336128:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.414447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29981) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.423622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241317544336128:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:19.431858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:19.431973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319423903040:2050] [node 3] ICS01 socket: 622 reason# EndOfStream
2025-12-24T02:11:19.431985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319423902947:2050] [node 3] ICP30 unregister session Session# [4:7587241319423903040:2050] VirtualId# [4:7587241319423903030:0]
2025-12-24T02:11:19.431992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319423903040:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.432045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.432078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.433633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:19.433723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317544336222:2050] [node 3] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:19.433736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317544336129:2050] [node 3] ICP30 unregister session Session# [6:7587241317544336222:2050] VirtualId# [6:7587241317544336209:0]
2025-12-24T02:11:19.433743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317544336222:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.433822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.433854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.438776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:19.438883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319325605040:2050] [node 3] ICS01 socket: 648 reason# EndOfStream
2025-12-24T02:11:19.438892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319325604952:2050] [node 3] ICP30 unregister session Session# [5:7587241319325605040:2050] VirtualId# [5:7587241319325605032:0]
2025-12-24T02:11:19.438899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319325605040:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.438975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.439007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.444543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:19.444648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241317544336215:2111] [node 4] ICS07 socket disconnect 631 reason# EndOfStream
2025-12-24T02:11:19.444661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317544336215:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.444731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.444766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241317544336215:2111] [node 4] ICS15 start handshake
2025-12-24T02:11:19.445110Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317544336195:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27424) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.445125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317544336215:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.445132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317544336195:2111] [node 4] ICP30 unregister session Session# [6:7587241317544336215:2111] VirtualId# [6:7587241317544336213:0]
2025-12-24T02:11:19.445154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.445162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317544336195:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.445165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27424) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.455375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241317544336195:2111] [node 4] ICP33 wake up from error state
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:19.484956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318534466016:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.485049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318534466015:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:19.486758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317488756676:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.486767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317488756677:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.488411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315969233550:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.488421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315969233551:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.490151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316786741218:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.490161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316786741220:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.490164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316786741219:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.491777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316157480613:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.491787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316157480614:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.491795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316157480615:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.493561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241318215010601:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.493570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241318215010602:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.493577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241318215010603:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.503562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241318215010663:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.504408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480674:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.504478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756736:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
2025-12-24T02:11:19.504596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233610:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
2025-12-24T02:11:19.503983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318534466077:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:19.504510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318534466087:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.506929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318534466091:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:19.507034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480674:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.512409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315969233617:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.513408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010671:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.515062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756736:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.515115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241317488756676:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.515197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756748:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
2025-12-24T02:11:19.515271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316786741286:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.517043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316157480676:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.519675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466080:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.519704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233610:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.526414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010675:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.528639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756751:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
2025-12-24T02:11:19.529335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756748:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.529467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318534466094:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17652
2025-12-24T02:11:19.529742Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466094:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.529216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241318215010671:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.530876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318534466095:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17652
2025-12-24T02:11:19.530915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317488756738:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.531079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466095:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.531338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318534466080:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.531933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241316786741218:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23608866ns) {Configure:104(23607727ns) {ProcessConfigured:135(23113840ns) {StartInitialHandshake:157(23113188ns) {DropHandshakes:470(371ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(722ns) {SwitchToState:179(61ns)}}}}}
2025-12-24T02:11:19.531968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241316157480615:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20234929ns) {Configure:104(20234070ns) {ProcessConfigured:135(19898188ns) {StartInitialHandshake:157(19897311ns) {DropHandshakes:470(349ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(727ns) {SwitchToState:179(284ns)}}}}}
2025-12-24T02:11:19.530781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241318215010672:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:19.532127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318534466080:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.533395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318534466015:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.533417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.533424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318534466015:2048] [node 2] ICP22 created new session: [1:7587241318534466099:2048]
2025-12-24T02:11:19.533431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318534466099:2048] [node 2] ICS09 handshake done sender: [1:7587241318534466080:2109] self: [1:7587241318534466076:0] peer: [2:7587241317488756747:0] socket: 601 qp: -1
2025-12-24T02:11:19.533443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318534466099:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:19.533451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.534986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241315969233550:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27266868ns) {Configure:104(27265932ns) {ProcessConfigured:135(26890678ns) {StartInitialHandshake:157(26890347ns) {DropHandshakes:470(600ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(896ns) {SwitchToState:179(45ns)}}}}}
2025-12-24T02:11:19.535021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241315969233550:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.535029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756755:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
2025-12-24T02:11:19.535068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.535198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480689:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.535312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318534466102:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17652
2025-12-24T02:11:19.535471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480689:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.536378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316157480681:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:19.535859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241318215010601:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25814989ns) {Configure:104(25814363ns) {ProcessConfigured:135(25483913ns) {StartInitialHandshake:157(25483563ns) {DropHandshakes:470(302ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:477(1229ns) {SwitchToState:179(693ns)}}}}}
2025-12-24T02:11:19.536751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741293:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.538388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466081:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.538402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318534466081:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.538499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466089:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.538507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318534466089:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.539304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315969233616:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.539633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241315969233615:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.539714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741293:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.539743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316786741218:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.539929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241316157480676:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22808068ns) {Configure:104(22807813ns) {ProcessConfigured:135(22528056ns) {StartInitialHandshake:157(22517195ns) {DropHandshakes:470(353ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(307ns) PrepareNewSessionHandshake:477(598ns) {SwitchToState:179(16ns)}} IncomingHandshake:261(3251ns) {DropIncomingHandshake:442(26ns) PrepareNewSessionHandshake:477(138ns) {SwitchToState:179(96ns)} GenerateSessionVirtualId:988(162ns) IssueIncomingHandshakeReply:193(198ns)}}}}
2025-12-24T02:11:19.536472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317488756754:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7002
2025-12-24T02:11:19.536552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241317488756676:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241317488756736:2107] held: no
2025-12-24T02:11:19.536590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317488756736:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.538747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317488756744:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:19.538850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756754:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.538858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241317488756676:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.538870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.538874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317488756676:2048] [node 1] ICP22 created new session: [2:7587241317488756757:2048]
2025-12-24T02:11:19.538880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317488756757:2048] [node 1] ICS09 handshake done sender: [2:7587241317488756736:2107] self: [2:7587241317488756747:0] peer: [1:7587241318534466076:0] socket: 605 qp: -1
2025-12-24T02:11:19.538887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317488756757:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.538904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.538927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317488756743:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.539852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466085:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.539859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318534466085:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.540321Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318534466085:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.540423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318534466077:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.540434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.540439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318534466077:2107] [node 4] ICP22 created new session: [1:7587241318534466104:2107]
2025-12-24T02:11:19.540443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318534466104:2107] [node 4] ICS09 handshake done sender: [1:7587241318534466085:2111] self: [1:7587241318534466084:0] peer: [4:7587241316786741295:0] socket: 606 qp: -1
2025-12-24T02:11:19.540447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318534466104:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:19.540453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.541721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466093:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.541726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318534466093:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.542125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318534466093:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.542212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318534466091:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.542224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.542230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318534466091:2113] [node 6] ICP22 created new session: [1:7587241318534466108:2113]
2025-12-24T02:11:19.542233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318534466108:2113] [node 6] ICS09 handshake done sender: [1:7587241318534466093:2114] self: [1:7587241318534466092:0] peer: [6:7587241318215010681:0] socket: 641 qp: -1
2025-12-24T02:11:19.542238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318534466108:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:19.542243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.540760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010680:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.541625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241318215010680:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.541651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241318215010601:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.542220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010682:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.542386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241318215010680:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.542445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241318215010601:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.542456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.542473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241318215010601:2048] [node 1] ICP22 created new session: [6:7587241318215010683:2048]
2025-12-24T02:11:19.542479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241318215010683:2048] [node 1] ICS09 handshake done sender: [6:7587241318215010680:2116] self: [6:7587241318215010681:0] peer: [1:7587241318534466092:0] socket: 642 qp: -1
2025-12-24T02:11:19.542485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241318215010683:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.542513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.547038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318534466110:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17652
2025-12-24T02:11:19.546826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741296:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.547269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480694:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.547310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233621:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.547322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315969233621:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.546868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756751:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.548103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241316786741284:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.548520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318534466102:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.548048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315969233621:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.548463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241315969233617:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.548476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.548483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315969233617:2110] [node 6] ICP22 created new session: [3:7587241315969233630:2110]
2025-12-24T02:11:19.548505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315969233630:2110] [node 6] ICS09 handshake done sender: [3:7587241315969233621:2112] self: [3:7587241315969233620:0] peer: [6:7587241318215010686:0] socket: 611 qp: -1
2025-12-24T02:11:19.548513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315969233630:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:19.548521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.549615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316157480613:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.546932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241318215010603:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34995670ns) {Configure:104(34994916ns) {ProcessConfigured:135(34602149ns) {StartInitialHandshake:157(34602049ns) {DropHandshakes:470(447ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(136ns) PrepareNewSessionHandshake:477(1070ns) {SwitchToState:179(132ns)}}}}}
2025-12-24T02:11:19.546975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241318215010603:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.552510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316786741293:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.553108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.554639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241318534466091:2113] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.554747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318534466089:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.554979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480696:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.555253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316157480689:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.555344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316157480613:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.555361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.555367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316157480613:2048] [node 1] ICP22 created new session: [5:7587241316157480697:2048]
2025-12-24T02:11:19.555373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316157480697:2048] [node 1] ICS09 handshake done sender: [5:7587241316157480689:2114] self: [5:7587241316157480690:0] peer: [1:7587241318534466088:0] socket: 637 qp: -1
2025-12-24T02:11:19.555379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316157480697:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.555400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.556209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316786741218:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.556241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.556249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316786741218:2048] [node 1] ICP22 created new session: [4:7587241316786741301:2048]
2025-12-24T02:11:19.556256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316786741301:2048] [node 1] ICS09 handshake done sender: [4:7587241316786741293:2113] self: [4:7587241316786741295:0] peer: [1:7587241318534466084:0] socket: 609 qp: -1
2025-12-24T02:11:19.556263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316786741301:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.556286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.557518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318534466087:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.557543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.557551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318534466087:2108] [node 5] ICP22 created new session: [1:7587241318534466114:2108]
2025-12-24T02:11:19.557558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318534466114:2108] [node 5] ICS09 handshake done sender: [1:7587241318534466089:2112] self: [1:7587241318534466088:0] peer: [5:7587241316157480690:0] socket: 634 qp: -1
2025-12-24T02:11:19.557564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318534466114:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.557573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.558801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241315969233616:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19421799ns) {Configure:104(19421490ns) {ProcessConfigured:135(19148188ns) {StartInitialHandshake:157(19147768ns) {DropHandshakes:470(145ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(648ns)}}}}}
2025-12-24T02:11:19.558936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233634:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.559015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233633:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.559053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233636:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.559212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241315969233615:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19530441ns) {Configure:104(19529003ns) {ProcessConfigured:135(19280318ns) {StartInitialHandshake:157(19280085ns) {DropHandshakes:470(473ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(392ns) {SwitchToState:179(58ns)}}}}}
2025-12-24T02:11:19.559290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233634:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.559336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480700:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.559363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741304:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.561037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241318215010675:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.561094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241318215010672:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.561192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741304:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.561239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741291:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.561248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241316786741291:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.561022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233633:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.561689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316786741220:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.561857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233635:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.561866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315969233635:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.562454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315969233635:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.562550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241315969233616:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.562570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.562578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315969233616:2108] [node 4] ICP22 created new session: [3:7587241315969233642:2108]
2025-12-24T02:11:19.562585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315969233642:2108] [node 4] ICS09 handshake done sender: [3:7587241315969233635:2117] self: [3:7587241315969233627:0] peer: [4:7587241316786741306:0] socket: 659 qp: -1
2025-12-24T02:11:19.562593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315969233642:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:19.562601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.562858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316786741291:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.563050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741297:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.563148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233632:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
2025-12-24T02:11:19.563600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318534466081:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.564582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241316786741286:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.564597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.564604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316786741286:2109] [node 6] ICP22 created new session: [4:7587241316786741307:2109]
2025-12-24T02:11:19.564610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316786741307:2109] [node 6] ICS09 handshake done sender: [4:7587241316786741291:2112] self: [4:7587241316786741287:0] peer: [6:7587241318215010688:0] socket: 629 qp: -1
2025-12-24T02:11:19.564616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316786741307:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.564622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.562967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756755:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.565534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241316786741220:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241316786741304:2119] held: no
2025-12-24T02:11:19.565599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741299:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.565608Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241316786741299:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.566325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316786741299:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.566406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241316786741284:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.566422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.566429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316786741284:2108] [node 5] ICP22 created new session: [4:7587241316786741310:2108]
2025-12-24T02:11:19.566439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316786741310:2108] [node 5] ICS09 handshake done sender: [4:7587241316786741299:2117] self: [4:7587241316786741298:0] peer: [5:7587241316157480701:0] socket: 619 qp: -1
2025-12-24T02:11:19.566445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316786741310:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.566455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.564805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480700:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.564849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316157480681:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.566756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233632:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.566795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241315969233551:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.567351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480694:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.567388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316157480614:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.567312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010689:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.567547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010690:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.568015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241318215010671:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.568143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741300:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.568177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241316786741286:2109] [node 6] ICP18 handshake (actor [4:7587241316786741300:2118]) is too old
2025-12-24T02:11:19.569162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241318215010690:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.569204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241318215010663:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.569446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241318215010672:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25198) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241318215010675:2113] held: no
2025-12-24T02:11:19.569941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241318215010603:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.569961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.569966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241318215010603:2050] [node 3] ICP22 created new session: [6:7587241318215010693:2050]
2025-12-24T02:11:19.569972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241318215010693:2050] [node 3] ICS09 handshake done sender: [6:7587241318215010671:2111] self: [6:7587241318215010686:0] peer: [3:7587241315969233620:0] socket: 612 qp: -1
2025-12-24T02:11:19.569977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241318215010693:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.569984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.567942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315969233610:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.568395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241315969233550:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.568406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.568411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315969233550:2048] [node 1] ICP22 created new session: [3:7587241315969233646:2048]
2025-12-24T02:11:19.568416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315969233646:2048] [node 1] ICS09 handshake done sender: [3:7587241315969233610:2107] self: [3:7587241315969233626:0] peer: [1:7587241318534466079:0] socket: 607 qp: -1
2025-12-24T02:11:19.568422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315969233646:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.568438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.571377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480692:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.571390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241316157480692:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.572508Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756760:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.572522Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317488756760:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.573429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317488756760:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.571358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241315969233617:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(22860224ns) {HandleHandshakeStatus:329(22859290ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(232ns) ProcessPendingSessionEvents:495(8192551ns) {ForwardSessionEventToSession:612(8180665ns) ForwardSessionEventToSession:612(2061ns) ForwardSessionEventToSession:612(882ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(488ns) ForwardSessionEventToSession:612(921ns) ForwardSessionEventToSession:612(680ns) ForwardSessionEventToSession:612(845ns) ForwardSessionEventToSession:612(719ns) ForwardSessionEventToSession:612(1192ns)}}}
2025-12-24T02:11:19.571081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741312:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.571543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316786741304:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.571599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316786741220:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.571609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.571616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316786741220:2050] [node 3] ICP22 created new session: [4:7587241316786741313:2050]
2025-12-24T02:11:19.571622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316786741313:2050] [node 3] ICS09 handshake done sender: [4:7587241316786741304:2119] self: [4:7587241316786741306:0] peer: [3:7587241315969233627:0] socket: 660 qp: -1
2025-12-24T02:11:19.571628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316786741313:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.571635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.572323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741297:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.572359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241316786741219:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.573323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741316:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.573616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241316786741297:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.573692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241316786741219:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.573703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.573708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241316786741219:2049] [node 2] ICP22 created new session: [4:7587241316786741317:2049]
2025-12-24T02:11:19.573714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241316786741317:2049] [node 2] ICS09 handshake done sender: [4:7587241316786741297:2116] self: [4:7587241316786741315:0] peer: [2:7587241317488756753:0] socket: 648 qp: -1
2025-12-24T02:11:19.573722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241316786741317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.573729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.572148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010695:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.574935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316157480692:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.575002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010691:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.575047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241318215010690:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.574564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317488756738:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.574583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.574589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317488756738:2108] [node 4] ICP22 created new session: [2:7587241317488756770:2108]
2025-12-24T02:11:19.574595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317488756770:2108] [node 4] ICS09 handshake done sender: [2:7587241317488756760:2118] self: [2:7587241317488756753:0] peer: [4:7587241316786741315:0] socket: 645 qp: -1
2025-12-24T02:11:19.574600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317488756770:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:19.574607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.575045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756762:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.575052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317488756762:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.575088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241318215010663:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.575100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.575105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241318215010663:2108] [node 5] ICP22 created new session: [6:7587241318215010696:2108]
2025-12-24T02:11:19.575112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241318215010696:2108] [node 5] ICS09 handshake done sender: [6:7587241318215010690:2120] self: [6:7587241318215010692:0] peer: [5:7587241316157480684:0] socket: 640 qp: -1
2025-12-24T02:11:19.575117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241318215010696:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.575144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.575380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241316157480676:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.575404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.575410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316157480676:2109] [node 6] ICP22 created new session: [5:7587241316157480704:2109]
2025-12-24T02:11:19.575418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316157480704:2109] [node 6] ICS09 handshake done sender: [5:7587241316157480692:2115] self: [5:7587241316157480684:0] peer: [6:7587241318215010692:0] socket: 658 qp: -1
2025-12-24T02:11:19.575436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316157480704:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.575455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.578733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756746:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.578758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317488756746:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.579461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318534466016:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.579502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.579511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318534466016:2049] [node 3] ICP22 created new session: [1:7587241318534466116:2049]
2025-12-24T02:11:19.579520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318534466116:2049] [node 3] ICS09 handshake done sender: [1:7587241318534466081:2110] self: [1:7587241318534466079:0] peer: [3:7587241315969233626:0] socket: 604 qp: -1
2025-12-24T02:11:19.579533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318534466116:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.579551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.583408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317488756762:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.583933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317488756743:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.583956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.583963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317488756743:2109] [node 5] ICP22 created new session: [2:7587241317488756774:2109]
2025-12-24T02:11:19.583969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317488756774:2109] [node 5] ICS09 handshake done sender: [2:7587241317488756762:2119] self: [2:7587241317488756761:0] peer: [5:7587241316157480702:0] socket: 646 qp: -1
2025-12-24T02:11:19.583975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317488756774:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:19.583983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.585441Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316157480614:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241316157480694:2116] held: no
2025-12-24T02:11:19.585506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480709:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.585661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316157480694:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.585713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316157480614:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.585722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.585728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316157480614:2049] [node 2] ICP22 created new session: [5:7587241316157480710:2049]
2025-12-24T02:11:19.585734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316157480710:2049] [node 2] ICS09 handshake done sender: [5:7587241316157480694:2116] self: [5:7587241316157480702:0] peer: [2:7587241317488756761:0] socket: 649 qp: -1
2025-12-24T02:11:19.585740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316157480710:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.585758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.591400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480708:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.591586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317488756746:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.591708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317488756677:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.591725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.591732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317488756677:2049] [node 3] ICP22 created new session: [2:7587241317488756777:2049]
2025-12-24T02:11:19.591737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317488756777:2049] [node 3] ICS09 handshake done sender: [2:7587241317488756746:2112] self: [2:7587241317488756742:0] peer: [3:7587241315969233645:0] socket: 616 qp: -1
2025-12-24T02:11:19.591741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317488756777:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.591747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.591687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316157480700:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.592118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480707:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
2025-12-24T02:11:19.592189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316157480681:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.592200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.592206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316157480681:2110] [node 4] ICP22 created new session: [5:7587241316157480712:2110]
2025-12-24T02:11:19.592212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316157480712:2110] [node 4] ICS09 handshake done sender: [5:7587241316157480700:2119] self: [5:7587241316157480701:0] peer: [4:7587241316786741298:0] socket: 661 qp: -1
2025-12-24T02:11:19.592218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316157480712:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:19.592226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.592913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241318534466016:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(28990995ns) {HandleHandshakeStatus:329(28990299ns) {DropIncomingHandshake:442(13154ns) DropOutgoingHandshake:457(251ns) SwitchToState:179(161ns) ProcessPendingSessionEvents:495(42650ns) {ForwardSessionEventToSession:612(31860ns) ForwardSessionEventToSession:612(776ns) ForwardSessionEventToSession:612(1015ns) ForwardSessionEventToSession:612(1229ns) ForwardSessionEventToSession:612(523ns) ForwardSessionEventToSession:612(1205ns) ForwardSessionEventToSession:612(742ns) ForwardSessionEventToSession:612(531ns) ForwardSessionEventToSession:612(673ns) ForwardSessionEventToSession:612(1055ns)}}}
2025-12-24T02:11:19.591336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241316786741319:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28258
2025-12-24T02:11:19.591821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241316786741319:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.591861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241316786741284:2108] [node 5] ICP18 handshake (actor [4:7587241316786741319:2122]) is too old
2025-12-24T02:11:19.591393Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241318215010603:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(21418707ns) {HandleHandshakeStatus:329(21417876ns) {DropIncomingHandshake:442(163ns) DropOutgoingHandshake:457(6902ns) SwitchToState:179(44ns) ProcessPendingSessionEvents:495(16362765ns) {ForwardSessionEventToSession:612(16345335ns) ForwardSessionEventToSession:612(1952ns) ForwardSessionEventToSession:612(1009ns) ForwardSessionEventToSession:612(513ns) ForwardSessionEventToSession:612(517ns) ForwardSessionEventToSession:612(974ns) ForwardSessionEventToSession:612(460ns) ForwardSessionEventToSession:612(4907ns) ForwardSessionEventToSession:612(617ns) ForwardSessionEventToSession:612(1066ns)}}}
2025-12-24T02:11:19.592890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241318215010663:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(17768576ns) {HandleHandshakeStatus:329(17768135ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(1357ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(13566708ns) {ForwardSessionEventToSession:612(13553971ns) ForwardSessionEventToSession:612(1599ns) ForwardSessionEventToSession:612(876ns) ForwardSessionEventToSession:612(611ns) ForwardSessionEventToSession:612(917ns) ForwardSessionEventToSession:612(955ns) ForwardSessionEventToSession:612(608ns) ForwardSessionEventToSession:612(529ns) ForwardSessionEventToSession:612(596ns) ForwardSessionEventToSession:612(943ns)}}}
2025-12-24T02:11:19.592928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241318215010675:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.593003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241318215010672:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.593011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.593018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241318215010672:2110] [node 4] ICP22 created new session: [6:7587241318215010698:2110]
2025-12-24T02:11:19.593023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241318215010698:2110] [node 4] ICS09 handshake done sender: [6:7587241318215010675:2113] self: [6:7587241318215010688:0] peer: [4:7587241316786741287:0] socket: 630 qp: -1
2025-12-24T02:11:19.593029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241318215010698:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:19.593036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.595492Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241316157480676:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20078558ns) {HandleHandshakeStatus:329(20077675ns) {DropIncomingHandshake:442(4524ns) DropOutgoingHandshake:457(121ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(15770823ns) {ForwardSessionEventToSession:612(15759131ns) ForwardSessionEventToSession:612(1523ns) ForwardSessionEventToSession:612(1327ns) ForwardSessionEventToSession:612(572ns) ForwardSessionEventToSession:612(632ns) ForwardSessionEventToSession:612(774ns) ForwardSessionEventToSession:612(538ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(627ns) ForwardSessionEventToSession:612(1088ns)}}}
2025-12-24T02:11:19.595697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316157480707:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.595737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316157480615:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.595991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241315969233638:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.596003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241315969233638:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.598867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315969233638:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.603163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316157480715:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54390
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.603478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316157480707:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.603606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316157480615:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.603631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.603645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316157480615:2050] [node 3] ICP22 created new session: [5:7587241316157480716:2050]
2025-12-24T02:11:19.603657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316157480716:2050] [node 3] ICS09 handshake done sender: [5:7587241316157480707:2120] self: [5:7587241316157480714:0] peer: [3:7587241315969233628:0] socket: 665 qp: -1
2025-12-24T02:11:19.603665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316157480716:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.603678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.603947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317488756766:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.603958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317488756766:2120] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.604583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317488756766:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.604658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317488756744:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.604682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.604692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317488756744:2110] [node 6] ICP22 created new session: [2:7587241317488756780:2110]
2025-12-24T02:11:19.604697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317488756780:2110] [node 6] ICS09 handshake done sender: [2:7587241317488756766:2120] self: [2:7587241317488756759:0] peer: [6:7587241318215010701:0] socket: 635 qp: -1
2025-12-24T02:11:19.604704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317488756780:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:19.604726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.602860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010700:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.603356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241318215010700:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.603412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241318215010602:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.604614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241318215010702:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49259
2025-12-24T02:11:19.606746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241318215010700:2123] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.603335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241315969233649:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7444
2025-12-24T02:11:19.603584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241315969233551:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241315969233632:2114] held: no
2025-12-24T02:11:19.603897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241315969233632:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.606780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241318215010602:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241318215010700:2123] held: no
2025-12-24T02:11:19.610797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241315969233615:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.610803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241315969233551:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.610821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.610828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315969233551:2049] [node 2] ICP22 created new session: [3:7587241315969233650:2049]
2025-12-24T02:11:19.610835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315969233650:2049] [node 2] ICS09 handshake done sender: [3:7587241315969233632:2114] self: [3:7587241315969233645:0] peer: [2:7587241317488756742:0] socket: 628 qp: -1
2025-12-24T02:11:19.610838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.610843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315969233650:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.610849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241315969233615:2109] [node 5] ICP22 created new session: [3:7587241315969233651:2109]
2025-12-24T02:11:19.610857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241315969233651:2109] [node 5] ICS09 handshake done sender: [3:7587241315969233638:2119] self: [3:7587241315969233628:0] peer: [5:7587241316157480714:0] socket: 652 qp: -1
2025-12-24T02:11:19.610857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.610866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241315969233651:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:19.610875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.619150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241318215010602:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.619179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.619192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241318215010602:2049] [node 2] ICP22 created new session: [6:7587241318215010703:2049]
2025-12-24T02:11:19.619198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241318215010703:2049] [node 2] ICS09 handshake done sender: [6:7587241318215010700:2123] self: [6:7587241318215010701:0] peer: [2:7587241317488756759:0] socket: 670 qp: -1
2025-12-24T02:11:19.619207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241318215010703:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.619235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.635098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:19.635193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241317488756757:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-12-24T02:11:19.635201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241317488756676:2048] [node 1] ICP30 unregister session Session# [2:7587241317488756757:2048] VirtualId# [2:7587241317488756747:0]
2025-12-24T02:11:19.635208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241317488756757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.635249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.635272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.638977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:19.639075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241315969233646:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:19.639084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241315969233550:2048] [node 1] ICP30 unregister session Session# [3:7587241315969233646:2048] VirtualId# [3:7587241315969233626:0]
2025-12-24T02:11:19.639091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241315969233646:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.639141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.639169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.640008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:19.640101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241316786741301:2048] [node 1] ICS07 socket disconnect 609 reason# EndOfStream
2025-12-24T02:11:19.640113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241316786741301:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.640164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.640189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241316786741301:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.641066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241316786741218:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.641083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241316786741301:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.641088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241316786741218:2048] [node 1] ICP30 unregister session Session# [4:7587241316786741301:2048] VirtualId# [4:7587241316786741295:0]
2025-12-24T02:11:19.641103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.641109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241316786741218:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.641112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.642822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:19.642901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241318215010683:2048] [node 1] ICS07 socket disconnect 642 reason# EndOfStream
2025-12-24T02:11:19.642907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241318215010683:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.642949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.642968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241318215010683:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:19.643322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241318215010601:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.643333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241318215010683:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.643338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241318215010601:2048] [node 1] ICP30 unregister session Session# [6:7587241318215010683:2048] VirtualId# [6:7587241318215010681:0]
2025-12-24T02:11:19.643356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.643362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241318215010601:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.643365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62532) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.643649Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:19.643725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316157480697:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:19.643737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316157480613:2048] [node 1] ICP30 unregister session Session# [5:7587241316157480697:2048] VirtualId# [5:7587241316157480690:0]
2025-12-24T02:11:19.643743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316157480697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.643794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.643825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.649423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:19.649523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241315969233650:2049] [node 2] ICS01 socket: 628 reason# EndOfStream
2025-12-24T02:11:19.649535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241315969233551:2049] [node 2] ICP30 unregister session Session# [3:7587241315969233650:2049] VirtualId# [3:7587241315969233645:0]
2025-12-24T02:11:19.649542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241315969233650:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.649597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.649620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.651980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241316786741218:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.654728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241318215010601:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:19.660065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:19.660177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241316786741317:2049] [node 2] ICS01 socket: 648 reason# EndOfStream
2025-12-24T02:11:19.660187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241316786741219:2049] [node 2] ICP30 unregister session Session# [4:7587241316786741317:2049] VirtualId# [4:7587241316786741315:0]
2025-12-24T02:11:19.660195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241316786741317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.660287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.660319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.662740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:19.662831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241318215010703:2049] [node 2] ICS01 socket: 670 reason# EndOfStream
2025-12-24T02:11:19.662840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241318215010602:2049] [node 2] ICP30 unregister session Session# [6:7587241318215010703:2049] VirtualId# [6:7587241318215010701:0]
2025-12-24T02:11:19.662846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241318215010703:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.662892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.662922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.662777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:19.662870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316157480710:2049] [node 2] ICS01 socket: 649 reason# EndOfStream
2025-12-24T02:11:19.662879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316157480614:2049] [node 2] ICP30 unregister session Session# [5:7587241316157480710:2049] VirtualId# [5:7587241316157480702:0]
2025-12-24T02:11:19.662886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316157480710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.662942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.662969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.680043Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:19.680151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241316786741313:2050] [node 3] ICS01 socket: 660 reason# EndOfStream
2025-12-24T02:11:19.680161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241316786741220:2050] [node 3] ICP30 unregister session Session# [4:7587241316786741313:2050] VirtualId# [4:7587241316786741306:0]
2025-12-24T02:11:19.680168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241316786741313:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.680229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.680259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.680584Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:19.680668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241318215010693:2050] [node 3] ICS07 socket disconnect 612 reason# EndOfStream
2025-12-24T02:11:19.680673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241318215010693:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.680722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.680744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241318215010693:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:19.681328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241318215010603:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.681342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241318215010693:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.681348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241318215010603:2050] [node 3] ICP30 unregister session Session# [6:7587241318215010693:2050] VirtualId# [6:7587241318215010686:0]
2025-12-24T02:11:19.681364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.681369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241318215010603:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.681372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.682791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:19.682877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241316157480716:2050] [node 3] ICS07 socket disconnect 665 reason# EndOfStream
2025-12-24T02:11:19.682883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316157480716:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.682942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.682967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241316157480716:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:19.683361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316157480615:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.683379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316157480716:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.683385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316157480615:2050] [node 3] ICP30 unregister session Session# [5:7587241316157480716:2050] VirtualId# [5:7587241316157480714:0]
2025-12-24T02:11:19.683409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.683416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241316157480615:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.683419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.691066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241318215010603:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:19.694731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:19.694747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241316157480615:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:19.694836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241318215010698:2110] [node 4] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:19.694850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241318215010672:2110] [node 4] ICP30 unregister session Session# [6:7587241318215010698:2110] VirtualId# [6:7587241318215010688:0]
2025-12-24T02:11:19.694858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241318215010698:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.694916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.694946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.694857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:19.694939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241316157480712:2110] [node 4] ICS07 socket disconnect 661 reason# EndOfStream
2025-12-24T02:11:19.694945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316157480712:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.695002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.695028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241316157480712:2110] [node 4] ICS15 start handshake
2025-12-24T02:11:19.695520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316157480681:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25198) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.695534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316157480712:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.695540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316157480681:2110] [node 4] ICP30 unregister session Session# [5:7587241316157480712:2110] VirtualId# [5:7587241316157480701:0]
2025-12-24T02:11:19.695555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.695560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241316157480681:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25198) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.695562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25198) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.718842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:19.718967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241318215010696:2108] [node 5] ICS01 socket: 640 reason# EndOfStream
2025-12-24T02:11:19.718985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241318215010663:2108] [node 5] ICP30 unregister session Session# [6:7587241318215010696:2108] VirtualId# [6:7587241318215010692:0]
2025-12-24T02:11:19.718993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241318215010696:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.719062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:19.719085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:19.737271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315746448450:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:19.737283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315746448451:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.738203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317402610287:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.738217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317402610288:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.739633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241316761286357:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.739649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241316761286358:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.741164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241318004148950:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.741178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241318004148951:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.741184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241318004148952:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.743692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316841285220:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.743706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316841285221:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.743710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316841285222:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.746600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241316880767731:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.746615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241316880767732:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.746619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241316880767733:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.769665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241316880767787:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:19.770419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610343:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
2025-12-24T02:11:19.771015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241316761286417:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:19.770776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285277:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.771385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316841285285:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.774996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241318004149013:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.775706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610344:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.778980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285291:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.779524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149018:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.782435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285291:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.782987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317402610348:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:19.786815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149019:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
2025-12-24T02:11:19.786898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149018:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.787065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241318004149022:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.794384Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241316880767732:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16682991ns) {Configure:104(16681979ns) {ProcessConfigured:135(16539780ns) {StartInitialHandshake:157(16539548ns) {DropHandshakes:470(138ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(842ns) {SwitchToState:179(343ns)}}}}}
2025-12-24T02:11:19.792129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317402610358:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:19.792244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317402610359:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:19.793163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610357:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
2025-12-24T02:11:19.794642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241316880767802:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:19.799911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315746448516:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.800087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315746448518:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.800183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241315746448519:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.807288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315746448517:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15685
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.810898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241316761286357:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19470349ns) {Configure:104(19468499ns) {ProcessConfigured:135(19009337ns) {StartInitialHandshake:157(19008667ns) {DropHandshakes:470(897ns) {DropIncomingHandshake:442(150ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(298ns) PrepareNewSessionHandshake:477(1356ns) {SwitchToState:179(794ns)}}}}}
2025-12-24T02:11:19.811256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241316761286428:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.811345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241316761286429:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:19.812353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286426:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
2025-12-24T02:11:19.812374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286427:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.816988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767803:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.819059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285277:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.819277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241316841285295:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:19.819416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316841285285:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.820273Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241318004149022:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33151393ns) {Configure:104(33150340ns) {ProcessConfigured:135(32828843ns) {StartInitialHandshake:157(32828614ns) {DropHandshakes:470(1061ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1146ns) {SwitchToState:179(67ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.828454Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286438:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
No RDMA context found for GID: 2025-12-24T02:11:19.829941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448517:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:19.830072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315746448530:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15685
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.832171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149015:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.832189Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241318004149015:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.833028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241318004148952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.833325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149025:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
2025-12-24T02:11:19.833341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149026:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.833858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285302:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.833899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767810:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.833907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286438:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.834141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149019:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.834202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241318004149015:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.834217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448530:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.834347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286419:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.834355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241316761286419:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.834782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286427:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.834912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241316761286357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.835166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610343:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.835363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241316761286419:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.838997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316841285291:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.839117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149026:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.839134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241318004149013:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.839148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.839158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241318004149013:2107] [node 5] ICP22 created new session: [4:7587241318004149034:2107]
2025-12-24T02:11:19.839164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241318004149034:2107] [node 5] ICS09 handshake done sender: [4:7587241318004149015:2109] self: [4:7587241318004149014:0] peer: [5:7587241316841285296:0] socket: 613 qp: -1
2025-12-24T02:11:19.839171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241318004149034:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:19.839183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.840675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241318004149013:2107] [node 5] ICP18 handshake (actor [4:7587241318004149019:2111]) is too old
2025-12-24T02:11:19.841012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448521:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.841031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315746448521:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.842225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610344:2104] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.842383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610357:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241317402610287:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.842517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610369:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
2025-12-24T02:11:19.842412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316880767803:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610370:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
2025-12-24T02:11:19.842453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241316880767732:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.842901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316880767810:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241316880767802:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.842715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241318004148950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.842957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149025:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149024:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241318004149024:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.843499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149036:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
2025-12-24T02:11:19.843955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610370:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.842061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241316761286417:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.842078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.842084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241316761286417:2106] [node 4] ICP22 created new session: [3:7587241316761286442:2106]
2025-12-24T02:11:19.842090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241316761286442:2106] [node 4] ICS09 handshake done sender: [3:7587241316761286419:2107] self: [3:7587241316761286418:0] peer: [4:7587241318004149029:0] socket: 616 qp: -1
2025-12-24T02:11:19.842096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241316761286442:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:19.842104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.841653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315746448521:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.841733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315746448451:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.841749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.841757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315746448451:2049] [node 3] ICP22 created new session: [1:7587241315746448534:2049]
2025-12-24T02:11:19.841763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315746448534:2049] [node 3] ICS09 handshake done sender: [1:7587241315746448521:2112] self: [1:7587241315746448520:0] peer: [3:7587241316761286441:0] socket: 632 qp: -1
2025-12-24T02:11:19.841770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315746448534:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.841777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.844265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286426:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.845111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610356:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.845128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317402610356:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.845204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448511:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.845216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315746448511:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.845698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448523:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.845702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315746448523:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.848245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610369:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.848455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317402610375:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65129
2025-12-24T02:11:19.848622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316841285285:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.848646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.848653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316841285285:2108] [node 4] ICP22 created new session: [5:7587241316841285303:2108]
2025-12-24T02:11:19.848659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316841285303:2108] [node 4] ICS09 handshake done sender: [5:7587241316841285291:2113] self: [5:7587241316841285296:0] peer: [4:7587241318004149014:0] socket: 618 qp: -1
2025-12-24T02:11:19.848665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316841285303:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:19.848674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.848752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241316841285285:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.849020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285305:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.849231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767813:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.849328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317402610356:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.849425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317402610348:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.849444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.849450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317402610348:2107] [node 6] ICP22 created new session: [2:7587241317402610378:2107]
2025-12-24T02:11:19.849455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317402610378:2107] [node 6] ICS09 handshake done sender: [2:7587241317402610356:2113] self: [2:7587241317402610355:0] peer: [6:7587241316880767811:0] socket: 627 qp: -1
2025-12-24T02:11:19.849461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317402610378:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:19.849468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.849562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316880767803:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.849747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241316880767732:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.849762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.849767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241316880767732:2049] [node 2] ICP22 created new session: [6:7587241316880767814:2049]
2025-12-24T02:11:19.849772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316880767814:2049] [node 2] ICS09 handshake done sender: [6:7587241316880767803:2113] self: [6:7587241316880767811:0] peer: [2:7587241317402610355:0] socket: 630 qp: -1
2025-12-24T02:11:19.849782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316880767814:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.849787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.853190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241318004149018:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.855048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241318004148952:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.855072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.855081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241318004148952:2050] [node 3] ICP22 created new session: [4:7587241318004149041:2050]
2025-12-24T02:11:19.855087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241318004149041:2050] [node 3] ICS09 handshake done sender: [4:7587241318004149018:2110] self: [4:7587241318004149029:0] peer: [3:7587241316761286418:0] socket: 617 qp: -1
2025-12-24T02:11:19.855093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241318004149041:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.855101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.855737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315746448511:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.855630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241316761286357:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241316761286427:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.856146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286444:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.856380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317402610343:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.856465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241317402610287:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.856504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.856516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317402610287:2048] [node 1] ICP22 created new session: [2:7587241317402610381:2048]
2025-12-24T02:11:19.856523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317402610381:2048] [node 1] ICS09 handshake done sender: [2:7587241317402610343:2103] self: [2:7587241317402610371:0] peer: [1:7587241315746448510:0] socket: 607 qp: -1
2025-12-24T02:11:19.856530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317402610381:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.856537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.856324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286445:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
2025-12-24T02:11:19.856405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241316761286358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.856490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241316761286427:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.856763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767817:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.856664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241316761286357:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.856678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.856683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241316761286357:2048] [node 1] ICP22 created new session: [3:7587241316761286450:2048]
2025-12-24T02:11:19.856688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241316761286450:2048] [node 1] ICS09 handshake done sender: [3:7587241316761286427:2113] self: [3:7587241316761286441:0] peer: [1:7587241315746448520:0] socket: 635 qp: -1
2025-12-24T02:11:19.856693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241316761286450:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.856698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.856839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286447:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.858327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285307:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.858498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610354:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.858512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317402610354:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.859236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285308:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.859394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315746448523:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.859652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315746448450:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.859668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.859675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315746448450:2048] [node 2] ICP22 created new session: [1:7587241315746448540:2048]
2025-12-24T02:11:19.859681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315746448540:2048] [node 2] ICS09 handshake done sender: [1:7587241315746448511:2107] self: [1:7587241315746448510:0] peer: [2:7587241317402610371:0] socket: 606 qp: -1
2025-12-24T02:11:19.859687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315746448540:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:19.859693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.859829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315746448516:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.859834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.859838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315746448516:2108] [node 4] ICP22 created new session: [1:7587241315746448542:2108]
2025-12-24T02:11:19.859843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315746448542:2108] [node 4] ICS09 handshake done sender: [1:7587241315746448523:2113] self: [1:7587241315746448522:0] peer: [4:7587241318004149038:0] socket: 633 qp: -1
2025-12-24T02:11:19.859846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315746448542:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:19.859851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.859981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315746448539:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15685
2025-12-24T02:11:19.860279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285305:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.860308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316841285220:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.860467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317402610354:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.860636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317402610288:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.860655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.860661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317402610288:2049] [node 3] ICP22 created new session: [2:7587241317402610385:2049]
2025-12-24T02:11:19.860666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317402610385:2049] [node 3] ICS09 handshake done sender: [2:7587241317402610354:2112] self: [2:7587241317402610349:0] peer: [3:7587241316761286448:0] socket: 626 qp: -1
2025-12-24T02:11:19.860678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317402610385:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.860685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.862256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241315746448450:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.862334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448525:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.862338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315746448525:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.862879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286447:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.863073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241316761286452:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53618
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.863148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767818:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.863174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285307:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.863196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316841285222:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.863523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149039:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.863767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767819:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.865202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149040:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
2025-12-24T02:11:19.865283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241318004149024:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.865504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316880767810:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.865866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241318004149022:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.865878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.865886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241318004149022:2113] [node 6] ICP22 created new session: [4:7587241318004149044:2113]
2025-12-24T02:11:19.865891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241318004149044:2113] [node 6] ICS09 handshake done sender: [4:7587241318004149024:2114] self: [4:7587241318004149023:0] peer: [6:7587241316880767812:0] socket: 645 qp: -1
2025-12-24T02:11:19.865898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241318004149044:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:19.865906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.866044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241318004149026:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.866352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315746448525:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.865811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241316880767802:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.865826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.865842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241316880767802:2110] [node 4] ICP22 created new session: [6:7587241316880767820:2110]
2025-12-24T02:11:19.865854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316880767820:2110] [node 4] ICS09 handshake done sender: [6:7587241316880767810:2116] self: [6:7587241316880767812:0] peer: [4:7587241318004149023:0] socket: 647 qp: -1
2025-12-24T02:11:19.865861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316880767820:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:19.865870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.866641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285308:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.866674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285311:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.866855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241318004149040:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.866867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241318004148950:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.866875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.866880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241318004148950:2048] [node 1] ICP22 created new session: [4:7587241318004149046:2048]
2025-12-24T02:11:19.866885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241318004149046:2048] [node 1] ICS09 handshake done sender: [4:7587241318004149026:2116] self: [4:7587241318004149038:0] peer: [1:7587241315746448522:0] socket: 643 qp: -1
2025-12-24T02:11:19.866890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241318004149046:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.866896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.867096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241318004148951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.867262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241318004149022:2113] [node 6] ICP18 handshake (actor [4:7587241318004149040:2121]) is too old
2025-12-24T02:11:19.866972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241316841285221:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.866991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316841285305:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.867536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315746448518:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.867552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.867558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315746448518:2109] [node 5] ICP22 created new session: [1:7587241315746448546:2109]
2025-12-24T02:11:19.867563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315746448546:2109] [node 5] ICS09 handshake done sender: [1:7587241315746448525:2114] self: [1:7587241315746448524:0] peer: [5:7587241316841285309:0] socket: 642 qp: -1
2025-12-24T02:11:19.867569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315746448546:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:19.867575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.868082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241316761286358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27134) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241316761286426:2112] held: no
2025-12-24T02:11:19.868143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241316761286417:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.868155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241316761286426:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.867618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316841285220:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.867637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.867644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316841285220:2048] [node 1] ICP22 created new session: [5:7587241316841285313:2048]
2025-12-24T02:11:19.867649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316841285313:2048] [node 1] ICS09 handshake done sender: [5:7587241316841285305:2118] self: [5:7587241316841285309:0] peer: [1:7587241315746448524:0] socket: 621 qp: -1
2025-12-24T02:11:19.867655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316841285313:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.867661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.868892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610361:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.868904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317402610361:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.869255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317402610365:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.869264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317402610365:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.869836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317402610361:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:19.869867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241318004149049:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59157
2025-12-24T02:11:19.870202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317402610365:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.870568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241318004149025:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.870571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767823:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.870498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317402610358:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.870516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:19.870523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317402610358:2110] [node 4] ICP22 created new session: [2:7587241317402610389:2110]
2025-12-24T02:11:19.870533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317402610389:2110] [node 4] ICS09 handshake done sender: [2:7587241317402610361:2115] self: [2:7587241317402610360:0] peer: [4:7587241318004149048:0] socket: 628 qp: -1
2025-12-24T02:11:19.870539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317402610389:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:19.870548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:19.870646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317402610359:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.870653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.870657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317402610359:2111] [node 5] ICP22 created new session: [2:7587241317402610391:2111]
2025-12-24T02:11:19.870660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317402610391:2111] [node 5] ICS09 handshake done sender: [2:7587241317402610365:2117] self: [2:7587241317402610364:0] peer: [5:7587241316841285312:0] socket: 667 qp: -1
2025-12-24T02:11:19.870663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317402610391:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:19.870668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.870953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316880767818:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.870983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241316880767731:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.871090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241318004148951:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.871106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.871111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241318004148951:2049] [node 2] ICP22 created new session: [4:7587241318004149050:2049]
2025-12-24T02:11:19.871116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241318004149050:2049] [node 2] ICS09 handshake done sender: [4:7587241318004149025:2115] self: [4:7587241318004149048:0] peer: [2:7587241317402610360:0] socket: 640 qp: -1
2025-12-24T02:11:19.871121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241318004149050:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.871128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.872295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.872609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316841285308:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.872813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316880767817:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.872840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241316880767733:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.872959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448527:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.872972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241315746448527:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.872667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316841285221:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.872680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.872687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316841285221:2049] [node 2] ICP22 created new session: [5:7587241316841285317:2049]
2025-12-24T02:11:19.872693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316841285317:2049] [node 2] ICS09 handshake done sender: [5:7587241316841285308:2120] self: [5:7587241316841285312:0] peer: [2:7587241317402610364:0] socket: 668 qp: -1
2025-12-24T02:11:19.872699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316841285317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.872706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.872250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286433:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.872265Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241316761286433:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.872593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241316761286358:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.872602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.872608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241316761286358:2049] [node 2] ICP22 created new session: [3:7587241316761286454:2049]
2025-12-24T02:11:19.872612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241316761286454:2049] [node 2] ICS09 handshake done sender: [3:7587241316761286426:2112] self: [3:7587241316761286448:0] peer: [2:7587241317402610349:0] socket: 634 qp: -1
2025-12-24T02:11:19.872618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241316761286454:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.872627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.872947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241316761286435:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.872950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241316761286435:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.873384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241316761286433:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.873917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241315746448548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.873977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241316841285221:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:19.874008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241316841285319:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3594
2025-12-24T02:11:19.874162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241316761286428:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.874185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.874191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241316761286428:2109] [node 5] ICP22 created new session: [3:7587241316761286458:2109]
2025-12-24T02:11:19.874197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241316761286458:2109] [node 5] ICS09 handshake done sender: [3:7587241316761286433:2115] self: [3:7587241316761286432:0] peer: [5:7587241316841285310:0] socket: 663 qp: -1
2025-12-24T02:11:19.874202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241316761286458:2109] [node 5] ICS10 traffic start
No RDMA context found for GID: 2025-12-24T02:11:19.874208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
::ffff:127.1.0.0
2025-12-24T02:11:19.874197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241315746448527:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.874136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241316880767823:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.874225Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767826:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.874193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316841285307:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.874270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241316841285222:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.874280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.874285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316841285222:2050] [node 3] ICP22 created new session: [5:7587241316841285320:2050]
2025-12-24T02:11:19.874291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316841285320:2050] [node 3] ICS09 handshake done sender: [5:7587241316841285307:2119] self: [5:7587241316841285310:0] peer: [3:7587241316761286432:0] socket: 666 qp: -1
2025-12-24T02:11:19.874296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316841285320:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.874302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.874454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241315746448548:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.874470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241315746448519:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.874477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.874485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241315746448519:2110] [node 6] ICP22 created new session: [1:7587241315746448550:2110]
2025-12-24T02:11:19.874494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241315746448550:2110] [node 6] ICS09 handshake done sender: [1:7587241315746448527:2115] self: [1:7587241315746448526:0] peer: [6:7587241316880767824:0] socket: 669 qp: -1
2025-12-24T02:11:19.874500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241315746448550:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:19.874512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.874462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241316880767787:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.874508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316880767818:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.874673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241315746448519:2110] [node 6] ICP18 handshake (actor [1:7587241315746448548:2118]) is too old
2025-12-24T02:11:19.875134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241316761286435:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.875223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241316841285300:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.875234Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241316841285300:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.875134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241316880767731:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.875146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.875150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241316880767731:2048] [node 1] ICP22 created new session: [6:7587241316880767829:2048]
2025-12-24T02:11:19.875156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316880767829:2048] [node 1] ICS09 handshake done sender: [6:7587241316880767818:2119] self: [6:7587241316880767824:0] peer: [1:7587241315746448526:0] socket: 671 qp: -1
2025-12-24T02:11:19.875160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316880767829:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.875167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.875238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241316761286429:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.875256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.875261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241316761286429:2110] [node 6] ICP22 created new session: [3:7587241316761286460:2110]
2025-12-24T02:11:19.875266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241316761286460:2110] [node 6] ICS09 handshake done sender: [3:7587241316761286435:2116] self: [3:7587241316761286434:0] peer: [6:7587241316880767825:0] socket: 664 qp: -1
2025-12-24T02:11:19.875271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241316761286460:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:19.875278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.875823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241316841285300:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.876013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241316841285295:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.876031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.876041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241316841285295:2114] [node 6] ICP22 created new session: [5:7587241316841285323:2114]
2025-12-24T02:11:19.876047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241316841285323:2114] [node 6] ICS09 handshake done sender: [5:7587241316841285300:2116] self: [5:7587241316841285299:0] peer: [6:7587241316880767827:0] socket: 619 qp: -1
2025-12-24T02:11:19.876053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241316841285323:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:19.876060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.876840Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241316880767733:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29393) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241316880767817:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.877570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767828:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.877737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316880767817:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.877791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241316880767733:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.877802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.877806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241316880767733:2050] [node 3] ICP22 created new session: [6:7587241316880767832:2050]
2025-12-24T02:11:19.877809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316880767832:2050] [node 3] ICS09 handshake done sender: [6:7587241316880767817:2118] self: [6:7587241316880767825:0] peer: [3:7587241316761286434:0] socket: 665 qp: -1
2025-12-24T02:11:19.877813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316880767832:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:19.877817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:19.877951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241316880767787:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2574) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241316880767823:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.878010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241316880767831:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9547
2025-12-24T02:11:19.878136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241316880767823:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.878188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241316880767787:2102] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.878195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.878200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241316880767787:2102] [node 5] ICP22 created new session: [6:7587241316880767834:2102]
2025-12-24T02:11:19.878206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241316880767834:2102] [node 5] ICS09 handshake done sender: [6:7587241316880767823:2121] self: [6:7587241316880767827:0] peer: [5:7587241316841285299:0] socket: 662 qp: -1
2025-12-24T02:11:19.878212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241316880767834:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:19.878230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.884246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:19.884342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241316761286450:2048] [node 1] ICS01 socket: 635 reason# EndOfStream
2025-12-24T02:11:19.884356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241316761286357:2048] [node 1] ICP30 unregister session Session# [3:7587241316761286450:2048] VirtualId# [3:7587241316761286441:0]
2025-12-24T02:11:19.884364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241316761286450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.884435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.884467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.884362Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:19.884447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241317402610381:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:19.884455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241317402610287:2048] [node 1] ICP30 unregister session Session# [2:7587241317402610381:2048] VirtualId# [2:7587241317402610371:0]
2025-12-24T02:11:19.884460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241317402610381:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.884507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.884533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.885455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:19.885548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316841285313:2048] [node 1] ICS01 socket: 621 reason# EndOfStream
2025-12-24T02:11:19.885554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316841285220:2048] [node 1] ICP30 unregister session Session# [5:7587241316841285313:2048] VirtualId# [5:7587241316841285309:0]
2025-12-24T02:11:19.885561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316841285313:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.885601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.885620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.885462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:19.885560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241318004149046:2048] [node 1] ICS01 socket: 643 reason# EndOfStream
2025-12-24T02:11:19.885565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241318004148950:2048] [node 1] ICP30 unregister session Session# [4:7587241318004149046:2048] VirtualId# [4:7587241318004149038:0]
2025-12-24T02:11:19.885570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241318004149046:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.885626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.885647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.885782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:19.885826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316880767829:2048] [node 1] ICS01 socket: 671 reason# EndOfStream
2025-12-24T02:11:19.885830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241316880767731:2048] [node 1] ICP30 unregister session Session# [6:7587241316880767829:2048] VirtualId# [6:7587241316880767824:0]
2025-12-24T02:11:19.885834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316880767829:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.885864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:19.885890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:19.891476Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:19.891567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241316761286454:2049] [node 2] ICS01 socket: 634 reason# EndOfStream
2025-12-24T02:11:19.891577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241316761286358:2049] [node 2] ICP30 unregister session Session# [3:7587241316761286454:2049] VirtualId# [3:7587241316761286448:0]
2025-12-24T02:11:19.891582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241316761286454:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.891666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.891696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.892348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:19.892412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316880767814:2049] [node 2] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:19.892421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241316880767732:2049] [node 2] ICP30 unregister session Session# [6:7587241316880767814:2049] VirtualId# [6:7587241316880767811:0]
2025-12-24T02:11:19.892424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316880767814:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.892468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.892494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.892786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:19.892832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241316841285317:2049] [node 2] ICS07 socket disconnect 668 reason# EndOfStream
2025-12-24T02:11:19.892836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316841285317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.892864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.892880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241316841285317:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:19.893168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316841285221:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27134) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.893179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316841285317:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.893183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316841285221:2049] [node 2] ICP30 unregister session Session# [5:7587241316841285317:2049] VirtualId# [5:7587241316841285312:0]
2025-12-24T02:11:19.893194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.893200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241316841285221:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.893202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27134) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.894259Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:19.894344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241318004149050:2049] [node 2] ICS01 socket: 640 reason# EndOfStream
2025-12-24T02:11:19.894351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241318004148951:2049] [node 2] ICP30 unregister session Session# [4:7587241318004149050:2049] VirtualId# [4:7587241318004149048:0]
2025-12-24T02:11:19.894356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241318004149050:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.894407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:19.894432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:19.897905Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:19.897973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241316841285320:2050] [node 3] ICS07 socket disconnect 666 reason# EndOfStream
2025-12-24T02:11:19.897978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316841285320:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.898008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.898028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241316841285320:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:19.898335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241316841285222:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29393) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:19.898345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316841285320:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:19.898350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316841285222:2050] [node 3] ICP30 unregister session Session# [5:7587241316841285320:2050] VirtualId# [5:7587241316841285310:0]
2025-12-24T02:11:19.898361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.898366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241316841285222:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.898369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29393) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:19.898749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:19.898825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241318004149041:2050] [node 3] ICS01 socket: 617 reason# EndOfStream
2025-12-24T02:11:19.898832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241318004148952:2050] [node 3] ICP30 unregister session Session# [4:7587241318004149041:2050] VirtualId# [4:7587241318004149029:0]
2025-12-24T02:11:19.898837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241318004149041:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.898872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.898899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.899562Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:19.899641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316880767832:2050] [node 3] ICS01 socket: 665 reason# EndOfStream
2025-12-24T02:11:19.899651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241316880767733:2050] [node 3] ICP30 unregister session Session# [6:7587241316880767832:2050] VirtualId# [6:7587241316880767825:0]
2025-12-24T02:11:19.899656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316880767832:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.899702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:19.899723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:19.902845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241316841285221:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:19.907958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241316841285222:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:19.913083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:19.913180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241316841285303:2108] [node 4] ICS01 socket: 618 reason# EndOfStream
2025-12-24T02:11:19.913192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241316841285285:2108] [node 4] ICP30 unregister session Session# [5:7587241316841285303:2108] VirtualId# [5:7587241316841285296:0]
2025-12-24T02:11:19.913200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241316841285303:2108] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.913244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.913279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.913347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:19.913446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316880767820:2110] [node 4] ICS01 socket: 647 reason# EndOfStream
2025-12-24T02:11:19.913460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241316880767802:2110] [node 4] ICP30 unregister session Session# [6:7587241316880767820:2110] VirtualId# [6:7587241316880767812:0]
2025-12-24T02:11:19.913467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316880767820:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.913519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:19.913553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:19.918840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:19.918939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241316880767834:2102] [node 5] ICS01 socket: 662 reason# EndOfStream
2025-12-24T02:11:19.918953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241316880767787:2102] [node 5] ICP30 unregister session Session# [6:7587241316880767834:2102] VirtualId# [6:7587241316880767827:0]
2025-12-24T02:11:19.918960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241316880767834:2102] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:19.919019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:19.919060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:19.932568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318395223303:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:19.932589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318395223304:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.934074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317184441555:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.934090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317184441556:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:19.935470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241318158222350:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.935479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241318158222351:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.937362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241317775401373:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.937371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241317775401374:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.937378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241317775401375:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.939236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241318767245275:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.939244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241318767245276:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.939249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241318767245277:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.942170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317670433116:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:19.942186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317670433117:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:19.942195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317670433118:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:19.952211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317670433178:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.952961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245335:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.955147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317184441615:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
2025-12-24T02:11:19.956775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318395223370:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.956899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318395223371:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.957000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241318395223372:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:19.960287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241318767245335:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.960507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241318767245337:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:19.960992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241318767245337:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.961542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317670433180:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.961561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241317670433180:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.962741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317184441620:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:19.962906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317184441621:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:19.962998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241317184441622:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:19.963596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241318158222416:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.963683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241318158222417:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.964295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241318158222423:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.964366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241318158222418:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.964951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317670433182:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:19.965386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241317775401439:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:19.965466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241317775401440:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:19.968259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317670433180:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.968375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241317670433178:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.968388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.968395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317670433178:2108] [node 5] ICP22 created new session: [6:7587241317670433184:2108]
2025-12-24T02:11:19.968399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241317670433184:2108] [node 5] ICS09 handshake done sender: [6:7587241317670433180:2109] self: [6:7587241317670433179:0] peer: [5:7587241318767245338:0] socket: 613 qp: -1
2025-12-24T02:11:19.968405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241317670433184:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.968413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.969078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441615:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.969325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317184441632:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.970804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317184441631:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
2025-12-24T02:11:19.971007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241317184441555:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.970944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241318767245349:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:19.971132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245350:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:19.971146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245351:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:19.971475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241318767245335:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:19.971670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241318767245337:2108] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.971680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.971685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241318767245337:2108] [node 6] ICP22 created new session: [5:7587241318767245356:2108]
2025-12-24T02:11:19.971691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241318767245356:2108] [node 6] ICS09 handshake done sender: [5:7587241318767245335:2107] self: [5:7587241318767245338:0] peer: [6:7587241317670433179:0] socket: 615 qp: -1
2025-12-24T02:11:19.971698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241318767245356:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:19.971706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.972039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318395223383:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1654
2025-12-24T02:11:19.972087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241318395223382:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1654
2025-12-24T02:11:19.972248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223364:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.972259Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318395223364:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.971647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441631:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.973562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318395223364:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.973680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223382:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.973694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318395223303:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.973703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.973711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318395223303:2048] [node 2] ICP22 created new session: [1:7587241318395223385:2048]
2025-12-24T02:11:19.973721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318395223385:2048] [node 2] ICS09 handshake done sender: [1:7587241318395223364:2107] self: [1:7587241318395223363:0] peer: [2:7587241317184441636:0] socket: 612 qp: -1
2025-12-24T02:11:19.973726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318395223385:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:19.973734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.974719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:19.974772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222418:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.974811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241318158222350:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.974897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241318158222430:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:19.975157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245352:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:19.975343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222430:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.975479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241318767245350:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.975527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241318767245277:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.976531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223374:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.976541Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318395223374:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.977232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318395223374:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:19.977369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318395223304:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.977384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:19.977391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318395223304:2049] [node 3] ICP22 created new session: [1:7587241318395223390:2049]
2025-12-24T02:11:19.977396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241318395223390:2049] [node 3] ICS09 handshake done sender: [1:7587241318395223374:2111] self: [1:7587241318395223369:0] peer: [3:7587241318158222431:0] socket: 618 qp: -1
2025-12-24T02:11:19.977402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241318395223390:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:19.977409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.979140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317184441639:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
2025-12-24T02:11:19.979152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241318767245352:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.979183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241318767245276:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.979358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441628:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.979368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317184441628:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.979748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317184441615:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.979836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241317184441555:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.979853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.979860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317184441555:2048] [node 1] ICP22 created new session: [2:7587241317184441641:2048]
2025-12-24T02:11:19.979866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317184441641:2048] [node 1] ICS09 handshake done sender: [2:7587241317184441615:2107] self: [2:7587241317184441636:0] peer: [1:7587241318395223363:0] socket: 617 qp: -1
2025-12-24T02:11:19.979873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317184441641:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.979881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.980208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317184441628:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.980428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245361:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:19.981690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241318767245352:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:19.981784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241318767245276:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.981803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:19.981809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241318767245276:2049] [node 2] ICP22 created new session: [5:7587241318767245362:2049]
2025-12-24T02:11:19.981815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241318767245362:2049] [node 2] ICS09 handshake done sender: [5:7587241318767245352:2116] self: [5:7587241318767245360:0] peer: [2:7587241317184441627:0] socket: 639 qp: -1
2025-12-24T02:11:19.981822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241318767245362:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:19.981831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:19.986819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317184441621:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.986853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.986865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317184441621:2109] [node 5] ICP22 created new session: [2:7587241317184441643:2109]
2025-12-24T02:11:19.986873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241317184441643:2109] [node 5] ICS09 handshake done sender: [2:7587241317184441628:2113] self: [2:7587241317184441627:0] peer: [5:7587241318767245360:0] socket: 635 qp: -1
2025-12-24T02:11:19.986882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241317184441643:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:19.986892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:19.988043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222420:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.988061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241318158222420:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.991235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241317670433178:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(22836330ns) {HandleHandshakeStatus:329(22835324ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(300ns) SwitchToState:179(650ns) ProcessPendingSessionEvents:495(15328950ns) {ForwardSessionEventToSession:612(15323973ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.991488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241318158222433:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:19.991782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241318158222418:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:19.991856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241318158222350:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:19.991875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:19.991883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241318158222350:2048] [node 1] ICP22 created new session: [3:7587241318158222434:2048]
2025-12-24T02:11:19.991889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241318158222434:2048] [node 1] ICS09 handshake done sender: [3:7587241318158222418:2111] self: [3:7587241318158222431:0] peer: [1:7587241318395223369:0] socket: 621 qp: -1
2025-12-24T02:11:19.991896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241318158222434:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:19.991905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:19.991414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241317670433188:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:19.992432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317670433182:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.992461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317670433118:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.993072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317670433196:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:19.993121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317670433197:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:19.993350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317670433196:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.993728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222422:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.993745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241318158222422:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.994296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241318158222422:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.994499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241318158222437:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:19.994565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241318158222417:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.994585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:19.994592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241318158222417:2109] [node 6] ICP22 created new session: [3:7587241318158222439:2109]
2025-12-24T02:11:19.994600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241318158222439:2109] [node 6] ICS09 handshake done sender: [3:7587241318158222422:2113] self: [3:7587241318158222421:0] peer: [6:7587241317670433193:0] socket: 626 qp: -1
2025-12-24T02:11:19.994606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241318158222439:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:19.994615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:19.994787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241317775401373:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16496261ns) {Configure:104(16495264ns) {ProcessConfigured:135(15311502ns) {StartInitialHandshake:157(15310868ns) {DropHandshakes:470(482ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(222ns)}}}}}
2025-12-24T02:11:19.994722Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241317670433116:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17564326ns) {Configure:104(17563606ns) {ProcessConfigured:135(17264292ns) {StartInitialHandshake:157(17263808ns) {DropHandshakes:470(372ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(825ns) {SwitchToState:179(341ns)}}}}}
2025-12-24T02:11:19.994757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317670433116:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:19.994792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441632:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.996501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241318158222420:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:19.996713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241318158222416:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:19.996733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:19.996740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241318158222416:2108] [node 5] ICP22 created new session: [3:7587241318158222442:2108]
2025-12-24T02:11:19.996751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241318158222442:2108] [node 5] ICS09 handshake done sender: [3:7587241318158222420:2112] self: [3:7587241318158222419:0] peer: [5:7587241318767245359:0] socket: 625 qp: -1
2025-12-24T02:11:19.996756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241318158222442:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:19.996763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.997795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241317775401439:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22577838ns) {Configure:104(22576377ns) {ProcessConfigured:135(22238608ns) {StartInitialHandshake:157(22238220ns) {DropHandshakes:470(584ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(93ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(846ns) {SwitchToState:179(548ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.997846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241318767245358:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.997879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241318767245275:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:19.998814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241317184441646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
2025-12-24T02:11:19.998946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223378:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:19.998960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318395223378:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:19.998348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241317775401441:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.001598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241318767245366:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.001739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241318767245350:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.001894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241318395223303:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.002036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223381:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.002041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318395223381:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.001847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241318767245277:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.001872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.001880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241318767245277:2050] [node 3] ICP22 created new session: [5:7587241323062212663:2050]
2025-12-24T02:11:20.001889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241323062212663:2050] [node 3] ICS09 handshake done sender: [5:7587241318767245350:2114] self: [5:7587241318767245359:0] peer: [3:7587241318158222419:0] socket: 631 qp: -1
2025-12-24T02:11:20.001898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241323062212663:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.001907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.002559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441646:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.002671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321479408945:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35902
2025-12-24T02:11:20.002616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241322690190689:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1654
2025-12-24T02:11:20.002917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318395223381:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.003094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318395223372:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.003105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.003112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318395223372:2110] [node 6] ICP22 created new session: [1:7587241322690190691:2110]
2025-12-24T02:11:20.003119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241322690190691:2110] [node 6] ICS09 handshake done sender: [1:7587241318395223381:2114] self: [1:7587241318395223377:0] peer: [6:7587241317670433202:0] socket: 648 qp: -1
2025-12-24T02:11:20.003127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241322690190691:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.003134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.003582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318395223378:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.003825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318395223371:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.003849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.003856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318395223371:2109] [node 5] ICP22 created new session: [1:7587241322690190694:2109]
2025-12-24T02:11:20.003861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241322690190694:2109] [node 5] ICS09 handshake done sender: [1:7587241318395223378:2113] self: [1:7587241318395223375:0] peer: [5:7587241318767245365:0] socket: 644 qp: -1
No RDMA context found for GID: 2025-12-24T02:11:20.003867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241322690190694:2109] [node 5] ICS10 traffic start
::ffff:127.1.0.0
2025-12-24T02:11:20.003874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.003773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241317670433197:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.003903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317670433204:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:20.004085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241317670433203:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.004766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222437:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:20.004859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241318158222417:2109] [node 6] ICP18 handshake (actor [3:7587241318158222437:2118]) is too old
::ffff:127.1.0.0
2025-12-24T02:11:20.004870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322453189740:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:20.005009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441626:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.005026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317184441626:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.004849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241318395223376:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.004863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241318395223376:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.005200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322453189741:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:20.006083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322453189740:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.006110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241318158222351:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.006535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317184441626:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.001701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241317775401452:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.004324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241317775401453:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.006381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317670433117:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.004388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241317775401454:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.004424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241317775401455:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.004477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401441:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.004500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401452:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.004524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241317775401373:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.004545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241317775401374:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.004910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401454:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.004937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241317775401375:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.006521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401455:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.007399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441634:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.007416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317184441634:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.007571Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241318158222425:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.007581Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241318158222425:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.008219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241318158222425:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.008302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241318158222423:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.008318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.008324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241318158222423:2107] [node 4] ICP22 created new session: [3:7587241322453189744:2107]
2025-12-24T02:11:20.008329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322453189744:2107] [node 4] ICS09 handshake done sender: [3:7587241318158222425:2114] self: [3:7587241318158222424:0] peer: [4:7587241322070368754:0] socket: 653 qp: -1
2025-12-24T02:11:20.008335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322453189744:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.008343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.007500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322070368755:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.008639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317670433196:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.008766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317670433182:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.009027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317184441620:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.009042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.009047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317184441620:2108] [node 4] ICP22 created new session: [2:7587241321479408948:2108]
2025-12-24T02:11:20.009053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321479408948:2108] [node 4] ICS09 handshake done sender: [2:7587241317184441626:2112] self: [2:7587241317184441625:0] peer: [4:7587241322070368753:0] socket: 637 qp: -1
2025-12-24T02:11:20.009069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321479408948:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:20.009076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.008844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317670433118:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241317670433182:2110] held: no
2025-12-24T02:11:20.008851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317670433116:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.008863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.008868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317670433118:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.008872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317670433116:2048] [node 1] ICP22 created new session: [6:7587241321965400502:2048]
2025-12-24T02:11:20.008876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.008881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317670433118:2050] [node 3] ICP22 created new session: [6:7587241321965400503:2050]
2025-12-24T02:11:20.008885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241321965400503:2050] [node 3] ICS09 handshake done sender: [6:7587241317670433182:2110] self: [6:7587241317670433193:0] peer: [3:7587241318158222421:0] socket: 627 qp: -1
2025-12-24T02:11:20.008892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241321965400503:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.008899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.008898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241321965400502:2048] [node 1] ICS09 handshake done sender: [6:7587241317670433196:2115] self: [6:7587241317670433202:0] peer: [1:7587241318395223377:0] socket: 654 qp: -1
2025-12-24T02:11:20.008911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241321965400502:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.008923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.009230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241318395223376:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.009320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241318395223370:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.009334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.009342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241318395223370:2108] [node 4] ICP22 created new session: [1:7587241322690190697:2108]
2025-12-24T02:11:20.009347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241322690190697:2108] [node 4] ICS09 handshake done sender: [1:7587241318395223376:2112] self: [1:7587241318395223373:0] peer: [4:7587241322070368752:0] socket: 619 qp: -1
2025-12-24T02:11:20.009352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241322690190697:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:20.009358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.009771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241317184441630:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.009784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241317184441630:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.010310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401453:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.010329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241317775401452:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.010407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241321965400506:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:20.010449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241318767245275:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241318767245358:2118] held: no
2025-12-24T02:11:20.010519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241323062212665:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:20.010311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317184441630:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.010402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317184441622:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.010414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.010419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317184441622:2110] [node 6] ICP22 created new session: [2:7587241321479408951:2110]
2025-12-24T02:11:20.010424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321479408951:2110] [node 6] ICS09 handshake done sender: [2:7587241317184441630:2114] self: [2:7587241317184441629:0] peer: [6:7587241321965400501:0] socket: 633 qp: -1
2025-12-24T02:11:20.010428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321479408951:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.010447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.010750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241318767245358:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.010846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241317775401374:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.010861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.010866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241317775401374:2049] [node 2] ICP22 created new session: [4:7587241322070368759:2049]
2025-12-24T02:11:20.010870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322070368759:2049] [node 2] ICS09 handshake done sender: [4:7587241317775401452:2116] self: [4:7587241322070368753:0] peer: [2:7587241317184441625:0] socket: 664 qp: -1
2025-12-24T02:11:20.010875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322070368759:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.010893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.011968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317670433117:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16012) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241317670433197:2116] held: no
2025-12-24T02:11:20.012020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241317670433197:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.012002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241318767245275:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.012017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.012029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241318767245275:2048] [node 1] ICP22 created new session: [5:7587241323062212666:2048]
2025-12-24T02:11:20.012034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241323062212666:2048] [node 1] ICS09 handshake done sender: [5:7587241318767245358:2118] self: [5:7587241318767245365:0] peer: [1:7587241318395223375:0] socket: 645 qp: -1
2025-12-24T02:11:20.012040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241323062212666:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.012047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.012080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317670433117:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.012098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.012103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317670433117:2049] [node 2] ICP22 created new session: [6:7587241321965400507:2049]
2025-12-24T02:11:20.012108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241321965400507:2049] [node 2] ICS09 handshake done sender: [6:7587241317670433197:2116] self: [6:7587241321965400501:0] peer: [2:7587241317184441629:0] socket: 655 qp: -1
2025-12-24T02:11:20.012113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241321965400507:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.012121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.012310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241317184441634:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.012474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322453189746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57117
2025-12-24T02:11:20.012638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322070368757:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.012768Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322453189740:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.012752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241317184441556:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.012834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241318158222351:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.012773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.012780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241317184441556:2049] [node 3] ICP22 created new session: [2:7587241321479408954:2049]
2025-12-24T02:11:20.012786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321479408954:2049] [node 3] ICS09 handshake done sender: [2:7587241317184441634:2117] self: [2:7587241317184441623:0] peer: [3:7587241322453189742:0] socket: 640 qp: -1
2025-12-24T02:11:20.012791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321479408954:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.012798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.012846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.012852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241318158222351:2049] [node 2] ICP22 created new session: [3:7587241322453189747:2049]
2025-12-24T02:11:20.012858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322453189747:2049] [node 2] ICS09 handshake done sender: [3:7587241322453189740:2119] self: [3:7587241322453189742:0] peer: [2:7587241317184441623:0] socket: 673 qp: -1
2025-12-24T02:11:20.012864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322453189747:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.012870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.014193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322070368758:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38388
2025-12-24T02:11:20.014353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241317775401454:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.014946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241317775401375:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.014968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.014975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241317775401375:2050] [node 3] ICP22 created new session: [4:7587241322070368763:2050]
2025-12-24T02:11:20.014981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322070368763:2050] [node 3] ICS09 handshake done sender: [4:7587241317775401454:2118] self: [4:7587241322070368754:0] peer: [3:7587241318158222424:0] socket: 671 qp: -1
2025-12-24T02:11:20.014987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322070368763:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.014997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.015194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241317775401441:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.015425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241317775401373:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.015457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.015469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241317775401373:2048] [node 1] ICP22 created new session: [4:7587241322070368765:2048]
2025-12-24T02:11:20.015477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322070368765:2048] [node 1] ICS09 handshake done sender: [4:7587241317775401441:2111] self: [4:7587241322070368752:0] peer: [1:7587241318395223373:0] socket: 630 qp: -1
2025-12-24T02:11:20.015483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322070368765:2048] [node 1] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.015507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.015519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241323062212668:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.018076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241321965400509:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:20.017587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241318395223370:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.019173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241321965400509:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.019222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241317670433188:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.019217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241323062212668:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.019260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241318767245349:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.019333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401447:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.019345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241317775401447:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.019354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241317775401448:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.019360Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241317775401448:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.020173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241317775401447:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.020301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241317775401440:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.020324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.020312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241321965400511:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5393
2025-12-24T02:11:20.020337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241317775401440:2109] [node 6] ICP22 created new session: [4:7587241322070368770:2109]
2025-12-24T02:11:20.020348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322070368770:2109] [node 6] ICS09 handshake done sender: [4:7587241317775401447:2114] self: [4:7587241317775401446:0] peer: [6:7587241321965400510:0] socket: 642 qp: -1
2025-12-24T02:11:20.020356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322070368770:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.020365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.020393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241321965400509:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.020477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241317670433188:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.020495Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.020501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241317670433188:2111] [node 4] ICP22 created new session: [6:7587241321965400512:2111]
2025-12-24T02:11:20.020506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241321965400512:2111] [node 4] ICS09 handshake done sender: [6:7587241321965400509:2121] self: [6:7587241321965400510:0] peer: [4:7587241317775401446:0] socket: 661 qp: -1
2025-12-24T02:11:20.020512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241321965400512:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:20.020518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.020650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241317775401448:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.020623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241323062212670:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57200
2025-12-24T02:11:20.020773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241323062212668:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.020972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241318767245349:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.020981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.020992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241318767245349:2113] [node 4] ICP22 created new session: [5:7587241323062212671:2113]
2025-12-24T02:11:20.020998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241323062212671:2113] [node 4] ICS09 handshake done sender: [5:7587241323062212668:2122] self: [5:7587241323062212669:0] peer: [4:7587241317775401444:0] socket: 634 qp: -1
2025-12-24T02:11:20.021005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241323062212671:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:20.021012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.020916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241317775401439:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.020932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.020939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241317775401439:2108] [node 5] ICP22 created new session: [4:7587241322070368772:2108]
2025-12-24T02:11:20.020944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322070368772:2108] [node 5] ICS09 handshake done sender: [4:7587241317775401448:2115] self: [4:7587241317775401444:0] peer: [5:7587241323062212669:0] socket: 624 qp: -1
2025-12-24T02:11:20.020955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322070368772:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.020963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.028646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.028716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241317184441641:2048] [node 1] ICS07 socket disconnect 617 reason# EndOfStream
2025-12-24T02:11:20.028722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241317184441641:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.028760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.028779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241317184441641:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.032715Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.032821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241318158222434:2048] [node 1] ICS01 socket: 621 reason# EndOfStream
2025-12-24T02:11:20.032830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241318158222350:2048] [node 1] ICP30 unregister session Session# [3:7587241318158222434:2048] VirtualId# [3:7587241318158222431:0]
2025-12-24T02:11:20.032837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241318158222434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.032889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.032916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.034355Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241317184441555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:20.034372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241317184441641:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.034786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241317184441555:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.034793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241317184441641:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.034798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241317184441555:2048] [node 1] ICP30 unregister session Session# [2:7587241317184441641:2048] VirtualId# [2:7587241317184441636:0]
2025-12-24T02:11:20.034820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.034833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241317184441555:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.034836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.035640Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.035709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241323062212666:2048] [node 1] ICS01 socket: 645 reason# EndOfStream
2025-12-24T02:11:20.035717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241318767245275:2048] [node 1] ICP30 unregister session Session# [5:7587241323062212666:2048] VirtualId# [5:7587241318767245365:0]
2025-12-24T02:11:20.035722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241323062212666:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.035806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.035905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241321965400502:2048] [node 1] ICS07 socket disconnect 654 reason# EndOfStream
2025-12-24T02:11:20.035917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241321965400502:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.035975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.036007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241321965400502:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.035767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.035810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.036362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317670433116:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.036379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241321965400502:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.036383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317670433116:2048] [node 1] ICP30 unregister session Session# [6:7587241321965400502:2048] VirtualId# [6:7587241317670433202:0]
2025-12-24T02:11:20.036401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.036406Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317670433116:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.036408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30214) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.041727Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.041834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322070368765:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:20.041845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241317775401373:2048] [node 1] ICP30 unregister session Session# [4:7587241322070368765:2048] VirtualId# [4:7587241322070368752:0]
2025-12-24T02:11:20.041853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322070368765:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.041920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.041949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.045419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241317670433116:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.045897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.046009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322453189747:2049] [node 2] ICS01 socket: 673 reason# EndOfStream
2025-12-24T02:11:20.046019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241318158222351:2049] [node 2] ICP30 unregister session Session# [3:7587241322453189747:2049] VirtualId# [3:7587241322453189742:0]
2025-12-24T02:11:20.046027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322453189747:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.046081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.046114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.047932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.048022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322070368759:2049] [node 2] ICS01 socket: 664 reason# EndOfStream
2025-12-24T02:11:20.048031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241317775401374:2049] [node 2] ICP30 unregister session Session# [4:7587241322070368759:2049] VirtualId# [4:7587241322070368753:0]
2025-12-24T02:11:20.048037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322070368759:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.048099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.048132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.048278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.048384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241321965400507:2049] [node 2] ICS01 socket: 655 reason# EndOfStream
2025-12-24T02:11:20.048398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317670433117:2049] [node 2] ICP30 unregister session Session# [6:7587241321965400507:2049] VirtualId# [6:7587241321965400501:0]
2025-12-24T02:11:20.048404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241321965400507:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.048457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.048490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.049849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.049927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241318767245362:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-12-24T02:11:20.049939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241318767245276:2049] [node 2] ICP30 unregister session Session# [5:7587241318767245362:2049] VirtualId# [5:7587241318767245360:0]
2025-12-24T02:11:20.049944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241318767245362:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.049999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.050033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.059212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:20.059310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241323062212663:2050] [node 3] ICS01 socket: 631 reason# EndOfStream
2025-12-24T02:11:20.059318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241318767245277:2050] [node 3] ICP30 unregister session Session# [5:7587241323062212663:2050] VirtualId# [5:7587241318767245359:0]
2025-12-24T02:11:20.059325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241323062212663:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.059388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.059415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.060602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:20.060705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241321965400503:2050] [node 3] ICS01 socket: 627 reason# EndOfStream
2025-12-24T02:11:20.060716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317670433118:2050] [node 3] ICP30 unregister session Session# [6:7587241321965400503:2050] VirtualId# [6:7587241317670433193:0]
2025-12-24T02:11:20.060721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241321965400503:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.060791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.060824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.066123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:20.066217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241321965400512:2111] [node 4] ICS07 socket disconnect 661 reason# EndOfStream
2025-12-24T02:11:20.066228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241321965400512:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.066283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.066311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241321965400512:2111] [node 4] ICS15 start handshake
2025-12-24T02:11:20.066645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317670433188:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62613) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.066659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241321965400512:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.066664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317670433188:2111] [node 4] ICP30 unregister session Session# [6:7587241321965400512:2111] VirtualId# [6:7587241321965400510:0]
2025-12-24T02:11:20.066682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.066689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317670433188:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.066704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62613) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.075920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241317670433188:2111] [node 4] ICP33 wake up from error state
2025-12-24T02:11:20.082475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:20.082581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241317670433184:2108] [node 5] ICS07 socket disconnect 613 reason# EndOfStream
2025-12-24T02:11:20.082589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241317670433184:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.082655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:20.082681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241317670433184:2108] [node 5] ICS15 start handshake
2025-12-24T02:11:20.083370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241317670433178:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28895) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.083388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241317670433184:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.083393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241317670433178:2108] [node 5] ICP30 unregister session Session# [6:7587241317670433184:2108] VirtualId# [6:7587241317670433179:0]
2025-12-24T02:11:20.083420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:20.083445Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241317670433178:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28895) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.083448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28895) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:20.100645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321150260466:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:20.100684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321150260467:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.106564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319822868544:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.106578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319822868546:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.106600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319822868545:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.109586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320092508563:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.109602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320092508565:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.109976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320092508564:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.102019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320085814328:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.102032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320085814329:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.105086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322377630078:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.105095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322377630079:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.105102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322377630080:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.103572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241320949676380:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.103581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241320949676381:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.118884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320092508625:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.123002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868605:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.123055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320085814393:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.123161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320085814394:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.123247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320085814396:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.121987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241320949676443:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.122077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241320949676444:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.122145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241320949676446:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.127153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321150260533:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.127251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321150260534:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.127958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321150260539:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29000
2025-12-24T02:11:20.130802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321150260531:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.137572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320092508632:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:20.139086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319822868612:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.139521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676452:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.139652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508633:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.139683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508634:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.139866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868613:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.141347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868621:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.143043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508641:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.143529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676458:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
2025-12-24T02:11:20.143589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676459:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
2025-12-24T02:11:20.144163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241319822868545:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20106231ns) {Configure:104(20105392ns) {ProcessConfigured:135(18788391ns) {StartInitialHandshake:157(18787780ns) {DropHandshakes:470(189ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(25ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(616ns) {SwitchToState:179(188ns)}}}}}
2025-12-24T02:11:20.144908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676458:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.145705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868605:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.146437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320092508641:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.146480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320092508625:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.146576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868617:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.146585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241319822868617:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.146921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241322377630078:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24267065ns) {Configure:104(24266498ns) {ProcessConfigured:135(21355937ns) {StartInitialHandshake:157(21354568ns) {DropHandshakes:470(299ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(733ns) {SwitchToState:179(81ns)}}}}}
2025-12-24T02:11:20.146939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241322377630079:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24291252ns) {Configure:104(24288564ns) {ProcessConfigured:135(21338713ns) {StartInitialHandshake:157(21337937ns) {DropHandshakes:470(474ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(125ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(649ns)}}}}}
2025-12-24T02:11:20.146953Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241320092508563:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20864240ns) {Configure:104(20863062ns) {ProcessConfigured:135(20498879ns) {StartInitialHandshake:157(20498473ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(812ns) {SwitchToState:179(472ns)}}}}}
2025-12-24T02:11:20.147094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322377630145:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.147827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508644:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.147555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630144:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.147915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630144:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.149339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241319822868607:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:20.152611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868621:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.152649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319822868546:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.153061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322377630141:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:20.153370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241320949676446:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25115743ns) {Configure:104(25114878ns) {ProcessConfigured:135(24881115ns) {StartInitialHandshake:157(24880626ns) {DropHandshakes:470(303ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(39ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(1423ns) {SwitchToState:179(915ns)}}}}}
2025-12-24T02:11:20.153439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676459:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.153497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676451:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.153505Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241320949676451:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.154021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241320949676451:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.154112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241320949676444:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.154126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.154131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241320949676444:2108] [node 5] ICP22 created new session: [3:7587241320949676464:2108]
2025-12-24T02:11:20.154135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241320949676464:2108] [node 5] ICS09 handshake done sender: [3:7587241320949676451:2111] self: [3:7587241320949676448:0] peer: [5:7587241319822868625:0] socket: 635 qp: -1
2025-12-24T02:11:20.154140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241320949676464:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.154156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.158464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320092508633:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.158427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676452:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.158462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241320949676381:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.157567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320085814406:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63857
2025-12-24T02:11:20.157619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320085814407:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63857
2025-12-24T02:11:20.157635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320085814405:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63857
2025-12-24T02:11:20.158279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814406:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.158550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814395:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.158554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320085814395:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.158861Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241321150260531:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27973685ns) {Configure:104(27972736ns) {ProcessConfigured:135(27616857ns) {StartInitialHandshake:157(27616088ns) {DropHandshakes:470(577ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(946ns) {SwitchToState:179(595ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.159057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321150260547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.159098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321150260545:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.159154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321150260546:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29000
2025-12-24T02:11:20.159525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868626:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.162828Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241320949676443:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40631181ns) {Configure:104(40630091ns) {ProcessConfigured:135(40414881ns) {StartInitialHandshake:157(40414496ns) {DropHandshakes:470(202ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:477(985ns) {SwitchToState:179(341ns)}}}}}
2025-12-24T02:11:20.163506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319822868546:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319822868621:2115] held: no
2025-12-24T02:11:20.163548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319822868621:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.163628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319822868546:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.163644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.163652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319822868546:2050] [node 3] ICP22 created new session: [5:7587241319822868629:2050]
2025-12-24T02:11:20.163658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319822868629:2050] [node 3] ICS09 handshake done sender: [5:7587241319822868621:2115] self: [5:7587241319822868625:0] peer: [3:7587241320949676448:0] socket: 643 qp: -1
2025-12-24T02:11:20.163666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319822868629:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.163679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.164113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319822868617:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.165267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241319822868612:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.165290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.165298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319822868612:2110] [node 6] ICP22 created new session: [5:7587241319822868631:2110]
2025-12-24T02:11:20.165304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319822868631:2110] [node 6] ICS09 handshake done sender: [5:7587241319822868617:2114] self: [5:7587241319822868616:0] peer: [6:7587241320092508642:0] socket: 642 qp: -1
2025-12-24T02:11:20.165311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319822868631:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.165318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.166390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241320092508625:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11698) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241320092508641:2115] held: no
2025-12-24T02:11:20.162915Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241320085814393:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39678586ns) {Configure:104(39676950ns) {ProcessConfigured:135(39228316ns) {StartInitialHandshake:157(39228153ns) {DropHandshakes:470(367ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(1164ns) {SwitchToState:179(589ns)}}}}}
2025-12-24T02:11:20.166419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320085814410:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63857
2025-12-24T02:11:20.163268Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241321150260467:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43941990ns) {Configure:104(43941023ns) {ProcessConfigured:135(42592035ns) {StartInitialHandshake:157(42591587ns) {DropHandshakes:470(809ns) {DropIncomingHandshake:442(334ns) DropOutgoingHandshake:457(102ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(883ns) {SwitchToState:179(813ns)}}}}}
2025-12-24T02:11:20.166833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260545:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.166870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260539:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.166894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260546:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.166879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241320085814394:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43583465ns) {Configure:104(43582858ns) {ProcessConfigured:135(43172778ns) {StartInitialHandshake:157(43172115ns) {DropHandshakes:470(534ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(498ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.167000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321150260549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29000
2025-12-24T02:11:20.171234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320085814395:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.171264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241320092508564:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(45061843ns) {Configure:104(45060910ns) {ProcessConfigured:135(44736337ns) {StartInitialHandshake:157(44735540ns) {DropHandshakes:470(521ns) {DropIncomingHandshake:442(80ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(394ns) PrepareNewSessionHandshake:477(1155ns) {SwitchToState:179(654ns)}}}}}
2025-12-24T02:11:20.171313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320092508564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.171481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508649:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
::ffff:127.1.0.0
2025-12-24T02:11:20.172022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508648:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.171541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814401:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.171549Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320085814401:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.173002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814410:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.173137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320085814401:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.173207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320085814396:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.173221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.173236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320085814396:2109] [node 6] ICP22 created new session: [2:7587241320085814416:2109]
2025-12-24T02:11:20.173241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320085814416:2109] [node 6] ICS09 handshake done sender: [2:7587241320085814401:2112] self: [2:7587241320085814400:0] peer: [6:7587241320092508647:0] socket: 623 qp: -1
2025-12-24T02:11:20.173250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320085814416:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.173260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.173609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868634:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.174473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320092508641:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.174500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508650:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.174626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320092508625:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.174643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.174650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320092508625:2108] [node 5] ICP22 created new session: [6:7587241320092508651:2108]
2025-12-24T02:11:20.174656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320092508651:2108] [node 5] ICS09 handshake done sender: [6:7587241320092508641:2115] self: [6:7587241320092508642:0] peer: [5:7587241319822868616:0] socket: 647 qp: -1
2025-12-24T02:11:20.174663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320092508651:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.174670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.174972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320092508633:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.176236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868634:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.176284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319822868545:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.177360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630158:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.177008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676469:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.177659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320092508564:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.177679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.177686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320092508564:2049] [node 2] ICP22 created new session: [6:7587241320092508653:2049]
2025-12-24T02:11:20.177692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320092508653:2049] [node 2] ICS09 handshake done sender: [6:7587241320092508633:2111] self: [6:7587241320092508647:0] peer: [2:7587241320085814400:0] socket: 630 qp: -1
2025-12-24T02:11:20.177697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320092508653:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.177705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.177901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320092508648:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.178053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320092508565:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.175161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320085814329:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.175193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.175203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320085814329:2049] [node 3] ICP22 created new session: [2:7587241320085814419:2049]
2025-12-24T02:11:20.175210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320085814419:2049] [node 3] ICS09 handshake done sender: [2:7587241320085814395:2110] self: [2:7587241320085814388:0] peer: [3:7587241320949676467:0] socket: 621 qp: -1
2025-12-24T02:11:20.175218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320085814419:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.175225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.177957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630159:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.179330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630159:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.181225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241320949676452:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.181341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241320949676381:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.181360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.181368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241320949676381:2049] [node 2] ICP22 created new session: [3:7587241320949676470:2049]
2025-12-24T02:11:20.181374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241320949676470:2049] [node 2] ICS09 handshake done sender: [3:7587241320949676452:2112] self: [3:7587241320949676467:0] peer: [2:7587241320085814388:0] socket: 632 qp: -1
2025-12-24T02:11:20.181383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241320949676470:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.181392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.182732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814411:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.182746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320085814411:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.182821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260549:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.182974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676460:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.182990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241320949676460:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.182459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630161:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.183937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320085814411:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.184040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320085814394:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.184055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.184062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320085814394:2108] [node 5] ICP22 created new session: [2:7587241320085814422:2108]
2025-12-24T02:11:20.184041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630158:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.184067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320085814422:2108] [node 5] ICS09 handshake done sender: [2:7587241320085814411:2118] self: [2:7587241320085814398:0] peer: [5:7587241319822868635:0] socket: 671 qp: -1
2025-12-24T02:11:20.184072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320085814422:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.184078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322377630079:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.184079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.184648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241320949676460:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.184756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241320949676446:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.184782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.184789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241320949676446:2109] [node 6] ICP22 created new session: [3:7587241320949676474:2109]
2025-12-24T02:11:20.184801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241320949676474:2109] [node 6] ICS09 handshake done sender: [3:7587241320949676460:2116] self: [3:7587241320949676450:0] peer: [6:7587241320092508655:0] socket: 646 qp: -1
2025-12-24T02:11:20.184806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241320949676474:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.184828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.185239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868637:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.185361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630161:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.185396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322377630080:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.185660Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814409:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.185673Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320085814409:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.186190Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319822868545:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319822868634:2119] held: no
2025-12-24T02:11:20.186247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319822868634:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.186401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319822868545:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.186428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.186435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319822868545:2049] [node 2] ICP22 created new session: [5:7587241319822868638:2049]
2025-12-24T02:11:20.186441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319822868638:2049] [node 2] ICS09 handshake done sender: [5:7587241319822868634:2119] self: [5:7587241319822868635:0] peer: [2:7587241320085814398:0] socket: 672 qp: -1
2025-12-24T02:11:20.186448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319822868638:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.186459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.187889Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241320092508565:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241320092508648:2119] held: no
2025-12-24T02:11:20.191035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320092508634:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.191091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320092508563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.191998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508658:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.192225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320092508634:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.193147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320092508563:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.193182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.193188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320092508563:2048] [node 1] ICP22 created new session: [6:7587241320092508659:2048]
2025-12-24T02:11:20.193194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320092508659:2048] [node 1] ICS09 handshake done sender: [6:7587241320092508634:2112] self: [6:7587241320092508657:0] peer: [1:7587241321150260537:0] socket: 631 qp: -1
2025-12-24T02:11:20.193202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320092508659:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.193208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.193283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241320092508563:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.191224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260538:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.191240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321150260538:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.191783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321150260538:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.191916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321150260534:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.191932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.191940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321150260534:2109] [node 6] ICP22 created new session: [1:7587241321150260555:2109]
2025-12-24T02:11:20.191946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321150260555:2109] [node 6] ICS09 handshake done sender: [1:7587241321150260538:2112] self: [1:7587241321150260537:0] peer: [6:7587241320092508657:0] socket: 627 qp: -1
2025-12-24T02:11:20.191953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321150260555:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.191961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.195162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814407:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.195877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320085814409:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.196253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676468:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.196269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241320949676468:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.196333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508656:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.196443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320092508648:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.196785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260547:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.195974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320085814393:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.196003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.196013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320085814393:2107] [node 4] ICP22 created new session: [2:7587241320085814426:2107]
2025-12-24T02:11:20.196020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320085814426:2107] [node 4] ICS09 handshake done sender: [2:7587241320085814409:2116] self: [2:7587241320085814397:0] peer: [4:7587241322377630162:0] socket: 669 qp: -1
2025-12-24T02:11:20.196026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320085814426:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.196041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.196614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241322377630079:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241322377630158:2116] held: no
2025-12-24T02:11:20.198799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320092508565:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.198824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.198853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320092508565:2050] [node 3] ICP22 created new session: [6:7587241320092508661:2050]
2025-12-24T02:11:20.198860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320092508661:2050] [node 3] ICS09 handshake done sender: [6:7587241320092508648:2119] self: [6:7587241320092508655:0] peer: [3:7587241320949676450:0] socket: 659 qp: -1
2025-12-24T02:11:20.198868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320092508661:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.198888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.199418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260532:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.199428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321150260532:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.200090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868613:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.200139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319822868544:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.199266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320085814405:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.199308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241320085814328:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.199476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630164:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.199625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630165:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.199713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322377630158:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.202877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320092508644:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.202957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320092508632:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.203577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321150260532:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.204011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260536:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.204022Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321150260536:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.204430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321150260466:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.204454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.204463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321150260466:2048] [node 2] ICP22 created new session: [1:7587241321150260560:2048]
2025-12-24T02:11:20.204469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321150260560:2048] [node 2] ICS09 handshake done sender: [1:7587241321150260532:2110] self: [1:7587241321150260530:0] peer: [2:7587241320085814428:0] socket: 626 qp: -1
2025-12-24T02:11:20.204475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321150260560:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:20.204483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.204595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241320949676468:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.205081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241320949676443:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.205095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.205101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241320949676443:2107] [node 4] ICP22 created new session: [3:7587241320949676477:2107]
2025-12-24T02:11:20.205108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241320949676477:2107] [node 4] ICS09 handshake done sender: [3:7587241320949676468:2117] self: [3:7587241320949676447:0] peer: [4:7587241322377630163:0] socket: 676 qp: -1
2025-12-24T02:11:20.205113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241320949676477:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.205125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.206411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321150260536:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.203905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322377630079:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.206655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676479:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
2025-12-24T02:11:20.203924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.203931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322377630079:2049] [node 2] ICP22 created new session: [4:7587241322377630166:2049]
2025-12-24T02:11:20.203938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322377630166:2049] [node 2] ICS09 handshake done sender: [4:7587241322377630158:2116] self: [4:7587241322377630162:0] peer: [2:7587241320085814397:0] socket: 668 qp: -1
2025-12-24T02:11:20.203945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322377630166:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.203964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.204403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630164:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.204794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322377630078:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.204814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630168:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.205369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630150:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.205379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241322377630150:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.205324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320085814429:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63857
2025-12-24T02:11:20.205592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320085814405:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.205693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241320085814328:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.205705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.205712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320085814328:2048] [node 1] ICP22 created new session: [2:7587241320085814430:2048]
2025-12-24T02:11:20.205719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320085814430:2048] [node 1] ICS09 handshake done sender: [2:7587241320085814405:2113] self: [2:7587241320085814428:0] peer: [1:7587241321150260530:0] socket: 654 qp: -1
2025-12-24T02:11:20.205725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320085814430:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.205733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.207245Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322377630161:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.207465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676479:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.207513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241320949676443:2107] [node 4] ICP18 handshake (actor [3:7587241320949676479:2119]) is too old
2025-12-24T02:11:20.207402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322377630080:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.207416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.207421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322377630080:2050] [node 3] ICP22 created new session: [4:7587241322377630171:2050]
2025-12-24T02:11:20.207425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322377630171:2050] [node 3] ICS09 handshake done sender: [4:7587241322377630161:2118] self: [4:7587241322377630163:0] peer: [3:7587241320949676447:0] socket: 677 qp: -1
2025-12-24T02:11:20.207444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322377630171:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.207451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.208254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676480:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
2025-12-24T02:11:20.209446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322377630150:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.209536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241322377630145:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.209553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.209558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322377630145:2107] [node 6] ICP22 created new session: [4:7587241322377630175:2107]
2025-12-24T02:11:20.209563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322377630175:2107] [node 6] ICS09 handshake done sender: [4:7587241322377630150:2112] self: [4:7587241322377630148:0] peer: [6:7587241320092508663:0] socket: 651 qp: -1
2025-12-24T02:11:20.209569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322377630175:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:20.209576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.209716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320092508664:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38942
2025-12-24T02:11:20.209870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241320092508632:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62689) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241320092508644:2117] held: no
2025-12-24T02:11:20.209924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320092508644:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.210028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320092508632:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.210050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.210058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320092508632:2110] [node 4] ICP22 created new session: [6:7587241320092508665:2110]
2025-12-24T02:11:20.210064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320092508665:2110] [node 4] ICS09 handshake done sender: [6:7587241320092508644:2117] self: [6:7587241320092508663:0] peer: [4:7587241322377630148:0] socket: 653 qp: -1
2025-12-24T02:11:20.210071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320092508665:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:20.210089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.210525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241320949676480:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.210560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241320949676380:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.210586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322377630177:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57844
2025-12-24T02:11:20.206651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260544:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.206664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321150260544:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.206778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321150260533:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.206792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.206799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321150260533:2108] [node 5] ICP22 created new session: [1:7587241321150260564:2108]
2025-12-24T02:11:20.206805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321150260564:2108] [node 5] ICS09 handshake done sender: [1:7587241321150260536:2111] self: [1:7587241321150260535:0] peer: [5:7587241319822868640:0] socket: 628 qp: -1
2025-12-24T02:11:20.206810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321150260564:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.206828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.211912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868641:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.213539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321150260548:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.213560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321150260548:2118] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.214582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319822868613:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.214702Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241319822868544:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18545) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241319822868613:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.214731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319822868544:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.214744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.214749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319822868544:2048] [node 1] ICP22 created new session: [5:7587241319822868643:2048]
2025-12-24T02:11:20.214755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319822868643:2048] [node 1] ICS09 handshake done sender: [5:7587241319822868613:2112] self: [5:7587241319822868640:0] peer: [1:7587241321150260535:0] socket: 633 qp: -1
2025-12-24T02:11:20.214762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319822868643:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.214771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.214760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321150260544:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.214840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321150260531:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.214858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.214860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322377630164:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.214865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321150260531:2107] [node 4] ICP22 created new session: [1:7587241321150260567:2107]
2025-12-24T02:11:20.214871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321150260567:2107] [node 4] ICS09 handshake done sender: [1:7587241321150260544:2114] self: [1:7587241321150260543:0] peer: [4:7587241322377630170:0] socket: 645 qp: -1
2025-12-24T02:11:20.214877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321150260567:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.214886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.214927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322377630078:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.214942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.214951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322377630078:2048] [node 1] ICP22 created new session: [4:7587241322377630178:2048]
2025-12-24T02:11:20.214956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322377630178:2048] [node 1] ICS09 handshake done sender: [4:7587241322377630164:2119] self: [4:7587241322377630170:0] peer: [1:7587241321150260543:0] socket: 664 qp: -1
2025-12-24T02:11:20.214961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322377630178:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.214967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.215363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241320949676482:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11853
2025-12-24T02:11:20.218780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868642:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.219037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321150260548:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.219289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241319822868642:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.219372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241319822868607:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.219223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321150260467:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.219248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.219256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321150260467:2049] [node 3] ICP22 created new session: [1:7587241321150260570:2049]
2025-12-24T02:11:20.219261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321150260570:2049] [node 3] ICS09 handshake done sender: [1:7587241321150260548:2118] self: [1:7587241321150260529:0] peer: [3:7587241320949676481:0] socket: 684 qp: -1
2025-12-24T02:11:20.219267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321150260570:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.219276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.219478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322377630157:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.219491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241322377630157:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.220164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322377630157:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.220273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241322377630141:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.220272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241319822868646:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45613
2025-12-24T02:11:20.220289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.220296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322377630141:2106] [node 5] ICP22 created new session: [4:7587241322377630181:2106]
2025-12-24T02:11:20.220301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322377630181:2106] [node 5] ICS09 handshake done sender: [4:7587241322377630157:2115] self: [4:7587241322377630156:0] peer: [5:7587241319822868645:0] socket: 622 qp: -1
2025-12-24T02:11:20.220306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322377630181:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:20.220313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.220427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241319822868642:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.220532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241319822868607:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.220548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.220554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241319822868607:2109] [node 4] ICP22 created new session: [5:7587241319822868647:2109]
2025-12-24T02:11:20.220560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241319822868647:2109] [node 4] ICS09 handshake done sender: [5:7587241319822868642:2122] self: [5:7587241319822868645:0] peer: [4:7587241322377630156:0] socket: 688 qp: -1
2025-12-24T02:11:20.220566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241319822868647:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:20.220574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.223575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241320949676380:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18545) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241320949676480:2120] held: no
2025-12-24T02:11:20.223614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241320949676480:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.223711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241320949676380:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.223724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.223730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241320949676380:2048] [node 1] ICP22 created new session: [3:7587241320949676483:2048]
2025-12-24T02:11:20.223736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241320949676483:2048] [node 1] ICS09 handshake done sender: [3:7587241320949676480:2120] self: [3:7587241320949676481:0] peer: [1:7587241321150260529:0] socket: 686 qp: -1
2025-12-24T02:11:20.223744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241320949676483:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.223755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.230835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.230935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241320085814430:2048] [node 1] ICS01 socket: 654 reason# EndOfStream
2025-12-24T02:11:20.230949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241320085814328:2048] [node 1] ICP30 unregister session Session# [2:7587241320085814430:2048] VirtualId# [2:7587241320085814428:0]
2025-12-24T02:11:20.230957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241320085814430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.231019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.231066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.231138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.231232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241320949676483:2048] [node 1] ICS01 socket: 686 reason# EndOfStream
2025-12-24T02:11:20.231247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241320949676380:2048] [node 1] ICP30 unregister session Session# [3:7587241320949676483:2048] VirtualId# [3:7587241320949676481:0]
2025-12-24T02:11:20.231254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241320949676483:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.231307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.231339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.232174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.232247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322377630178:2048] [node 1] ICS01 socket: 664 reason# EndOfStream
2025-12-24T02:11:20.232259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322377630078:2048] [node 1] ICP30 unregister session Session# [4:7587241322377630178:2048] VirtualId# [4:7587241322377630170:0]
2025-12-24T02:11:20.232264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322377630178:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.232320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.232352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.232336Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.232422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319822868643:2048] [node 1] ICS01 socket: 633 reason# EndOfStream
2025-12-24T02:11:20.232429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319822868544:2048] [node 1] ICP30 unregister session Session# [5:7587241319822868643:2048] VirtualId# [5:7587241319822868640:0]
2025-12-24T02:11:20.232434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319822868643:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.232484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.232519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.247694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.247785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241320949676470:2049] [node 2] ICS01 socket: 632 reason# EndOfStream
2025-12-24T02:11:20.247798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241320949676381:2049] [node 2] ICP30 unregister session Session# [3:7587241320949676470:2049] VirtualId# [3:7587241320949676467:0]
2025-12-24T02:11:20.247805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241320949676470:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.247859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.247896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.248059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.248151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320092508659:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-12-24T02:11:20.248165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320092508563:2048] [node 1] ICP30 unregister session Session# [6:7587241320092508659:2048] VirtualId# [6:7587241320092508657:0]
2025-12-24T02:11:20.248171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320092508659:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.248237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.248282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.248612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.248692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322377630166:2049] [node 2] ICS01 socket: 668 reason# EndOfStream
2025-12-24T02:11:20.248699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322377630079:2049] [node 2] ICP30 unregister session Session# [4:7587241322377630166:2049] VirtualId# [4:7587241322377630162:0]
2025-12-24T02:11:20.248706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322377630166:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.248778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.248757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.248790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.248860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319822868638:2049] [node 2] ICS01 socket: 672 reason# EndOfStream
2025-12-24T02:11:20.248872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319822868545:2049] [node 2] ICP30 unregister session Session# [5:7587241319822868638:2049] VirtualId# [5:7587241319822868635:0]
2025-12-24T02:11:20.248931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.248880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319822868638:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.248949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.248980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.249003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320092508653:2049] [node 2] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:20.249016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320092508564:2049] [node 2] ICP30 unregister session Session# [6:7587241320092508653:2049] VirtualId# [6:7587241320092508647:0]
2025-12-24T02:11:20.249021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320092508653:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.249106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.249140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.258891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:20.259204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:20.258997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322377630171:2050] [node 3] ICS01 socket: 677 reason# EndOfStream
2025-12-24T02:11:20.259010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322377630080:2050] [node 3] ICP30 unregister session Session# [4:7587241322377630171:2050] VirtualId# [4:7587241322377630163:0]
2025-12-24T02:11:20.259016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322377630171:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.259073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.259103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.259307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320092508661:2050] [node 3] ICS01 socket: 659 reason# EndOfStream
2025-12-24T02:11:20.259318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320092508565:2050] [node 3] ICP30 unregister session Session# [6:7587241320092508661:2050] VirtualId# [6:7587241320092508655:0]
2025-12-24T02:11:20.259323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320092508661:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.259362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.259385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.259385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:20.259469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319822868629:2050] [node 3] ICS01 socket: 643 reason# EndOfStream
2025-12-24T02:11:20.259475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319822868546:2050] [node 3] ICP30 unregister session Session# [5:7587241319822868629:2050] VirtualId# [5:7587241319822868625:0]
2025-12-24T02:11:20.259481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319822868629:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.259524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.259566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.270161Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:20.270269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241319822868647:2109] [node 4] ICS01 socket: 688 reason# EndOfStream
2025-12-24T02:11:20.270285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241319822868607:2109] [node 4] ICP30 unregister session Session# [5:7587241319822868647:2109] VirtualId# [5:7587241319822868645:0]
2025-12-24T02:11:20.270296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241319822868647:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.270360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.270397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.270339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:20.270420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320092508665:2110] [node 4] ICS01 socket: 653 reason# EndOfStream
2025-12-24T02:11:20.270427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320092508632:2110] [node 4] ICP30 unregister session Session# [6:7587241320092508665:2110] VirtualId# [6:7587241320092508663:0]
2025-12-24T02:11:20.270434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320092508665:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.270480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.270502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.283457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:20.283569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320092508651:2108] [node 5] ICS01 socket: 647 reason# EndOfStream
2025-12-24T02:11:20.283585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320092508625:2108] [node 5] ICP30 unregister session Session# [6:7587241320092508651:2108] VirtualId# [6:7587241320092508642:0]
2025-12-24T02:11:20.283597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320092508651:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.283656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:20.283692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:20.301452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321024491751:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:20.301470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321024491752:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.302977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321823755781:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.302986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321823755782:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.304544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322718769842:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.304552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322718769843:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.306016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241321284036099:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.306022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241321284036100:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.306025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241321284036101:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.307666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322034070118:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.307672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322034070119:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.307678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322034070120:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.310302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320099567107:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.310308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320099567108:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.310311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320099567109:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.322795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320099567168:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.323529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321024491812:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.323639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070178:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.325154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321024491819:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.325238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241321024491820:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.327061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755841:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.327090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322034070178:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.327085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567176:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.327244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322034070181:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:20.327280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070180:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.327446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755841:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.327649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322034070181:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.327791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320099567171:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.327804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241320099567171:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.328364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320099567171:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.331349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755842:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.332212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322718769913:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:20.332141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321823755850:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:20.334925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241320099567168:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.334951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.334960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320099567168:2107] [node 5] ICP22 created new session: [6:7587241320099567182:2107]
2025-12-24T02:11:20.334968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320099567182:2107] [node 5] ICS09 handshake done sender: [6:7587241320099567171:2109] self: [6:7587241320099567170:0] peer: [5:7587241322034070182:0] socket: 625 qp: -1
2025-12-24T02:11:20.334977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320099567182:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.334986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.335197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241321823755782:2049] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.336713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321823755853:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:20.336779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755842:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.339486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322718769915:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:20.339532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322034070190:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:20.339564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769903:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.339574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322718769903:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.339614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322034070180:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.340265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322034070118:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.340380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491818:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.340393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321024491818:2110] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.340205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241321823755781:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.341776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322718769912:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:20.341852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755858:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.341000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491822:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.341009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321024491822:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.341018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321024491818:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.341356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321024491751:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.341367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.341374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321024491751:2048] [node 2] ICP22 created new session: [1:7587241321024491832:2048]
2025-12-24T02:11:20.341381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321024491832:2048] [node 2] ICS09 handshake done sender: [1:7587241321024491818:2110] self: [1:7587241321024491815:0] peer: [2:7587241321823755856:0] socket: 631 qp: -1
2025-12-24T02:11:20.341389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321024491832:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:20.341410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.343341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241320099567181:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.345533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321024491834:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22124
2025-12-24T02:11:20.345559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321024491835:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22124
2025-12-24T02:11:20.345666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321024491822:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.345733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320099567176:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.345834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321024491819:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.345854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.345862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321024491819:2108] [node 5] ICP22 created new session: [1:7587241321024491838:2108]
2025-12-24T02:11:20.345868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321024491838:2108] [node 5] ICS09 handshake done sender: [1:7587241321024491822:2111] self: [1:7587241321024491821:0] peer: [5:7587241322034070194:0] socket: 632 qp: -1
2025-12-24T02:11:20.345882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321024491838:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.345891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.346899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070183:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.346904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241321284036166:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.346956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320099567107:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.347009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322718769911:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29467
2025-12-24T02:11:20.346957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321024491828:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22124
2025-12-24T02:11:20.347120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322718769903:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.347661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241321284036164:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.349993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241322718769913:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17714575ns) {Configure:104(17713962ns) {ProcessConfigured:135(17380959ns) {StartInitialHandshake:157(17380618ns) {DropHandshakes:470(424ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(933ns) {SwitchToState:179(362ns)}}}}}
2025-12-24T02:11:20.350002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241321284036099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19211089ns) {Configure:104(19210247ns) {ProcessConfigured:135(17769286ns) {StartInitialHandshake:157(17768923ns) {DropHandshakes:470(437ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(98ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(847ns) {SwitchToState:179(161ns)}}}}}
2025-12-24T02:11:20.350212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769911:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.350238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322718769843:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.350252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.350260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322718769843:2049] [node 2] ICP22 created new session: [3:7587241322718769921:2049]
2025-12-24T02:11:20.350270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322718769921:2049] [node 2] ICS09 handshake done sender: [3:7587241322718769903:2107] self: [3:7587241322718769902:0] peer: [2:7587241321823755848:0] socket: 624 qp: -1
2025-12-24T02:11:20.348876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567190:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.349968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241320099567168:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(21514950ns) {HandleHandshakeStatus:329(21514125ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(362ns) SwitchToState:179(203ns) ProcessPendingSessionEvents:495(8221351ns) {ForwardSessionEventToSession:612(8219221ns)}}}
2025-12-24T02:11:20.350301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322718769921:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.350320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.350680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491824:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.350716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321024491824:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.350635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321823755849:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:20.350891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241320099567108:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16778267ns) {Configure:104(16777769ns) {ProcessConfigured:135(16606789ns) {StartInitialHandshake:157(16606554ns) {DropHandshakes:470(199ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(435ns) {SwitchToState:179(150ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.353097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036176:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.353111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036177:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.353128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036178:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.353036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491828:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.353301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036176:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.353331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036178:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.353381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241321284036101:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.353463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769920:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.353498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322718769920:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.354102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241321284036100:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23214226ns) {Configure:104(23213412ns) {ProcessConfigured:135(6468078ns) {StartInitialHandshake:157(6467349ns) {DropHandshakes:470(689ns) {DropIncomingHandshake:442(90ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(927ns) {SwitchToState:179(481ns)}}}}}
2025-12-24T02:11:20.354100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322718769920:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.354179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322718769912:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.354194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.354201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322718769912:2108] [node 4] ICP22 created new session: [3:7587241322718769928:2108]
2025-12-24T02:11:20.354215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322718769928:2108] [node 4] ICS09 handshake done sender: [3:7587241322718769920:2114] self: [3:7587241322718769918:0] peer: [4:7587241321284036180:0] socket: 669 qp: -1
2025-12-24T02:11:20.354224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322718769928:2108] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.354232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.354267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036182:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.354199Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241321823755853:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17382143ns) {Configure:104(17381736ns) {ProcessConfigured:135(17165917ns) {StartInitialHandshake:157(17165702ns) {DropHandshakes:470(492ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(634ns) {SwitchToState:179(71ns)}}}}}
2025-12-24T02:11:20.354222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321823755842:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.354527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241321284036178:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.354627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567192:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.354635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321024491841:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22124
2025-12-24T02:11:20.354967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241321284036101:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.355007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.355016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241321284036101:2050] [node 3] ICP22 created new session: [4:7587241321284036183:2050]
2025-12-24T02:11:20.355021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241321284036183:2050] [node 3] ICS09 handshake done sender: [4:7587241321284036178:2116] self: [4:7587241321284036180:0] peer: [3:7587241322718769918:0] socket: 673 qp: -1
2025-12-24T02:11:20.355029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241321284036183:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.355063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.355346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491841:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.355899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755861:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.355949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321024491824:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.354353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322034070178:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.354602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322034070181:2108] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.354612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.354618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322034070181:2108] [node 6] ICP22 created new session: [5:7587241322034070199:2108]
2025-12-24T02:11:20.354623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322034070199:2108] [node 6] ICS09 handshake done sender: [5:7587241322034070178:2107] self: [5:7587241322034070182:0] peer: [6:7587241320099567170:0] socket: 627 qp: -1
2025-12-24T02:11:20.354624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241322034070119:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17047063ns) {Configure:104(17046224ns) {ProcessConfigured:135(16766205ns) {StartInitialHandshake:157(16765865ns) {DropHandshakes:470(425ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(396ns) PrepareNewSessionHandshake:477(872ns) {SwitchToState:179(574ns)}}}}}
2025-12-24T02:11:20.354629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322034070199:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:20.354647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.355905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070197:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.356457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567193:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.356678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320099567176:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.356801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320099567107:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.356820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.356828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320099567107:2048] [node 1] ICP22 created new session: [6:7587241320099567194:2048]
2025-12-24T02:11:20.356836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320099567194:2048] [node 1] ICS09 handshake done sender: [6:7587241320099567176:2111] self: [6:7587241320099567188:0] peer: [1:7587241321024491823:0] socket: 637 qp: -1
2025-12-24T02:11:20.356850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320099567194:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.356857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.357826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070201:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.357907Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070202:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.359098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322718769932:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29467
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.359168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322718769933:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29467
2025-12-24T02:11:20.359394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241321823755781:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.359414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.359420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321823755781:2048] [node 1] ICP22 created new session: [2:7587241321823755865:2048]
2025-12-24T02:11:20.359426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321823755865:2048] [node 1] ICS09 handshake done sender: [2:7587241321823755842:2108] self: [2:7587241321823755856:0] peer: [1:7587241321024491815:0] socket: 636 qp: -1
2025-12-24T02:11:20.359432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321823755865:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.359463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.361023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321823755841:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.361479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769932:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.361527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241322718769842:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.361540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567197:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.362558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322034070180:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.362672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322034070118:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.362705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.362714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322034070118:2048] [node 1] ICP22 created new session: [5:7587241322034070204:2048]
2025-12-24T02:11:20.362720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322034070204:2048] [node 1] ICS09 handshake done sender: [5:7587241322034070180:2109] self: [5:7587241322034070194:0] peer: [1:7587241321024491821:0] socket: 638 qp: -1
2025-12-24T02:11:20.362726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322034070204:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.362734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.363113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491829:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.363127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321024491829:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.363169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241321024491845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22124
2025-12-24T02:11:20.363176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322718769931:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29467
2025-12-24T02:11:20.363230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321024491820:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.363257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.363267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321024491820:2109] [node 6] ICP22 created new session: [1:7587241321024491846:2109]
2025-12-24T02:11:20.363274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321024491846:2109] [node 6] ICS09 handshake done sender: [1:7587241321024491824:2112] self: [1:7587241321024491823:0] peer: [6:7587241320099567188:0] socket: 633 qp: -1
2025-12-24T02:11:20.363281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321024491846:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.363300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.363397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320099567192:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.363452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320099567109:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.363494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755863:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.363540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.363602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241321823755782:2049] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.363611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.363618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321823755782:2049] [node 3] ICP22 created new session: [2:7587241321823755868:2049]
2025-12-24T02:11:20.363623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321823755868:2049] [node 3] ICS09 handshake done sender: [2:7587241321823755841:2107] self: [2:7587241321823755848:0] peer: [3:7587241322718769902:0] socket: 635 qp: -1
2025-12-24T02:11:20.363628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321823755868:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.363634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.363817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769931:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.363875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036177:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.363912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241321284036099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.366000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036169:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241321284036169:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.365828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320099567190:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.365902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320099567181:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.366305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321024491829:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.366378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491830:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241321024491830:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.366390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070206:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.366449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322034070197:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322034070119:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.366567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241321284036169:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.366548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769923:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322718769923:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.366620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322034070202:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322034070206:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.366660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322034070190:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.366660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241321284036166:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.366684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.366686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322034070120:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.366712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241321284036166:2109] [node 6] ICP22 created new session: [4:7587241321284036188:2109]
2025-12-24T02:11:20.366722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241321284036188:2109] [node 6] ICS09 handshake done sender: [4:7587241321284036169:2111] self: [4:7587241321284036168:0] peer: [6:7587241320099567199:0] socket: 662 qp: -1
2025-12-24T02:11:20.366728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241321284036188:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.366734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.368130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241320099567181:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26061) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241320099567190:2115] held: no
2025-12-24T02:11:20.368581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755863:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.368681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241320099567197:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.368725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241320099567108:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.368986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241321024491845:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.369006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321024491752:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.369022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.369029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321024491752:2049] [node 3] ICP22 created new session: [1:7587241321024491849:2049]
2025-12-24T02:11:20.369036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321024491849:2049] [node 3] ICS09 handshake done sender: [1:7587241321024491829:2114] self: [1:7587241321024491816:0] peer: [3:7587241322718769934:0] socket: 656 qp: -1
2025-12-24T02:11:20.369042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321024491849:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.369049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.369770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241321024491751:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.369942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241321024491830:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.370053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241321024491812:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.370091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.370104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241321024491812:2107] [node 4] ICP22 created new session: [1:7587241321024491853:2107]
2025-12-24T02:11:20.370110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241321024491853:2107] [node 4] ICS09 handshake done sender: [1:7587241321024491830:2115] self: [1:7587241321024491817:0] peer: [4:7587241321284036186:0] socket: 657 qp: -1
2025-12-24T02:11:20.370117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241321024491853:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.370124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.370308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241322718769842:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241322718769932:2117] held: no
2025-12-24T02:11:20.370472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322718769923:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.370593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322718769915:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.370610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.370615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322718769915:2111] [node 6] ICP22 created new session: [3:7587241322718769938:2111]
2025-12-24T02:11:20.370619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322718769938:2111] [node 6] ICS09 handshake done sender: [3:7587241322718769923:2115] self: [3:7587241322718769917:0] peer: [6:7587241320099567198:0] socket: 676 qp: -1
2025-12-24T02:11:20.370599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567202:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.370624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322718769938:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:20.370636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.370724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322718769919:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.370717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755855:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.370731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322718769936:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29467
2025-12-24T02:11:20.370741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322718769919:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.370736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321823755855:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.370766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755860:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.370775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321823755860:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.370863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320099567192:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.370854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036171:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.370870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241321284036171:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.370922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567201:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
2025-12-24T02:11:20.370962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320099567109:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.370981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.370990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320099567109:2050] [node 3] ICP22 created new session: [6:7587241320099567203:2050]
2025-12-24T02:11:20.370997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320099567203:2050] [node 3] ICS09 handshake done sender: [6:7587241320099567192:2117] self: [6:7587241320099567198:0] peer: [3:7587241322718769917:0] socket: 677 qp: -1
2025-12-24T02:11:20.371007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320099567203:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.371013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.371244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320099567190:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.371696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036191:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.373409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036191:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.373442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241321284036100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.374840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241321024491820:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(18616591ns) {HandleHandshakeStatus:329(18615814ns) {DropIncomingHandshake:442(5642ns) DropOutgoingHandshake:457(251ns) SwitchToState:179(202ns) ProcessPendingSessionEvents:495(8140365ns) {ForwardSessionEventToSession:612(8128730ns) ForwardSessionEventToSession:612(2034ns) ForwardSessionEventToSession:612(786ns) ForwardSessionEventToSession:612(551ns) ForwardSessionEventToSession:612(610ns) ForwardSessionEventToSession:612(968ns) ForwardSessionEventToSession:612(867ns) ForwardSessionEventToSession:612(499ns) ForwardSessionEventToSession:612(548ns) ForwardSessionEventToSession:612(778ns)}}}
2025-12-24T02:11:20.374872Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241321024491819:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.375073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755862:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.375090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321823755862:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.375919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322718769932:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.376300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755864:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.376511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321823755855:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.376693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321823755850:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.376718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.376732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321823755850:2110] [node 5] ICP22 created new session: [2:7587241321823755874:2110]
2025-12-24T02:11:20.376744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321823755874:2110] [node 5] ICS09 handshake done sender: [2:7587241321823755855:2113] self: [2:7587241321823755851:0] peer: [5:7587241322034070207:0] socket: 647 qp: -1
2025-12-24T02:11:20.376751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321823755874:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:20.376776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.377495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320099567181:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.377514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.377521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320099567181:2112] [node 4] ICP22 created new session: [6:7587241320099567205:2112]
2025-12-24T02:11:20.377532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320099567205:2112] [node 4] ICS09 handshake done sender: [6:7587241320099567190:2115] self: [6:7587241320099567199:0] peer: [4:7587241321284036168:0] socket: 665 qp: -1
2025-12-24T02:11:20.377538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320099567205:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:20.377550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.377695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036193:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.377836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241322718769842:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.377854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.377861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322718769842:2048] [node 1] ICP22 created new session: [3:7587241322718769941:2048]
2025-12-24T02:11:20.377867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322718769941:2048] [node 1] ICS09 handshake done sender: [3:7587241322718769932:2117] self: [3:7587241322718769934:0] peer: [1:7587241321024491816:0] socket: 685 qp: -1
2025-12-24T02:11:20.377873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322718769941:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.377879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.377910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241321284036193:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.378207Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241321284036099:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27734) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241321284036177:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.379806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322718769919:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.379929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321823755860:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.380013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321823755862:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.380127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321823755853:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.380150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.380161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321823755853:2112] [node 6] ICP22 created new session: [2:7587241321823755879:2112]
2025-12-24T02:11:20.380167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321823755879:2112] [node 6] ICS09 handshake done sender: [2:7587241321823755860:2115] self: [2:7587241321823755854:0] peer: [6:7587241320099567200:0] socket: 661 qp: -1
2025-12-24T02:11:20.380174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321823755879:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:20.380182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.380271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241321284036171:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.380241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070211:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.380343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241322034070119:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241322034070197:2116] held: no
2025-12-24T02:11:20.380364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036195:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.380394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322034070197:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.380425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321823755877:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34833
2025-12-24T02:11:20.380949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322034070119:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.380970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.381027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241320099567207:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53491
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.381185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241320099567197:2119] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.380977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322034070119:2049] [node 2] ICP22 created new session: [5:7587241322034070212:2049]
2025-12-24T02:11:20.380980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241321284036164:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.380990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322034070212:2049] [node 2] ICS09 handshake done sender: [5:7587241322034070197:2116] self: [5:7587241322034070207:0] peer: [2:7587241321823755851:0] socket: 648 qp: -1
2025-12-24T02:11:20.381011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.381006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322034070212:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.381018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241321284036164:2108] [node 5] ICP22 created new session: [4:7587241321284036197:2108]
2025-12-24T02:11:20.381015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.381024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241321284036197:2108] [node 5] ICS09 handshake done sender: [4:7587241321284036171:2112] self: [4:7587241321284036170:0] peer: [5:7587241322034070208:0] socket: 663 qp: -1
2025-12-24T02:11:20.381184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070214:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.381029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241321284036197:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.381036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.381169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241321284036177:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.381336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241320099567108:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.381372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.381383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241320099567108:2049] [node 2] ICP22 created new session: [6:7587241320099567208:2049]
2025-12-24T02:11:20.381387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241320099567208:2049] [node 2] ICS09 handshake done sender: [6:7587241320099567197:2119] self: [6:7587241320099567200:0] peer: [2:7587241321823755854:0] socket: 688 qp: -1
2025-12-24T02:11:20.381393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241320099567208:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.381399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.381514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321823755849:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.381528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.381533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321823755849:2109] [node 4] ICP22 created new session: [2:7587241321823755881:2109]
2025-12-24T02:11:20.381539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321823755881:2109] [node 4] ICS09 handshake done sender: [2:7587241321823755862:2117] self: [2:7587241321823755859:0] peer: [4:7587241321284036192:0] socket: 643 qp: -1
2025-12-24T02:11:20.381545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321823755881:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:20.381552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.381702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322034070215:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28754
2025-12-24T02:11:20.381848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241321284036099:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.381865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.381871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241321284036099:2048] [node 1] ICP22 created new session: [4:7587241321284036199:2048]
2025-12-24T02:11:20.381858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321823755877:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.381877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241321284036199:2048] [node 1] ICS09 handshake done sender: [4:7587241321284036177:2115] self: [4:7587241321284036186:0] peer: [1:7587241321024491817:0] socket: 672 qp: -1
2025-12-24T02:11:20.381887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241321284036199:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.381894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.383085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322034070206:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.383122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322718769913:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.383154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.383308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241321284036201:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52581
2025-12-24T02:11:20.383264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241322034070190:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26061) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241322034070202:2119] held: no
2025-12-24T02:11:20.383307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322034070202:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.383163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322718769913:2110] [node 5] ICP22 created new session: [3:7587241322718769943:2110]
2025-12-24T02:11:20.383198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322718769943:2110] [node 5] ICS09 handshake done sender: [3:7587241322718769919:2113] self: [3:7587241322718769914:0] peer: [5:7587241322034070209:0] socket: 668 qp: -1
2025-12-24T02:11:20.383204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322718769943:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:20.383215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.383661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241321823755849:2109] [node 4] ICP18 handshake (actor [2:7587241321823755877:2120]) is too old
2025-12-24T02:11:20.383760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241321284036191:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.383515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322034070120:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.383543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.383550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322034070190:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.383551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322034070120:2050] [node 3] ICP22 created new session: [5:7587241322034070216:2050]
2025-12-24T02:11:20.383562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322034070216:2050] [node 3] ICS09 handshake done sender: [5:7587241322034070206:2120] self: [5:7587241322034070209:0] peer: [3:7587241322718769914:0] socket: 680 qp: -1
2025-12-24T02:11:20.383563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.383569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322034070190:2111] [node 4] ICP22 created new session: [5:7587241322034070217:2111]
2025-12-24T02:11:20.383571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322034070216:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.383573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322034070217:2111] [node 4] ICS09 handshake done sender: [5:7587241322034070202:2119] self: [5:7587241322034070208:0] peer: [4:7587241321284036170:0] socket: 678 qp: -1
2025-12-24T02:11:20.383579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322034070217:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:20.383587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.383602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.383819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241321284036100:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.383839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.383846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241321284036100:2049] [node 2] ICP22 created new session: [4:7587241321284036202:2049]
2025-12-24T02:11:20.383851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241321284036202:2049] [node 2] ICS09 handshake done sender: [4:7587241321284036191:2119] self: [4:7587241321284036192:0] peer: [2:7587241321823755859:0] socket: 660 qp: -1
2025-12-24T02:11:20.383857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241321284036202:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.383864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.391228Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.391332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241321823755865:2048] [node 1] ICS01 socket: 636 reason# EndOfStream
2025-12-24T02:11:20.391354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241321823755781:2048] [node 1] ICP30 unregister session Session# [2:7587241321823755865:2048] VirtualId# [2:7587241321823755856:0]
2025-12-24T02:11:20.391375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241321823755865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.391471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.391511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.391549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.391622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322718769941:2048] [node 1] ICS01 socket: 685 reason# EndOfStream
2025-12-24T02:11:20.391627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322718769842:2048] [node 1] ICP30 unregister session Session# [3:7587241322718769941:2048] VirtualId# [3:7587241322718769934:0]
2025-12-24T02:11:20.391633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322718769941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.391681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.391710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.392335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.392555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.392622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322034070204:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-12-24T02:11:20.392632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322034070118:2048] [node 1] ICP30 unregister session Session# [5:7587241322034070204:2048] VirtualId# [5:7587241322034070194:0]
2025-12-24T02:11:20.392636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322034070204:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.392481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241321284036199:2048] [node 1] ICS01 socket: 672 reason# EndOfStream
2025-12-24T02:11:20.392493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241321284036099:2048] [node 1] ICP30 unregister session Session# [4:7587241321284036199:2048] VirtualId# [4:7587241321284036186:0]
2025-12-24T02:11:20.392498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241321284036199:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.392553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.392598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.392673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.392703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.392753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.392847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320099567194:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:20.392861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320099567107:2048] [node 1] ICP30 unregister session Session# [6:7587241320099567194:2048] VirtualId# [6:7587241320099567188:0]
2025-12-24T02:11:20.392869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320099567194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.392924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.392961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.402990Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.403083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322718769921:2049] [node 2] ICS01 socket: 624 reason# EndOfStream
2025-12-24T02:11:20.403091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322718769843:2049] [node 2] ICP30 unregister session Session# [3:7587241322718769921:2049] VirtualId# [3:7587241322718769902:0]
2025-12-24T02:11:20.403099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322718769921:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.403146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.403168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.407475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.407582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241321284036202:2049] [node 2] ICS07 socket disconnect 660 reason# EndOfStream
2025-12-24T02:11:20.407596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241321284036202:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.407654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.407693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.407687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241321284036202:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:20.407801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322034070212:2049] [node 2] ICS01 socket: 648 reason# EndOfStream
2025-12-24T02:11:20.407809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322034070119:2049] [node 2] ICP30 unregister session Session# [5:7587241322034070212:2049] VirtualId# [5:7587241322034070207:0]
2025-12-24T02:11:20.407815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322034070212:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.407862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.407882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.408096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241321284036100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.408112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241321284036202:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.408117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241321284036100:2049] [node 2] ICP30 unregister session Session# [4:7587241321284036202:2049] VirtualId# [4:7587241321284036192:0]
2025-12-24T02:11:20.408155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.408170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241321284036100:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.408172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.407946Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.408029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241320099567208:2049] [node 2] ICS07 socket disconnect 688 reason# EndOfStream
2025-12-24T02:11:20.408035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320099567208:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.408081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.408104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241320099567208:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:20.408383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241320099567108:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.408399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320099567208:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.408404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320099567108:2049] [node 2] ICP30 unregister session Session# [6:7587241320099567208:2049] VirtualId# [6:7587241320099567200:0]
2025-12-24T02:11:20.408422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.408428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241320099567108:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.408430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.417930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241320099567108:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:20.417926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241321284036100:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:20.421635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:20.421735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241321284036183:2050] [node 3] ICS07 socket disconnect 673 reason# EndOfStream
2025-12-24T02:11:20.421747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241321284036183:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.421822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.421864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241321284036183:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:20.421782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:20.421861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241322034070216:2050] [node 3] ICS07 socket disconnect 680 reason# EndOfStream
2025-12-24T02:11:20.421875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322034070216:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.421924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.421950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241322034070216:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:20.422294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241321284036101:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.422311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241321284036183:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.422317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241321284036101:2050] [node 3] ICP30 unregister session Session# [4:7587241321284036183:2050] VirtualId# [4:7587241321284036180:0]
2025-12-24T02:11:20.422345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.422356Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241321284036101:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.422360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.422462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241322034070120:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.422473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322034070216:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.422477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322034070120:2050] [node 3] ICP30 unregister session Session# [5:7587241322034070216:2050] VirtualId# [5:7587241322034070209:0]
2025-12-24T02:11:20.422491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.422495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241322034070120:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.422497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7027) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.423522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:20.423608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320099567203:2050] [node 3] ICS01 socket: 677 reason# EndOfStream
2025-12-24T02:11:20.423616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320099567109:2050] [node 3] ICP30 unregister session Session# [6:7587241320099567203:2050] VirtualId# [6:7587241320099567198:0]
2025-12-24T02:11:20.423623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320099567203:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.423680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.423715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.432651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241322034070120:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:20.435217Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:20.435325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322034070217:2111] [node 4] ICS01 socket: 678 reason# EndOfStream
2025-12-24T02:11:20.435339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322034070190:2111] [node 4] ICP30 unregister session Session# [5:7587241322034070217:2111] VirtualId# [5:7587241322034070208:0]
2025-12-24T02:11:20.435346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322034070217:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.435519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.435552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.435659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:20.435746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320099567205:2112] [node 4] ICS01 socket: 665 reason# EndOfStream
2025-12-24T02:11:20.435754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320099567181:2112] [node 4] ICP30 unregister session Session# [6:7587241320099567205:2112] VirtualId# [6:7587241320099567199:0]
2025-12-24T02:11:20.435761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320099567205:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.435799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.435818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.451602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:20.451693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241320099567182:2107] [node 5] ICS01 socket: 625 reason# EndOfStream
2025-12-24T02:11:20.451713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241320099567168:2107] [node 5] ICP30 unregister session Session# [6:7587241320099567182:2107] VirtualId# [6:7587241320099567170:0]
2025-12-24T02:11:20.451721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241320099567182:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.451760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:20.451781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:20.492081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321357136321:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.492210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321357136320:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.494104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241323064123634:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.494113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241323064123635:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.496544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241323668268528:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.496552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241323668268529:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.496556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241323668268530:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.499166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322201425633:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.499173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322201425634:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.499178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322201425635:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.489472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241319967284250:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.489836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241319967284249:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:20.502424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241322975897340:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.502441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241322975897342:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.507293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241322975897341:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.518289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241322975897401:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.519481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425695:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.519641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241323064123695:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.519753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123696:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
2025-12-24T02:11:20.520152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241319967284317:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:20.520282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241319967284319:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:20.520394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241319967284320:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:20.521526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321357136383:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.538775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241319967284326:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24093
2025-12-24T02:11:20.539601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425695:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.546859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241319967284319:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20012378ns) {Configure:104(20010721ns) {ProcessConfigured:135(19584472ns) {StartInitialHandshake:157(19582978ns) {DropHandshakes:470(1121ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1543ns) {SwitchToState:179(954ns)}}}}}
2025-12-24T02:11:20.546965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241323668268528:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19905777ns) {Configure:104(19904749ns) {ProcessConfigured:135(19560447ns) {StartInitialHandshake:157(19560131ns) {DropHandshakes:470(729ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(589ns) {SwitchToState:179(357ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.550935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241321357136321:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29803878ns) {Configure:104(27291959ns) {ProcessConfigured:135(27005591ns) {StartInitialHandshake:157(27004410ns) {DropHandshakes:470(686ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(916ns) {SwitchToState:179(629ns)}}}}}
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.551427Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241322975897340:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29582596ns) {Configure:104(29581759ns) {ProcessConfigured:135(29230194ns) {StartInitialHandshake:157(29229882ns) {DropHandshakes:470(494ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(1201ns) {SwitchToState:179(965ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.551119Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241321357136383:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21336578ns) {Configure:104(21335787ns) {ProcessConfigured:135(21015948ns) {StartInitialHandshake:157(21015467ns) {DropHandshakes:470(511ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(598ns) {SwitchToState:179(176ns)}}}}}
2025-12-24T02:11:20.551213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136395:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
2025-12-24T02:11:20.551234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136393:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
2025-12-24T02:11:20.551278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136396:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
2025-12-24T02:11:20.551295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136392:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.551663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241322201425635:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22980160ns) {Configure:104(22979649ns) {ProcessConfigured:135(22719005ns) {StartInitialHandshake:157(22718583ns) {DropHandshakes:470(305ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(484ns) {SwitchToState:179(109ns)}}}}}
2025-12-24T02:11:20.551613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897416:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.552329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241322975897416:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.552398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241322975897340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.552616Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284324:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.552641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241319967284324:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.553236Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241319967284324:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.553281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897418:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.553363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241322975897416:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.553459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241322975897340:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.553459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241319967284320:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.553481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.553487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.553489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241319967284320:2111] [node 6] ICP22 created new session: [1:7587241319967284329:2111]
2025-12-24T02:11:20.553501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241322975897340:2048] [node 1] ICP22 created new session: [6:7587241322975897419:2048]
2025-12-24T02:11:20.553506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241322975897419:2048] [node 1] ICS09 handshake done sender: [6:7587241322975897416:2114] self: [6:7587241322975897417:0] peer: [1:7587241319967284323:0] socket: 645 qp: -1
2025-12-24T02:11:20.553497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241319967284329:2111] [node 6] ICS09 handshake done sender: [1:7587241319967284324:2112] self: [1:7587241319967284323:0] peer: [6:7587241322975897417:0] socket: 644 qp: -1
2025-12-24T02:11:20.553514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241322975897419:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.553505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241319967284329:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:20.553524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.553522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.553748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425708:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.550956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268602:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.551684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268603:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.554937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123696:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.555042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241323064123634:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.555165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123709:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.555244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123698:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.555248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241323064123698:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.555725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123709:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.554816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136392:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.554825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136393:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.554851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136395:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.554855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136396:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.554924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241321357136320:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.555959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241323064123698:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.556072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241323064123695:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.556093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.556099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241323064123695:2107] [node 4] ICP22 created new session: [3:7587241323064123713:2107]
2025-12-24T02:11:20.556118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241323064123713:2107] [node 4] ICS09 handshake done sender: [3:7587241323064123698:2109] self: [3:7587241323064123697:0] peer: [4:7587241323668268604:0] socket: 640 qp: -1
2025-12-24T02:11:20.556124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241323064123713:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.556131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.555069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268602:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.555117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241323668268530:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.558782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241323668268599:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:20.558856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268605:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.558869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241319967284317:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31618400ns) {Configure:104(31617507ns) {ProcessConfigured:135(31417591ns) {StartInitialHandshake:157(31416568ns) {DropHandshakes:470(169ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(1008ns) {SwitchToState:179(626ns)}}}}}
2025-12-24T02:11:20.559008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241323668268530:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241323668268602:2113] held: no
2025-12-24T02:11:20.559037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241323668268602:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.558763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321357136384:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.561392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284310:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.561411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241319967284310:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.561497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241323064123704:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:20.561515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123716:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.561608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241323064123708:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:20.561492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284313:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.561509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241319967284313:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.561354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241321357136389:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.561954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123716:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.562182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425709:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.561340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241323668268593:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:20.561399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241323668268530:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.561411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.561418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241323668268530:2050] [node 3] ICP22 created new session: [4:7587241323668268607:2050]
2025-12-24T02:11:20.561426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241323668268607:2050] [node 3] ICS09 handshake done sender: [4:7587241323668268602:2113] self: [4:7587241323668268604:0] peer: [3:7587241323064123697:0] socket: 641 qp: -1
2025-12-24T02:11:20.561434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241323668268607:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.561455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.562924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241319967284335:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24093
2025-12-24T02:11:20.562966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284326:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.563077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425708:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.563111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322201425633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.563201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284327:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.563208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241319967284327:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.563455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425709:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.563478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322201425635:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.563787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241319967284327:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.563869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425712:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.563984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241319967284319:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.564004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.564013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241319967284319:2110] [node 5] ICP22 created new session: [1:7587241319967284338:2110]
2025-12-24T02:11:20.564024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241319967284338:2110] [node 5] ICS09 handshake done sender: [1:7587241319967284327:2114] self: [1:7587241319967284321:0] peer: [5:7587241322201425710:0] socket: 667 qp: -1
2025-12-24T02:11:20.564032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241319967284338:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:20.564042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.564020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322201425708:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.564906Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241322201425633:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241322201425708:2114] held: no
2025-12-24T02:11:20.564929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322201425633:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.564937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.564943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322201425633:2048] [node 1] ICP22 created new session: [5:7587241322201425713:2048]
2025-12-24T02:11:20.564948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322201425713:2048] [node 1] ICS09 handshake done sender: [5:7587241322201425708:2114] self: [5:7587241322201425710:0] peer: [1:7587241319967284321:0] socket: 668 qp: -1
2025-12-24T02:11:20.564954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322201425713:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.564962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.567186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241319967284332:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24093
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.567190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241319967284334:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24093
2025-12-24T02:11:20.567211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123717:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
2025-12-24T02:11:20.567206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322201425705:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:20.567508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241322201425704:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:20.567176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241322975897414:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:20.567729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284332:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.567861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123720:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.567878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241323064123720:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.571007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284334:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.572244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241319967284310:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.572650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136407:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.572922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136407:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.573345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284333:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.573355Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241319967284333:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.574034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241319967284333:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.574134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241319967284317:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.574158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.574166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241319967284317:2109] [node 4] ICP22 created new session: [1:7587241319967284345:2109]
2025-12-24T02:11:20.574179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241319967284345:2109] [node 4] ICS09 handshake done sender: [1:7587241319967284333:2116] self: [1:7587241319967284322:0] peer: [4:7587241323668268614:0] socket: 648 qp: -1
2025-12-24T02:11:20.574186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241319967284345:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:20.574200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.572995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268613:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.573208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268613:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.573240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241323668268528:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.574709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268615:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.575410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241323064123720:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.575536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241323064123704:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.575551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.575560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241323064123704:2110] [node 5] ICP22 created new session: [3:7587241323064123725:2110]
2025-12-24T02:11:20.575567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241323064123725:2110] [node 5] ICS09 handshake done sender: [3:7587241323064123720:2117] self: [3:7587241323064123718:0] peer: [5:7587241322201425711:0] socket: 677 qp: -1
2025-12-24T02:11:20.575574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241323064123725:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:20.575580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.576092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123728:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
2025-12-24T02:11:20.576381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123728:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.576407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241323064123704:2110] [node 5] ICP18 handshake (actor [3:7587241323064123728:2119]) is too old
No RDMA context found for GID: No RDMA context found for GID: No RDMA context found for GID: 2025-12-24T02:11:20.577181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241319967284313:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:20.577275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.577292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241319967284249:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.577313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.577322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241319967284249:2048] [node 2] ICP22 created new session: [1:7587241319967284348:2048]
2025-12-24T02:11:20.577328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241319967284348:2048] [node 2] ICS09 handshake done sender: [1:7587241319967284310:2107] self: [1:7587241319967284309:0] peer: [2:7587241321357136400:0] socket: 630 qp: -1
2025-12-24T02:11:20.577336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241319967284348:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:20.577349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.577429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241322975897425:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.577476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241322975897401:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.577418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241319967284250:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.577442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.577449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241319967284250:2049] [node 3] ICP22 created new session: [1:7587241319967284350:2049]
2025-12-24T02:11:20.577462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241319967284350:2049] [node 3] ICS09 handshake done sender: [1:7587241319967284313:2108] self: [1:7587241319967284312:0] peer: [3:7587241323064123710:0] socket: 632 qp: -1
2025-12-24T02:11:20.577471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241319967284350:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.577504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.577506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241322201425635:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241322201425709:2115] held: no
2025-12-24T02:11:20.577593Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425717:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.577597Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241322201425717:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.575033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268615:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.575346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268616:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.575453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241323668268613:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.575508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241323668268528:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.575520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.575526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241323668268528:2048] [node 1] ICP22 created new session: [4:7587241323668268618:2048]
2025-12-24T02:11:20.575532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241323668268618:2048] [node 1] ICS09 handshake done sender: [4:7587241323668268613:2118] self: [4:7587241323668268614:0] peer: [1:7587241319967284322:0] socket: 647 qp: -1
2025-12-24T02:11:20.575541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241323668268618:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.575548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.578999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268620:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:20.579200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425723:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.579221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425722:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.580173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123717:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.580214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241323064123635:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.579695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268620:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.580486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241319967284335:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.580529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241319967284249:2048] [node 2] ICP18 handshake (actor [1:7587241319967284335:2118]) is too old
2025-12-24T02:11:20.580502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425723:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.580544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322201425705:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.580312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136390:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.580322Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321357136390:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.581600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123731:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
2025-12-24T02:11:20.581762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241323064123717:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.581528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321357136390:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.581637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321357136321:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.581715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.581722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321357136321:2049] [node 3] ICP22 created new session: [2:7587241321357136411:2049]
2025-12-24T02:11:20.581729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321357136411:2049] [node 3] ICS09 handshake done sender: [2:7587241321357136390:2111] self: [2:7587241321357136386:0] peer: [3:7587241323064123730:0] socket: 660 qp: -1
2025-12-24T02:11:20.581740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321357136411:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.581749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.581973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897428:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.581891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241323064123635:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.581907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.581913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241323064123635:2049] [node 2] ICP22 created new session: [3:7587241323064123732:2049]
2025-12-24T02:11:20.581918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241323064123732:2049] [node 2] ICS09 handshake done sender: [3:7587241323064123717:2116] self: [3:7587241323064123730:0] peer: [2:7587241321357136386:0] socket: 674 qp: -1
2025-12-24T02:11:20.581923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241323064123732:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.581933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.582760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322201425709:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.583074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322201425635:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.583091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.583099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322201425635:2050] [node 3] ICP22 created new session: [5:7587241322201425725:2050]
2025-12-24T02:11:20.583105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322201425725:2050] [node 3] ICS09 handshake done sender: [5:7587241322201425709:2115] self: [5:7587241322201425711:0] peer: [3:7587241323064123718:0] socket: 678 qp: -1
2025-12-24T02:11:20.583111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322201425725:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.583128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.583303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897424:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.583327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897426:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.583309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241323064123729:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.583514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241323064123696:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.583622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241323064123634:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.583640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.583646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241323064123634:2048] [node 1] ICP22 created new session: [3:7587241323064123734:2048]
2025-12-24T02:11:20.583653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241323064123734:2048] [node 1] ICS09 handshake done sender: [3:7587241323064123696:2108] self: [3:7587241323064123710:0] peer: [1:7587241319967284312:0] socket: 638 qp: -1
2025-12-24T02:11:20.583659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241323064123734:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.583675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.583579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241322975897424:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.583622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241322975897414:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.584533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322201425717:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.584543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897430:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.584633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241322975897425:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.584671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241322201425704:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.584703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.584720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322201425704:2112] [node 6] ICP22 created new session: [5:7587241322201425728:2112]
2025-12-24T02:11:20.584731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322201425728:2112] [node 6] ICS09 handshake done sender: [5:7587241322201425717:2117] self: [5:7587241322201425715:0] peer: [6:7587241322975897427:0] socket: 685 qp: -1
2025-12-24T02:11:20.584725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241322975897401:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.584738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322201425728:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:20.584745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.584750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241322975897401:2107] [node 5] ICP22 created new session: [6:7587241322975897431:2107]
2025-12-24T02:11:20.584755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241322975897431:2107] [node 5] ICS09 handshake done sender: [6:7587241322975897425:2118] self: [6:7587241322975897427:0] peer: [5:7587241322201425715:0] socket: 693 qp: -1
2025-12-24T02:11:20.584754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.584761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241322975897431:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.584767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.585760Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241321357136320:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241321357136392:2112] held: no
2025-12-24T02:11:20.586055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241322975897426:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.586100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241322975897342:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.583340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268612:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.583362Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241323668268612:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.585965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268603:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.586008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241323668268529:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.586039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241323668268610:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.586047Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241323668268610:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.586038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241321357136413:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45832
2025-12-24T02:11:20.586142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321357136392:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.587174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241321357136320:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.587197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.587208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321357136320:2048] [node 1] ICP22 created new session: [2:7587241321357136414:2048]
2025-12-24T02:11:20.587215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321357136414:2048] [node 1] ICS09 handshake done sender: [2:7587241321357136392:2112] self: [2:7587241321357136400:0] peer: [1:7587241319967284309:0] socket: 642 qp: -1
2025-12-24T02:11:20.587225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321357136414:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.587238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.587316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425730:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.587432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136394:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.587460Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321357136394:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.587633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241323064123721:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.587648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241323064123721:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.587680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241322975897428:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.587739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241322975897341:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.587840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322201425723:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.588325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241323064123634:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.590288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241323064123721:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.590378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241323064123708:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.590406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.590413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241323064123708:2111] [node 6] ICP22 created new session: [3:7587241323064123737:2111]
2025-12-24T02:11:20.590420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241323064123737:2111] [node 6] ICS09 handshake done sender: [3:7587241323064123721:2118] self: [3:7587241323064123719:0] peer: [6:7587241322975897433:0] socket: 669 qp: -1
2025-12-24T02:11:20.590433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241323064123737:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:20.590446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.590644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241322975897342:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13590) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241322975897426:2119] held: no
2025-12-24T02:11:20.587720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241323668268612:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.587822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241323668268593:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.587848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.587856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241323668268593:2109] [node 5] ICP22 created new session: [4:7587241323668268625:2109]
2025-12-24T02:11:20.587862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241323668268625:2109] [node 5] ICS09 handshake done sender: [4:7587241323668268612:2117] self: [4:7587241323668268609:0] peer: [5:7587241322201425724:0] socket: 695 qp: -1
2025-12-24T02:11:20.587868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241323668268625:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:20.587877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.590637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241323668268610:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.591359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321357136394:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.591499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321357136383:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.591519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.591525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321357136383:2107] [node 4] ICP22 created new session: [2:7587241321357136418:2107]
No RDMA context found for GID: 2025-12-24T02:11:20.591531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321357136418:2107] [node 4] ICS09 handshake done sender: [2:7587241321357136394:2114] self: [2:7587241321357136388:0] peer: [4:7587241323668268623:0] socket: 661 qp: -1
::ffff:127.1.0.0
2025-12-24T02:11:20.591537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321357136418:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.591543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.591639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322201425705:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.591661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.591668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322201425705:2111] [node 4] ICP22 created new session: [5:7587241322201425731:2111]
2025-12-24T02:11:20.591672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322201425731:2111] [node 4] ICS09 handshake done sender: [5:7587241322201425723:2120] self: [5:7587241322201425724:0] peer: [4:7587241323668268609:0] socket: 696 qp: -1
2025-12-24T02:11:20.591677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322201425731:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:20.591689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.591795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136403:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.591807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321357136403:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.592482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241322975897414:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241322975897424:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.590826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241323668268599:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.590847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.590853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241323668268599:2110] [node 6] ICP22 created new session: [4:7587241323668268628:2110]
2025-12-24T02:11:20.590859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241323668268628:2110] [node 6] ICS09 handshake done sender: [4:7587241323668268610:2116] self: [4:7587241323668268606:0] peer: [6:7587241322975897429:0] socket: 646 qp: -1
2025-12-24T02:11:20.590865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241323668268628:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.590871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.592898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241323668268529:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2227) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241323668268603:2114] held: no
2025-12-24T02:11:20.595930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241323668268630:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45587
2025-12-24T02:11:20.596083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241323668268603:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.597299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321357136403:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.597422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321357136389:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.597448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.597456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321357136389:2109] [node 6] ICP22 created new session: [2:7587241321357136421:2109]
2025-12-24T02:11:20.597463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321357136421:2109] [node 6] ICS09 handshake done sender: [2:7587241321357136403:2117] self: [2:7587241321357136402:0] peer: [6:7587241322975897434:0] socket: 699 qp: -1
2025-12-24T02:11:20.597470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321357136421:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.597537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.597539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897435:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.597592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241323668268529:2049] [node 2] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:20.597613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.02025-12-24T02:11:20.597621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241323668268529:2049] [node 2] ICP22 created new session: [4:7587241323668268631:2049]
2025-12-24T02:11:20.597628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241323668268631:2049] [node 2] ICS09 handshake done sender: [4:7587241323668268603:2114] self: [4:7587241323668268623:0] peer: [2:7587241321357136388:0] socket: 662 qp: -1
2025-12-24T02:11:20.597640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241323668268631:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.597650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.597696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241322975897424:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.597714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897436:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.598390Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241322201425705:2111] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.598411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241322975897414:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.598432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.598440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241322975897414:2110] [node 4] ICP22 created new session: [6:7587241322975897437:2110]
2025-12-24T02:11:20.598447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241322975897437:2110] [node 4] ICS09 handshake done sender: [6:7587241322975897424:2117] self: [6:7587241322975897429:0] peer: [4:7587241323668268606:0] socket: 683 qp: -1
2025-12-24T02:11:20.598453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241322975897437:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:20.598489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.598617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241322975897341:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2227) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241322975897428:2120] held: no
2025-12-24T02:11:20.598762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241322975897426:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.598875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241322975897342:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.598893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.598899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241322975897342:2050] [node 3] ICP22 created new session: [6:7587241322975897439:2050]
2025-12-24T02:11:20.598903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241322975897439:2050] [node 3] ICS09 handshake done sender: [6:7587241322975897426:2119] self: [6:7587241322975897433:0] peer: [3:7587241323064123719:0] socket: 694 qp: -1
2025-12-24T02:11:20.598909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241322975897439:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.598920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.600371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241322975897441:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56434
2025-12-24T02:11:20.600522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241322975897428:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.600589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241322975897341:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.600603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.600617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241322975897341:2049] [node 2] ICP22 created new session: [6:7587241322975897442:2049]
2025-12-24T02:11:20.600623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241322975897442:2049] [node 2] ICS09 handshake done sender: [6:7587241322975897428:2120] self: [6:7587241322975897434:0] peer: [2:7587241321357136402:0] socket: 700 qp: -1
2025-12-24T02:11:20.600629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241322975897442:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.600640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.600896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425733:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.601873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241322201425733:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.601928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241322201425634:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.602158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241321357136405:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.602174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241321357136405:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.603039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241322201425735:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30002
2025-12-24T02:11:20.603426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241321357136405:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.603558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241321357136384:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.603583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.603591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241321357136384:2108] [node 5] ICP22 created new session: [2:7587241321357136424:2108]
2025-12-24T02:11:20.603596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241321357136424:2108] [node 5] ICS09 handshake done sender: [2:7587241321357136405:2118] self: [2:7587241321357136401:0] peer: [5:7587241322201425734:0] socket: 656 qp: -1
2025-12-24T02:11:20.603602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241321357136424:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.603621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.603584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241322201425733:2122] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.603725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241322201425634:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.603747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.603761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241322201425634:2049] [node 2] ICP22 created new session: [5:7587241322201425736:2049]
2025-12-24T02:11:20.603767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241322201425736:2049] [node 2] ICS09 handshake done sender: [5:7587241322201425733:2122] self: [5:7587241322201425734:0] peer: [2:7587241321357136401:0] socket: 663 qp: -1
2025-12-24T02:11:20.603773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241322201425736:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.603784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.611031Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.611114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241321357136414:2048] [node 1] ICS01 socket: 642 reason# EndOfStream
2025-12-24T02:11:20.611131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241321357136320:2048] [node 1] ICP30 unregister session Session# [2:7587241321357136414:2048] VirtualId# [2:7587241321357136400:0]
2025-12-24T02:11:20.611139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241321357136414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.611187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.611218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.612454Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.612516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322201425713:2048] [node 1] ICS01 socket: 668 reason# EndOfStream
2025-12-24T02:11:20.612523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322201425633:2048] [node 1] ICP30 unregister session Session# [5:7587241322201425713:2048] VirtualId# [5:7587241322201425710:0]
2025-12-24T02:11:20.612527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322201425713:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.612574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.612594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.613841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.614133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.614212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241323668268618:2048] [node 1] ICS07 socket disconnect 647 reason# EndOfStream
2025-12-24T02:11:20.614223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241323668268618:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.614272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.614294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241323668268618:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.614633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241323668268528:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.614646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241323668268618:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.614753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.614661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241323668268528:2048] [node 1] ICP30 unregister session Session# [4:7587241323668268618:2048] VirtualId# [4:7587241323668268614:0]
2025-12-24T02:11:20.614674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.614687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241323668268528:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.614747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.614826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241323064123734:2048] [node 1] ICS07 socket disconnect 638 reason# EndOfStream
2025-12-24T02:11:20.614836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241323064123734:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.614885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.614915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241323064123734:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.615352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241322975897419:2048] [node 1] ICS07 socket disconnect 645 reason# EndOfStream
2025-12-24T02:11:20.615366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241322975897419:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.615421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.615452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241322975897419:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.615822Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241322975897340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.615833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241322975897419:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.615837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241322975897340:2048] [node 1] ICP30 unregister session Session# [6:7587241322975897419:2048] VirtualId# [6:7587241322975897417:0]
2025-12-24T02:11:20.615851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.615856Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241322975897340:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.615858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.615887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241323064123634:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.615901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241323064123734:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.615907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241323064123634:2048] [node 1] ICP30 unregister session Session# [3:7587241323064123734:2048] VirtualId# [3:7587241323064123710:0]
2025-12-24T02:11:20.615923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.615928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241323064123634:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.615930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.624774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241323668268528:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.625196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.625290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241323064123732:2049] [node 2] ICS01 socket: 674 reason# EndOfStream
2025-12-24T02:11:20.625302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241323064123635:2049] [node 2] ICP30 unregister session Session# [3:7587241323064123732:2049] VirtualId# [3:7587241323064123730:0]
2025-12-24T02:11:20.625308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241323064123732:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.625367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.625393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.625959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241323064123634:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.627191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.627252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241323668268631:2049] [node 2] ICS01 socket: 662 reason# EndOfStream
2025-12-24T02:11:20.627262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241323668268529:2049] [node 2] ICP30 unregister session Session# [4:7587241323668268631:2049] VirtualId# [4:7587241323668268623:0]
2025-12-24T02:11:20.627267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241323668268631:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.627288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241322975897340:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.627312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.627357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.627576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.627649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241322975897442:2049] [node 2] ICS01 socket: 700 reason# EndOfStream
2025-12-24T02:11:20.627655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241322975897341:2049] [node 2] ICP30 unregister session Session# [6:7587241322975897442:2049] VirtualId# [6:7587241322975897434:0]
2025-12-24T02:11:20.627660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241322975897442:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.627697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.627718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.627719Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.627781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322201425736:2049] [node 2] ICS01 socket: 663 reason# EndOfStream
2025-12-24T02:11:20.627786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322201425634:2049] [node 2] ICP30 unregister session Session# [5:7587241322201425736:2049] VirtualId# [5:7587241322201425734:0]
2025-12-24T02:11:20.627791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322201425736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.627826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.627847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.641846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:20.641941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241323668268607:2050] [node 3] ICS01 socket: 641 reason# EndOfStream
2025-12-24T02:11:20.641959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241323668268530:2050] [node 3] ICP30 unregister session Session# [4:7587241323668268607:2050] VirtualId# [4:7587241323668268604:0]
2025-12-24T02:11:20.641965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241323668268607:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.642019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.642051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.642398Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:20.642472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241322975897439:2050] [node 3] ICS01 socket: 694 reason# EndOfStream
2025-12-24T02:11:20.642481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241322975897342:2050] [node 3] ICP30 unregister session Session# [6:7587241322975897439:2050] VirtualId# [6:7587241322975897433:0]
2025-12-24T02:11:20.642486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241322975897439:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.642532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.642563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.642736Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:20.642818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241322201425725:2050] [node 3] ICS01 socket: 678 reason# EndOfStream
2025-12-24T02:11:20.642825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241322201425635:2050] [node 3] ICP30 unregister session Session# [5:7587241322201425725:2050] VirtualId# [5:7587241322201425711:0]
2025-12-24T02:11:20.642830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241322201425725:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.642875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.642901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.653779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:20.653859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241322975897437:2110] [node 4] ICS01 socket: 683 reason# EndOfStream
2025-12-24T02:11:20.653867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241322975897414:2110] [node 4] ICP30 unregister session Session# [6:7587241322975897437:2110] VirtualId# [6:7587241322975897429:0]
2025-12-24T02:11:20.653872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241322975897437:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.653923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.653952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.668590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:20.668679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241322975897431:2107] [node 5] ICS01 socket: 693 reason# EndOfStream
2025-12-24T02:11:20.668687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241322975897401:2107] [node 5] ICP30 unregister session Session# [6:7587241322975897431:2107] VirtualId# [6:7587241322975897427:0]
2025-12-24T02:11:20.668694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241322975897431:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.668750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:20.668773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
Error.size# 1
2025-12-24T02:11:20.700122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241323232161973:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:20.700134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241323232161974:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.701328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320394217759:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.701338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320394217760:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.704251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319901512890:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.704264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319901512891:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.704268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319901512892:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.706866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241320411944101:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.706879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241320411944102:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.706884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241320411944103:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.709334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323195275077:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.709347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323195275078:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.709351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323195275079:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.718821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322994635877:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.718840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322994635878:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.720396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323195275138:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.721312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944162:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.721387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241323232162033:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:20.721527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241323232162035:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:20.724363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241323232162044:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.725080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241320411944162:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.725501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320394217823:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:20.725614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320394217824:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:20.725716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241320411944164:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.726328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241320411944164:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.728646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323195275140:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.728668Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241323195275140:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.731592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241320411944167:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:20.735599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241320394217827:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.748022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217826:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.751053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322994635942:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:20.751195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322994635943:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:20.751345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322994635944:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:20.752787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635940:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.752877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635941:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.754755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323195275144:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:20.756382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323195275140:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.756517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241323195275138:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.756528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.756536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323195275138:2107] [node 5] ICP22 created new session: [6:7587241323195275156:2107]
2025-12-24T02:11:20.756542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323195275156:2107] [node 5] ICS09 handshake done sender: [6:7587241323195275140:2108] self: [6:7587241323195275139:0] peer: [5:7587241320411944165:0] socket: 36 qp: -1
2025-12-24T02:11:20.756548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323195275156:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.756557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.759879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241323232162052:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48158
2025-12-24T02:11:20.759899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241323232162053:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48158
2025-12-24T02:11:20.759471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944176:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.759496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944177:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.759896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217826:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.763733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241319901512892:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27938269ns) {Configure:104(27936850ns) {ProcessConfigured:135(27609429ns) {StartInitialHandshake:157(27608896ns) {DropHandshakes:470(537ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(1685ns) {SwitchToState:179(112ns)}}}}}
2025-12-24T02:11:20.763784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162053:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.763909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319901512960:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:20.764026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241319901512961:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:20.764415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241320411944162:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.764534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241320411944164:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.764551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.764558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241320411944164:2109] [node 6] ICP22 created new session: [5:7587241320411944181:2109]
2025-12-24T02:11:20.764568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241320411944181:2109] [node 6] ICS09 handshake done sender: [5:7587241320411944162:2108] self: [5:7587241320411944165:0] peer: [6:7587241323195275139:0] socket: 37 qp: -1
2025-12-24T02:11:20.764574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241320411944181:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.764594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.769509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217839:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
2025-12-24T02:11:20.769576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217840:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
2025-12-24T02:11:20.769607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635940:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.770768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512962:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.770821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512963:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.770856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512964:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
2025-12-24T02:11:20.773374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241320394217759:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.773447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217840:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.773590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217841:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.773629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217842:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
2025-12-24T02:11:20.775069Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217841:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.775310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.775474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162052:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.775548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162031:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.775551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241323232162031:2104] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.775959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241323232162055:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48158
2025-12-24T02:11:20.775992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241323232162056:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48158
2025-12-24T02:11:20.776113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162055:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.776151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241323232162031:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.776344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217839:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.775296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241320411944176:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.775334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241320411944103:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.776231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241323232161973:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.776248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.776254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241323232161973:2048] [node 2] ICP22 created new session: [1:7587241323232162059:2048]
2025-12-24T02:11:20.776497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241320394217846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64569
2025-12-24T02:11:20.776258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241323232162059:2048] [node 2] ICS09 handshake done sender: [1:7587241323232162031:2104] self: [1:7587241323232162030:0] peer: [2:7587241320394217843:0] socket: 39 qp: -1
2025-12-24T02:11:20.776264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241323232162059:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:20.776271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.776743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320394217826:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.777962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241320394217759:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.777978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.777983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320394217759:2048] [node 1] ICP22 created new session: [2:7587241320394217848:2048]
2025-12-24T02:11:20.777987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320394217848:2048] [node 1] ICS09 handshake done sender: [2:7587241320394217826:2111] self: [2:7587241320394217843:0] peer: [1:7587241323232162030:0] socket: 40 qp: -1
2025-12-24T02:11:20.777993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320394217848:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.777999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.778119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241320394217759:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.777912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635959:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.777980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.777997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241322994635877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.778027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635950:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.778030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322994635950:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.778303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635958:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.778321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635941:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.778426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241322994635878:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.778716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162037:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.778725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241323232162037:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.779370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241323232162037:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.779481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322994635950:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.779621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322994635943:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.779635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.779641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322994635943:2108] [node 5] ICP22 created new session: [3:7587241322994635966:2108]
2025-12-24T02:11:20.779647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322994635966:2108] [node 5] ICS09 handshake done sender: [3:7587241322994635950:2114] self: [3:7587241322994635949:0] peer: [5:7587241320411944183:0] socket: 53 qp: -1
2025-12-24T02:11:20.779652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322994635966:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:20.779659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.779989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635965:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.780182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322994635940:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.781243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241322994635877:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.781264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.781271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322994635877:2048] [node 1] ICP22 created new session: [3:7587241322994635969:2048]
2025-12-24T02:11:20.781277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322994635969:2048] [node 1] ICS09 handshake done sender: [3:7587241322994635940:2110] self: [3:7587241322994635961:0] peer: [1:7587241323232162036:0] socket: 44 qp: -1
2025-12-24T02:11:20.781284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322994635969:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.781291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.781983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241323232161974:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.781998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.782004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241323232161974:2049] [node 3] ICP22 created new session: [1:7587241323232162065:2049]
2025-12-24T02:11:20.782009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241323232162065:2049] [node 3] ICS09 handshake done sender: [1:7587241323232162037:2107] self: [1:7587241323232162036:0] peer: [3:7587241322994635961:0] socket: 41 qp: -1
2025-12-24T02:11:20.782014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241323232162065:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.782020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.782494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241323232162064:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48158
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.782917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512963:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.783014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512964:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.783037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512962:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.783061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217831:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.783071Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320394217831:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.783102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319901512891:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.783118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319901512892:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.783132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241319901512890:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.783426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635948:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.783435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322994635948:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.784132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635960:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.782867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275159:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.784316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322994635948:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.784971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162056:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.785013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217842:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.785040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512974:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.785169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944184:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.785237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.785198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319901512964:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.785375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241320411944176:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.785432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241320411944103:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.785448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.785454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241320411944103:2050] [node 3] ICP22 created new session: [5:7587241320411944186:2050]
2025-12-24T02:11:20.785462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241320411944186:2050] [node 3] ICS09 handshake done sender: [5:7587241320411944176:2115] self: [5:7587241320411944183:0] peer: [3:7587241322994635949:0] socket: 59 qp: -1
2025-12-24T02:11:20.785472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241320411944186:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.785480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.786328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319901512892:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.786348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.786354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319901512892:2050] [node 3] ICP22 created new session: [4:7587241319901512975:2050]
2025-12-24T02:11:20.786360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319901512975:2050] [node 3] ICS09 handshake done sender: [4:7587241319901512964:2115] self: [4:7587241319901512972:0] peer: [3:7587241322994635947:0] socket: 74 qp: -1
2025-12-24T02:11:20.786366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319901512975:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.786373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.787229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322994635942:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.787248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.787254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322994635942:2107] [node 4] ICP22 created new session: [3:7587241322994635974:2107]
2025-12-24T02:11:20.787259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322994635974:2107] [node 4] ICS09 handshake done sender: [3:7587241322994635948:2113] self: [3:7587241322994635947:0] peer: [4:7587241319901512972:0] socket: 52 qp: -1
2025-12-24T02:11:20.787264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322994635974:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:20.787271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.787276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217829:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.787291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320394217829:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.787303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162039:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.787319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241323232162039:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.787745Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241323232161974:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.787939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.788640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635959:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.788781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320394217831:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.789035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320394217829:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.789271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320394217760:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.789290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.789296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320394217760:2049] [node 3] ICP22 created new session: [2:7587241320394217854:2049]
2025-12-24T02:11:20.789302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320394217854:2049] [node 3] ICS09 handshake done sender: [2:7587241320394217831:2113] self: [2:7587241320394217830:0] peer: [3:7587241322994635964:0] socket: 47 qp: -1
2025-12-24T02:11:20.789310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320394217854:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.789316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.789682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320394217823:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.789714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.789721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320394217823:2106] [node 4] ICP22 created new session: [2:7587241320394217856:2106]
2025-12-24T02:11:20.789727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320394217856:2106] [node 4] ICS09 handshake done sender: [2:7587241320394217829:2112] self: [2:7587241320394217828:0] peer: [4:7587241319901512971:0] socket: 46 qp: -1
2025-12-24T02:11:20.789733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320394217856:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:20.789739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.790633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241323232162039:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.790952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323195275159:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.790992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323195275077:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.790875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241323232162033:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.790893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.790899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241323232162033:2105] [node 4] ICP22 created new session: [1:7587241323232162070:2105]
2025-12-24T02:11:20.790905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241323232162070:2105] [node 4] ICS09 handshake done sender: [1:7587241323232162039:2108] self: [1:7587241323232162038:0] peer: [4:7587241319901512973:0] socket: 42 qp: -1
2025-12-24T02:11:20.790910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241323232162070:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:20.790917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.791497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162049:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.791508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241323232162049:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.792066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241323232162049:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.792126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275162:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.792223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241323232162044:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.792240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.792246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241323232162044:2109] [node 6] ICP22 created new session: [1:7587241323232162073:2109]
2025-12-24T02:11:20.792251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241323232162073:2109] [node 6] ICS09 handshake done sender: [1:7587241323232162049:2112] self: [1:7587241323232162047:0] peer: [6:7587241323195275161:0] socket: 89 qp: -1
2025-12-24T02:11:20.792262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241323232162073:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.792275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.792290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323195275159:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.792353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323195275077:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.792368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.792373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323195275077:2048] [node 1] ICP22 created new session: [6:7587241323195275164:2048]
2025-12-24T02:11:20.792378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323195275164:2048] [node 1] ICS09 handshake done sender: [6:7587241323195275159:2115] self: [6:7587241323195275161:0] peer: [1:7587241323232162047:0] socket: 90 qp: -1
2025-12-24T02:11:20.792384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323195275164:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.792389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.796468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241320411944184:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.796538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241320411944101:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.796564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944188:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.796599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275163:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.796612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944189:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.796822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241323232162043:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.796833Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241323232162043:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.797806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323195275163:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.797840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323195275078:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.797981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217835:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.797998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320394217835:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.798220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241323232162043:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.798966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320394217835:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.798979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241320411944189:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.799031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944192:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.799089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320394217827:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.799108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.799115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320394217827:2108] [node 6] ICP22 created new session: [2:7587241320394217860:2108]
2025-12-24T02:11:20.799121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320394217860:2108] [node 6] ICS09 handshake done sender: [2:7587241320394217835:2115] self: [2:7587241320394217834:0] peer: [6:7587241323195275167:0] socket: 79 qp: -1
2025-12-24T02:11:20.799127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320394217860:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:20.799135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.799367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241323232162035:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.799388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.799394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241323232162035:2106] [node 5] ICP22 created new session: [1:7587241323232162076:2106]
2025-12-24T02:11:20.799399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241323232162076:2106] [node 5] ICS09 handshake done sender: [1:7587241323232162043:2110] self: [1:7587241323232162041:0] peer: [5:7587241320411944191:0] socket: 49 qp: -1
2025-12-24T02:11:20.799405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241323232162076:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:20.799412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.799396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275168:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.799524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241322994635878:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14844) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241322994635941:2111] held: no
2025-12-24T02:11:20.799601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241322994635943:2108] [node 5] ICP18 handshake (actor [3:7587241322994635959:2118]) is too old
2025-12-24T02:11:20.800414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241320411944102:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.800431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241320411944184:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.800517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323195275160:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.800532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323195275163:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.800613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323195275078:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.800636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.800643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323195275078:2049] [node 2] ICP22 created new session: [6:7587241323195275169:2049]
2025-12-24T02:11:20.800648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323195275169:2049] [node 2] ICS09 handshake done sender: [6:7587241323195275163:2118] self: [6:7587241323195275167:0] peer: [2:7587241320394217834:0] socket: 81 qp: -1
2025-12-24T02:11:20.800655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323195275169:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.800663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.800581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241320411944101:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.800601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.800607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241320411944101:2048] [node 1] ICP22 created new session: [5:7587241320411944194:2048]
2025-12-24T02:11:20.800612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241320411944194:2048] [node 1] ICS09 handshake done sender: [5:7587241320411944184:2117] self: [5:7587241320411944191:0] peer: [1:7587241323232162041:0] socket: 84 qp: -1
2025-12-24T02:11:20.800623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241320411944194:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.800631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.800881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322994635976:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61733
2025-12-24T02:11:20.801140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322994635941:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.801281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241320394217833:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.801288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241320394217833:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.800978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241319901512890:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241319901512962:2113] held: no
2025-12-24T02:11:20.801285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241319901512891:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14844) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241319901512963:2114] held: no
2025-12-24T02:11:20.802299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241322994635878:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.802314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.802320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322994635878:2049] [node 2] ICP22 created new session: [3:7587241322994635977:2049]
2025-12-24T02:11:20.802326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322994635977:2049] [node 2] ICS09 handshake done sender: [3:7587241322994635941:2111] self: [3:7587241322994635964:0] peer: [2:7587241320394217830:0] socket: 50 qp: -1
2025-12-24T02:11:20.802333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322994635977:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.802339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.802283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512978:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.802758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241320411944188:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.802777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
2025-12-24T02:11:20.802767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241320394217833:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.803435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323195275079:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.802844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241320411944167:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.803637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322994635954:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.803646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322994635954:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.804407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322994635954:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.804484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322994635944:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.804512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.804518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322994635944:2109] [node 6] ICP22 created new session: [3:7587241322994635980:2109]
2025-12-24T02:11:20.804523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322994635980:2109] [node 6] ICS09 handshake done sender: [3:7587241322994635954:2116] self: [3:7587241322994635953:0] peer: [6:7587241323195275171:0] socket: 87 qp: -1
2025-12-24T02:11:20.804528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322994635980:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:20.804536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.806851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512980:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
2025-12-24T02:11:20.806937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241320394217824:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.806958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.806964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241320394217824:2107] [node 5] ICP22 created new session: [2:7587241320394217863:2107]
2025-12-24T02:11:20.806969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241320394217863:2107] [node 5] ICS09 handshake done sender: [2:7587241320394217833:2114] self: [2:7587241320394217832:0] peer: [5:7587241320411944193:0] socket: 94 qp: -1
2025-12-24T02:11:20.806974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241320394217863:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.806982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.807106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944196:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.807203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241319901512981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41514
2025-12-24T02:11:20.807324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241320411944189:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.807968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241320411944102:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.807988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.807995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241320411944102:2049] [node 2] ICP22 created new session: [5:7587241320411944198:2049]
2025-12-24T02:11:20.808001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241320411944198:2049] [node 2] ICS09 handshake done sender: [5:7587241320411944189:2120] self: [5:7587241320411944193:0] peer: [2:7587241320394217832:0] socket: 61 qp: -1
2025-12-24T02:11:20.808006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241320411944198:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.808013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.808102Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323195275079:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9713) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241323195275160:2116] held: no
2025-12-24T02:11:20.808357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275172:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.808370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275173:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.809244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323195275160:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.809259Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241320411944102:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.809274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512981:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.809308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512980:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.809358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319901512963:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.809470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319901512962:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.809644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512966:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.809656Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241319901512966:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.810148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319901512891:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.810163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.810169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319901512891:2049] [node 2] ICP22 created new session: [4:7587241319901512986:2049]
2025-12-24T02:11:20.810203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319901512986:2049] [node 2] ICS09 handshake done sender: [4:7587241319901512963:2114] self: [4:7587241319901512971:0] peer: [2:7587241320394217828:0] socket: 73 qp: -1
2025-12-24T02:11:20.810213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319901512986:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.810219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.810258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241320411944200:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59677
2025-12-24T02:11:20.809550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323195275079:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.809566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.809572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323195275079:2050] [node 3] ICP22 created new session: [6:7587241323195275174:2050]
2025-12-24T02:11:20.809577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323195275174:2050] [node 3] ICS09 handshake done sender: [6:7587241323195275160:2116] self: [6:7587241323195275171:0] peer: [3:7587241322994635953:0] socket: 93 qp: -1
2025-12-24T02:11:20.809583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323195275174:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.809589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.809883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323195275172:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.809927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323195275144:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.810534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241319901512890:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.810547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.810552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319901512890:2048] [node 1] ICP22 created new session: [4:7587241319901512988:2048]
2025-12-24T02:11:20.810556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319901512988:2048] [node 1] ICS09 handshake done sender: [4:7587241319901512962:2113] self: [4:7587241319901512973:0] peer: [1:7587241323232162038:0] socket: 65 qp: -1
2025-12-24T02:11:20.810561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319901512988:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.811548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241320411944188:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.811603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241320411944167:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.811616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.810566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.810866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241319901512968:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.810870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241319901512968:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.811403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319901512966:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.811621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241320411944167:2110] [node 4] ICP22 created new session: [5:7587241320411944201:2110]
2025-12-24T02:11:20.811645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241320411944201:2110] [node 4] ICS09 handshake done sender: [5:7587241320411944188:2119] self: [5:7587241320411944197:0] peer: [4:7587241319901512965:0] socket: 56 qp: -1
2025-12-24T02:11:20.811651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241320411944201:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:20.811658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.814134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241319901512960:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.814155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.814163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319901512960:2109] [node 5] ICP22 created new session: [4:7587241319901512990:2109]
2025-12-24T02:11:20.814169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319901512990:2109] [node 5] ICS09 handshake done sender: [4:7587241319901512966:2116] self: [4:7587241319901512965:0] peer: [5:7587241320411944197:0] socket: 68 qp: -1
2025-12-24T02:11:20.814175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319901512990:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:20.814182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.815207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241319901512968:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.815258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323195275177:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38724
2025-12-24T02:11:20.815472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323195275172:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.815530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241319901512961:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.815549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.815561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241319901512961:2110] [node 6] ICP22 created new session: [4:7587241319901512993:2110]
2025-12-24T02:11:20.815568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241319901512993:2110] [node 6] ICS09 handshake done sender: [4:7587241319901512968:2117] self: [4:7587241319901512967:0] peer: [6:7587241323195275176:0] socket: 57 qp: -1
2025-12-24T02:11:20.815574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241319901512993:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.815582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.815578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323195275144:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.815592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.815597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323195275144:2109] [node 4] ICP22 created new session: [6:7587241323195275178:2109]
2025-12-24T02:11:20.815603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323195275178:2109] [node 4] ICS09 handshake done sender: [6:7587241323195275172:2120] self: [6:7587241323195275176:0] peer: [4:7587241319901512967:0] socket: 64 qp: -1
2025-12-24T02:11:20.815608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323195275178:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:20.815616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.820954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.821050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241320394217848:2048] [node 1] ICS01 socket: 40 reason# EndOfStream
2025-12-24T02:11:20.821059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241320394217759:2048] [node 1] ICP30 unregister session Session# [2:7587241320394217848:2048] VirtualId# [2:7587241320394217843:0]
2025-12-24T02:11:20.821066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241320394217848:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.821130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.821162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.821294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.821388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322994635969:2048] [node 1] ICS01 socket: 44 reason# EndOfStream
2025-12-24T02:11:20.821396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322994635877:2048] [node 1] ICP30 unregister session Session# [3:7587241322994635969:2048] VirtualId# [3:7587241322994635961:0]
2025-12-24T02:11:20.821401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322994635969:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.821463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.821484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.822754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.822847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319901512988:2048] [node 1] ICS01 socket: 65 reason# EndOfStream
2025-12-24T02:11:20.822861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319901512890:2048] [node 1] ICP30 unregister session Session# [4:7587241319901512988:2048] VirtualId# [4:7587241319901512973:0]
2025-12-24T02:11:20.822867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319901512988:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.822942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.822964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.823094Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.823181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241320411944194:2048] [node 1] ICS07 socket disconnect 84 reason# EndOfStream
2025-12-24T02:11:20.823191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241320411944194:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.823237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.823271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241320411944194:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.823289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.823380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241323195275164:2048] [node 1] ICS07 socket disconnect 90 reason# EndOfStream
2025-12-24T02:11:20.823386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323195275164:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.823433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.823467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241323195275164:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:20.823798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241320411944101:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.823812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241320411944194:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.823816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241320411944101:2048] [node 1] ICP30 unregister session Session# [5:7587241320411944194:2048] VirtualId# [5:7587241320411944191:0]
2025-12-24T02:11:20.823841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.823846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241320411944101:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.823850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.823919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323195275077:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.823937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323195275164:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.823942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323195275077:2048] [node 1] ICP30 unregister session Session# [6:7587241323195275164:2048] VirtualId# [6:7587241323195275161:0]
2025-12-24T02:11:20.823973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.823983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241323195275077:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.823985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7868) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.830853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.830951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322994635977:2049] [node 2] ICS01 socket: 50 reason# EndOfStream
2025-12-24T02:11:20.830964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322994635878:2049] [node 2] ICP30 unregister session Session# [3:7587241322994635977:2049] VirtualId# [3:7587241322994635964:0]
2025-12-24T02:11:20.830971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322994635977:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.831027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.831058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.831154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.831251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241320411944198:2049] [node 2] ICS01 socket: 61 reason# EndOfStream
2025-12-24T02:11:20.831263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241320411944102:2049] [node 2] ICP30 unregister session Session# [5:7587241320411944198:2049] VirtualId# [5:7587241320411944193:0]
2025-12-24T02:11:20.831269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241320411944198:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.831323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.831339Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.831364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.831424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323195275169:2049] [node 2] ICS01 socket: 81 reason# EndOfStream
2025-12-24T02:11:20.831436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323195275078:2049] [node 2] ICP30 unregister session Session# [6:7587241323195275169:2049] VirtualId# [6:7587241323195275167:0]
2025-12-24T02:11:20.831454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323195275169:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.831505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.831539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.833199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.833343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319901512986:2049] [node 2] ICS01 socket: 73 reason# EndOfStream
2025-12-24T02:11:20.833355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319901512891:2049] [node 2] ICP30 unregister session Session# [4:7587241319901512986:2049] VirtualId# [4:7587241319901512971:0]
2025-12-24T02:11:20.833361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319901512986:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.833412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.833445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.833651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241323195275077:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.833708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241320411944101:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:20.838296Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:20.838395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241319901512975:2050] [node 3] ICS01 socket: 74 reason# EndOfStream
2025-12-24T02:11:20.838410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241319901512892:2050] [node 3] ICP30 unregister session Session# [4:7587241319901512975:2050] VirtualId# [4:7587241319901512972:0]
2025-12-24T02:11:20.838417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241319901512975:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.838433Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:20.838476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.838512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.838519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241320411944186:2050] [node 3] ICS07 socket disconnect 59 reason# EndOfStream
2025-12-24T02:11:20.838530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241320411944186:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.838590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.838588Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:20.838733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323195275174:2050] [node 3] ICS01 socket: 93 reason# EndOfStream
2025-12-24T02:11:20.838746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323195275079:2050] [node 3] ICP30 unregister session Session# [6:7587241323195275174:2050] VirtualId# [6:7587241323195275171:0]
2025-12-24T02:11:20.838752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323195275174:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.838632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241320411944186:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:20.838797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:20.838830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.839595Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241320411944103:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9713) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.839612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241320411944186:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.839617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241320411944103:2050] [node 3] ICP30 unregister session Session# [5:7587241320411944186:2050] VirtualId# [5:7587241320411944183:0]
2025-12-24T02:11:20.839649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:20.839660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241320411944103:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.839663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9713) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.845631Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:20.845711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241323195275178:2109] [node 4] ICS07 socket disconnect 64 reason# EndOfStream
2025-12-24T02:11:20.845721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323195275178:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.845772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.845800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241323195275178:2109] [node 4] ICS15 start handshake
2025-12-24T02:11:20.845468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:20.845584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241320411944201:2110] [node 4] ICS01 socket: 56 reason# EndOfStream
2025-12-24T02:11:20.845593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241320411944167:2110] [node 4] ICP30 unregister session Session# [5:7587241320411944201:2110] VirtualId# [5:7587241320411944197:0]
2025-12-24T02:11:20.845599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241320411944201:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.845661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:20.845691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.846145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323195275144:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10914) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.846157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323195275178:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.846162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323195275144:2109] [node 4] ICP30 unregister session Session# [6:7587241323195275178:2109] VirtualId# [6:7587241323195275176:0]
2025-12-24T02:11:20.846191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:20.846196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241323195275144:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.846199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10914) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.852925Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:20.853031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323195275156:2107] [node 5] ICS01 socket: 36 reason# EndOfStream
2025-12-24T02:11:20.853040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323195275138:2107] [node 5] ICP30 unregister session Session# [6:7587241323195275156:2107] VirtualId# [6:7587241323195275139:0]
2025-12-24T02:11:20.853047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323195275156:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.853092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:20.853126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:20.866473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241320487114739:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:20.866484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241320487114740:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.870880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322815504405:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.870886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322815504406:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.870889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322815504407:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.872788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241321301941333:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.872796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241321301941334:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.872802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241321301941335:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.874732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323143620048:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.874746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323143620049:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:20.874779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323143620047:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.869216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322604906453:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.869225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322604906454:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:20.867895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241323523278394:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:20.867907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241323523278395:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:20.889317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323143620096:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:20.889877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241320487114800:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:20.890358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241320487114805:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:20.891521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241320487114809:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:20.894988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322815504466:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.895221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322604906514:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.900816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941397:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.899302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241323523278458:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:20.899439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241323523278460:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:20.899573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241323523278461:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:20.900275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278463:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
2025-12-24T02:11:20.900289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278464:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
2025-12-24T02:11:20.902085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278462:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
2025-12-24T02:11:20.906997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241321301941399:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:20.907056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241322815504475:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.907778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504474:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.908075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241320487114815:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-12-24T02:11:20.908101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241320487114816:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-12-24T02:11:20.911150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322604906519:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:20.911303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241322604906520:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.915259Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241320487114805:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23611092ns) {Configure:104(23610467ns) {ProcessConfigured:135(23345643ns) {StartInitialHandshake:157(23345027ns) {DropHandshakes:470(309ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(145ns) PrepareNewSessionHandshake:477(990ns) {SwitchToState:179(732ns)}}}}}
2025-12-24T02:11:20.915862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620121:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.919185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241323143620123:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.921309Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504484:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.921358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504485:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.929134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278464:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.929783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241321301941408:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:20.931086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941406:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.931105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941407:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.936909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114816:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.937229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278463:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.937303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241323523278394:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.939023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906532:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.939067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323143620121:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.939107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323143620049:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.939680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504474:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.939817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322815504406:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.939263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906534:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-12-24T02:11:20.939994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114798:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.939337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-12-24T02:11:20.939439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906518:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241320487114798:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.939456Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322604906518:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.939919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906533:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-12-24T02:11:20.939962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906536:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.940738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906534:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241320487114798:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.940818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322604906518:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.940863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504485:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322815504405:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.940980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504473:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241322815504473:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.941585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322815504473:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.940994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906533:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.941035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906535:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.942103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906536:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.942148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906524:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.942156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322604906524:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.942552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906532:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.939259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941397:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941406:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.940878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241321301941399:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.942712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241322604906454:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.941364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941407:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.941383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241321301941335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.942067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941417:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.939743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278462:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.942770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241322604906453:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.942820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322604906524:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.943073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322604906520:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.943092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.943099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322604906520:2109] [node 5] ICP22 created new session: [3:7587241322604906545:2109]
2025-12-24T02:11:20.943104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322604906545:2109] [node 5] ICS09 handshake done sender: [3:7587241322604906524:2112] self: [3:7587241322604906523:0] peer: [5:7587241321301941416:0] socket: 68 qp: -1
2025-12-24T02:11:20.943109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322604906545:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:20.943117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.941696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241322815504466:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.941710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.941717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322815504466:2107] [node 5] ICP22 created new session: [4:7587241322815504489:2107]
2025-12-24T02:11:20.941722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322815504489:2107] [node 5] ICS09 handshake done sender: [4:7587241322815504473:2111] self: [4:7587241322815504467:0] peer: [5:7587241321301941415:0] socket: 61 qp: -1
2025-12-24T02:11:20.941728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322815504489:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.941736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.943045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504484:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.943069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241322815504407:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.943404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278466:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.943417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241323523278466:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.943410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241321301941406:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.943908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278479:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
2025-12-24T02:11:20.943990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620127:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.943989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278468:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.944000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241323523278468:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.944452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241323523278466:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.944554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241321301941399:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.944573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.944579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241321301941399:2110] [node 4] ICP22 created new session: [5:7587241321301941419:2110]
2025-12-24T02:11:20.944584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241321301941419:2110] [node 4] ICS09 handshake done sender: [5:7587241321301941406:2114] self: [5:7587241321301941415:0] peer: [4:7587241322815504467:0] socket: 76 qp: -1
2025-12-24T02:11:20.944597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241321301941419:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:20.944604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.944639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241323523278395:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.944676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.944683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241323523278395:2049] [node 3] ICP22 created new session: [2:7587241323523278483:2049]
2025-12-24T02:11:20.944689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241323523278483:2049] [node 3] ICS09 handshake done sender: [2:7587241323523278466:2114] self: [2:7587241323523278465:0] peer: [3:7587241322604906541:0] socket: 56 qp: -1
2025-12-24T02:11:20.944695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241323523278483:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.944701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.944908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241323523278463:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.945461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941418:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.945571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322604906514:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.945594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.945604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322604906514:2107] [node 6] ICP22 created new session: [3:7587241322604906547:2107]
2025-12-24T02:11:20.945611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322604906547:2107] [node 6] ICS09 handshake done sender: [3:7587241322604906518:2110] self: [3:7587241322604906515:0] peer: [6:7587241323143620126:0] socket: 66 qp: -1
2025-12-24T02:11:20.945615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241322604906522:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.945619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322604906547:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:20.945624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241322604906522:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.945626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.945732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241323523278394:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.945790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241322604906514:2107] [node 6] ICP18 handshake (actor [3:7587241322604906533:2116]) is too old
2025-12-24T02:11:20.945752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.945760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241323523278394:2048] [node 1] ICP22 created new session: [2:7587241323523278485:2048]
2025-12-24T02:11:20.945765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241323523278485:2048] [node 1] ICS09 handshake done sender: [2:7587241323523278463:2112] self: [2:7587241323523278477:0] peer: [1:7587241320487114797:0] socket: 52 qp: -1
2025-12-24T02:11:20.945771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241323523278485:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.945778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.945829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323143620121:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.945994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.946747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323143620049:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.946763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.946770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323143620049:2050] [node 3] ICP22 created new session: [6:7587241323143620129:2050]
2025-12-24T02:11:20.946782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323143620129:2050] [node 3] ICS09 handshake done sender: [6:7587241323143620121:2113] self: [6:7587241323143620126:0] peer: [3:7587241322604906515:0] socket: 71 qp: -1
2025-12-24T02:11:20.946790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323143620129:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.946797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.948470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114812:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.948494Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241320487114812:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.949889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114808:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.949912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241320487114808:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.950024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114815:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.950053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241320487114739:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.950070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.950085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241320487114739:2048] [node 2] ICP22 created new session: [1:7587241320487114824:2048]
2025-12-24T02:11:20.950098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241320487114824:2048] [node 2] ICS09 handshake done sender: [1:7587241320487114798:2105] self: [1:7587241320487114797:0] peer: [2:7587241323523278477:0] socket: 48 qp: -1
2025-12-24T02:11:20.950107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241320487114824:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:20.950131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.949722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.949958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620128:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.952075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241323523278468:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.952641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278480:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.952660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241323523278458:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.952669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.952675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241323523278458:2105] [node 4] ICP22 created new session: [2:7587241323523278489:2105]
2025-12-24T02:11:20.952681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241323523278489:2105] [node 4] ICS09 handshake done sender: [2:7587241323523278468:2115] self: [2:7587241323523278467:0] peer: [4:7587241322815504486:0] socket: 57 qp: -1
2025-12-24T02:11:20.952687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241323523278489:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:20.952692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.953126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241320487114822:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-12-24T02:11:20.953192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241320487114823:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
2025-12-24T02:11:20.953491Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241320487114812:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:20.953697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241320487114808:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.953781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241320487114740:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.953798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.953805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241320487114740:2049] [node 3] ICP22 created new session: [1:7587241320487114829:2049]
2025-12-24T02:11:20.953811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241320487114829:2049] [node 3] ICS09 handshake done sender: [1:7587241320487114812:2112] self: [1:7587241320487114802:0] peer: [3:7587241322604906542:0] socket: 73 qp: -1
2025-12-24T02:11:20.953817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241320487114829:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:20.953827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.954055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241320487114800:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.954079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.954084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241320487114800:2106] [node 4] ICP22 created new session: [1:7587241320487114831:2106]
2025-12-24T02:11:20.954089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241320487114831:2106] [node 4] ICS09 handshake done sender: [1:7587241320487114808:2110] self: [1:7587241320487114806:0] peer: [4:7587241322815504487:0] socket: 72 qp: -1
2025-12-24T02:11:20.954094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241320487114831:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:20.954099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.954103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241321301941335:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241321301941407:2115] held: no
2025-12-24T02:11:20.954388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241321301941407:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.954598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241321301941335:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.954614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.954621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241321301941335:2050] [node 3] ICP22 created new session: [5:7587241321301941423:2050]
2025-12-24T02:11:20.954627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241321301941423:2050] [node 3] ICS09 handshake done sender: [5:7587241321301941407:2115] self: [5:7587241321301941416:0] peer: [3:7587241322604906523:0] socket: 77 qp: -1
2025-12-24T02:11:20.954633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241321301941423:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.954636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941422:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.954639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.952252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322815504474:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.954653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241320487114739:2048] [node 2] ICP18 handshake (actor [1:7587241320487114815:2114]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.956830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941425:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.956567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114822:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.956810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241323523278458:2105] [node 4] ICP18 handshake (actor [2:7587241323523278480:2119]) is too old
2025-12-24T02:11:20.958400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941425:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.958431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241321301941333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.958430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941410:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.958436Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241321301941410:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.955550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322815504406:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.955570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.955578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322815504406:2049] [node 2] ICP22 created new session: [4:7587241322815504494:2049]
2025-12-24T02:11:20.955594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322815504494:2049] [node 2] ICS09 handshake done sender: [4:7587241322815504474:2112] self: [4:7587241322815504486:0] peer: [2:7587241323523278467:0] socket: 60 qp: -1
2025-12-24T02:11:20.955601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322815504494:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.955608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.958851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241321301941410:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.958955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241321301941408:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.958976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.958983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241321301941408:2111] [node 6] ICP22 created new session: [5:7587241321301941428:2111]
2025-12-24T02:11:20.958989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241321301941428:2111] [node 6] ICS09 handshake done sender: [5:7587241321301941410:2116] self: [5:7587241321301941409:0] peer: [6:7587241323143620132:0] socket: 95 qp: -1
2025-12-24T02:11:20.958996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241321301941428:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:20.959005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.960017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114819:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.960029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241320487114819:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.958210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323143620128:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.958263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323143620096:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.960534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241320487114835:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30250
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.960625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941430:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.960604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241320487114819:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.960696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241321301941425:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.960786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241321301941333:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.960798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.960804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241321301941333:2048] [node 1] ICP22 created new session: [5:7587241321301941431:2048]
2025-12-24T02:11:20.960810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241321301941431:2048] [node 1] ICS09 handshake done sender: [5:7587241321301941425:2121] self: [5:7587241321301941426:0] peer: [1:7587241320487114810:0] socket: 109 qp: -1
2025-12-24T02:11:20.960817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241321301941431:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.960823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.960773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241320487114805:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.960791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.960797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241320487114805:2107] [node 5] ICP22 created new session: [1:7587241320487114837:2107]
2025-12-24T02:11:20.960803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241320487114837:2107] [node 5] ICS09 handshake done sender: [1:7587241320487114819:2116] self: [1:7587241320487114810:0] peer: [5:7587241321301941426:0] socket: 64 qp: -1
2025-12-24T02:11:20.960810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241320487114837:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.960818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.961329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114835:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.960800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323143620096:2095] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241323143620128:2117] held: no
2025-12-24T02:11:20.962220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241323523278488:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39012
2025-12-24T02:11:20.963286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322604906522:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.963944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241322604906519:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.963962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.963966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322604906519:2108] [node 4] ICP22 created new session: [3:7587241322604906552:2108]
2025-12-24T02:11:20.963971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322604906552:2108] [node 4] ICS09 handshake done sender: [3:7587241322604906522:2111] self: [3:7587241322604906521:0] peer: [4:7587241322815504492:0] socket: 67 qp: -1
2025-12-24T02:11:20.963975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322604906552:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:20.963981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.964014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114823:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.964044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241320487114740:2049] [node 3] ICP18 handshake (actor [1:7587241320487114823:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.964238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906551:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.964360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620134:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.964431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620135:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.964460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620136:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.964644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323143620128:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.964938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241322604906453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241322604906535:2118] held: no
2025-12-24T02:11:20.965091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322604906535:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.965301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278488:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.965329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241323523278395:2049] [node 3] ICP18 handshake (actor [2:7587241323523278488:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.966096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241322604906549:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7273
2025-12-24T02:11:20.965713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323143620096:2095] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.965734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.965742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323143620096:2095] [node 5] ICP22 created new session: [6:7587241323143620137:2095]
2025-12-24T02:11:20.965748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323143620137:2095] [node 5] ICS09 handshake done sender: [6:7587241323143620128:2117] self: [6:7587241323143620132:0] peer: [5:7587241321301941409:0] socket: 97 qp: -1
2025-12-24T02:11:20.965755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323143620137:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:20.965763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.966035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323143620134:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.966204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323143620047:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.966368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241321301941422:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.966394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241321301941334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.966518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278470:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.966529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241323523278470:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.966786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323143620135:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.966812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323143620048:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.967135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241321301941434:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51207
2025-12-24T02:11:20.967329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241320487114814:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.967338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241320487114814:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.967421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241322604906453:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.967436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.967467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322604906453:2048] [node 1] ICP22 created new session: [3:7587241322604906554:2048]
2025-12-24T02:11:20.967472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322604906554:2048] [node 1] ICS09 handshake done sender: [3:7587241322604906535:2118] self: [3:7587241322604906542:0] peer: [1:7587241320487114802:0] socket: 79 qp: -1
2025-12-24T02:11:20.967481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322604906554:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.967488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.967807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241323523278470:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:20.967921Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241321301941422:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.967972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241323523278473:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.967982Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241323523278473:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.968365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241323523278460:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.968381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:20.968385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241323523278460:2107] [node 5] ICP22 created new session: [2:7587241323523278493:2107]
2025-12-24T02:11:20.968389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241323523278493:2107] [node 5] ICS09 handshake done sender: [2:7587241323523278470:2116] self: [2:7587241323523278469:0] peer: [5:7587241321301941433:0] socket: 101 qp: -1
2025-12-24T02:11:20.968394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241323523278493:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:20.968399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:20.969295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241322604906534:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.970003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241320487114814:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.970084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241320487114809:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.970100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.970106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241320487114809:2108] [node 6] ICP22 created new session: [1:7587241320487114841:2108]
2025-12-24T02:11:20.970111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241320487114841:2108] [node 6] ICS09 handshake done sender: [1:7587241320487114814:2113] self: [1:7587241320487114813:0] peer: [6:7587241323143620139:0] socket: 105 qp: -1
2025-12-24T02:11:20.970116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241320487114841:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:20.970122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.970535Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241322604906454:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241322604906534:2117] held: no
2025-12-24T02:11:20.970625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241322604906454:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.970639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.970645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241322604906454:2049] [node 2] ICP22 created new session: [3:7587241322604906556:2049]
2025-12-24T02:11:20.970651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241322604906556:2049] [node 2] ICS09 handshake done sender: [3:7587241322604906534:2117] self: [3:7587241322604906541:0] peer: [2:7587241323523278465:0] socket: 78 qp: -1
2025-12-24T02:11:20.970663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241322604906556:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.970674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.970732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241321301941334:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.970759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.970767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241321301941334:2049] [node 2] ICP22 created new session: [5:7587241321301941435:2049]
2025-12-24T02:11:20.970772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241321301941435:2049] [node 2] ICS09 handshake done sender: [5:7587241321301941422:2120] self: [5:7587241321301941433:0] peer: [2:7587241323523278469:0] socket: 106 qp: -1
2025-12-24T02:11:20.970779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241321301941435:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.970785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.967206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504497:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.970678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504496:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.970884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241323523278473:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:20.971519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241322815504498:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7174
2025-12-24T02:11:20.971614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241322815504405:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241322815504485:2117] held: no
2025-12-24T02:11:20.972226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241323523278461:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.972249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.972256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241323523278461:2110] [node 6] ICP22 created new session: [2:7587241323523278496:2110]
2025-12-24T02:11:20.972261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241323523278496:2110] [node 6] ICS09 handshake done sender: [2:7587241323523278473:2117] self: [2:7587241323523278472:0] peer: [6:7587241323143620140:0] socket: 108 qp: -1
2025-12-24T02:11:20.972266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241323523278496:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.972273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.972818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322815504485:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.973661Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323143620047:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241323143620134:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.973670Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241322815504407:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241322815504484:2116] held: no
2025-12-24T02:11:20.973753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322815504405:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.973767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.973776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322815504405:2048] [node 1] ICP22 created new session: [4:7587241322815504499:2048]
2025-12-24T02:11:20.973782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322815504499:2048] [node 1] ICS09 handshake done sender: [4:7587241322815504485:2117] self: [4:7587241322815504487:0] peer: [1:7587241320487114806:0] socket: 74 qp: -1
2025-12-24T02:11:20.973788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322815504499:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.973795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.973853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322815504484:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.974137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.974142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.975070Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323143620048:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25752) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241323143620135:2119] held: no
2025-12-24T02:11:20.975148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323143620134:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:20.975411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323143620135:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:20.974114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241322815504407:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.974128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:20.974133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322815504407:2050] [node 3] ICP22 created new session: [4:7587241322815504502:2050]
2025-12-24T02:11:20.974137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322815504502:2050] [node 3] ICS09 handshake done sender: [4:7587241322815504484:2116] self: [4:7587241322815504492:0] peer: [3:7587241322604906521:0] socket: 70 qp: -1
2025-12-24T02:11:20.974142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322815504502:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:20.974148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:20.975265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504498:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.975483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323143620047:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.975500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:20.975505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323143620048:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.975506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323143620047:2048] [node 1] ICP22 created new session: [6:7587241323143620144:2048]
2025-12-24T02:11:20.975511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323143620144:2048] [node 1] ICS09 handshake done sender: [6:7587241323143620134:2118] self: [6:7587241323143620139:0] peer: [1:7587241320487114813:0] socket: 107 qp: -1
2025-12-24T02:11:20.975514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:20.975516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323143620144:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:20.975520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323143620048:2049] [node 2] ICP22 created new session: [6:7587241323143620145:2049]
2025-12-24T02:11:20.975522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:20.975525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323143620145:2049] [node 2] ICS09 handshake done sender: [6:7587241323143620135:2119] self: [6:7587241323143620140:0] peer: [2:7587241323523278472:0] socket: 55 qp: -1
2025-12-24T02:11:20.975532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323143620145:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:20.975538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:20.975699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620143:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.975864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241323143620143:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.975894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241323143620123:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:20.975981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241322815504481:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:20.975991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241322815504481:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:20.976569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241322815504481:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:20.976664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241322815504475:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:20.976690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:20.976701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241322815504475:2110] [node 6] ICP22 created new session: [4:7587241322815504506:2110]
2025-12-24T02:11:20.976707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241322815504506:2110] [node 6] ICS09 handshake done sender: [4:7587241322815504481:2115] self: [4:7587241322815504480:0] peer: [6:7587241323143620148:0] socket: 46 qp: -1
2025-12-24T02:11:20.976717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241322815504506:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:20.976724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:20.976675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241323143620149:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28018
2025-12-24T02:11:20.976763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241323143620143:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:20.976846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241323143620123:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:20.976868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:20.976878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241323143620123:2114] [node 4] ICP22 created new session: [6:7587241323143620150:2114]
2025-12-24T02:11:20.976884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241323143620150:2114] [node 4] ICS09 handshake done sender: [6:7587241323143620143:2123] self: [6:7587241323143620148:0] peer: [4:7587241322815504480:0] socket: 45 qp: -1
2025-12-24T02:11:20.976896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241323143620150:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:20.976908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:20.977046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241323143620123:2114] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:20.983108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:20.983203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241323523278485:2048] [node 1] ICS01 socket: 52 reason# EndOfStream
2025-12-24T02:11:20.983211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241323523278394:2048] [node 1] ICP30 unregister session Session# [2:7587241323523278485:2048] VirtualId# [2:7587241323523278477:0]
2025-12-24T02:11:20.983218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241323523278485:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.983281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.983305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.983612Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:20.983696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322604906554:2048] [node 1] ICS01 socket: 79 reason# EndOfStream
2025-12-24T02:11:20.983707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322604906453:2048] [node 1] ICP30 unregister session Session# [3:7587241322604906554:2048] VirtualId# [3:7587241322604906542:0]
2025-12-24T02:11:20.983713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322604906554:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.983759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.983793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.984637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:20.984715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322815504499:2048] [node 1] ICS01 socket: 74 reason# EndOfStream
2025-12-24T02:11:20.984726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322815504405:2048] [node 1] ICP30 unregister session Session# [4:7587241322815504499:2048] VirtualId# [4:7587241322815504487:0]
2025-12-24T02:11:20.984732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322815504499:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.984788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.984822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.984808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:20.984885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241321301941431:2048] [node 1] ICS01 socket: 109 reason# EndOfStream
2025-12-24T02:11:20.984896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241321301941333:2048] [node 1] ICP30 unregister session Session# [5:7587241321301941431:2048] VirtualId# [5:7587241321301941426:0]
2025-12-24T02:11:20.984901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241321301941431:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.984953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.984994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.984980Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:20.985057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323143620144:2048] [node 1] ICS01 socket: 107 reason# EndOfStream
2025-12-24T02:11:20.985067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323143620047:2048] [node 1] ICP30 unregister session Session# [6:7587241323143620144:2048] VirtualId# [6:7587241323143620139:0]
2025-12-24T02:11:20.985072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323143620144:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.985112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:20.985141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:20.993583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:20.993684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241322604906556:2049] [node 2] ICS01 socket: 78 reason# EndOfStream
2025-12-24T02:11:20.993697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241322604906454:2049] [node 2] ICP30 unregister session Session# [3:7587241322604906556:2049] VirtualId# [3:7587241322604906541:0]
2025-12-24T02:11:20.993705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241322604906556:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.993753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.993791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.994766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:20.994854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322815504494:2049] [node 2] ICS01 socket: 60 reason# EndOfStream
2025-12-24T02:11:20.994868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322815504406:2049] [node 2] ICP30 unregister session Session# [4:7587241322815504494:2049] VirtualId# [4:7587241322815504486:0]
2025-12-24T02:11:20.994873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322815504494:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.994836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:20.994921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.994950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.994917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241321301941435:2049] [node 2] ICS07 socket disconnect 106 reason# EndOfStream
2025-12-24T02:11:20.994931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241321301941435:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.994972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.995001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241321301941435:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:20.995003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:20.995089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323143620145:2049] [node 2] ICS01 socket: 55 reason# EndOfStream
2025-12-24T02:11:20.995100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323143620048:2049] [node 2] ICP30 unregister session Session# [6:7587241323143620145:2049] VirtualId# [6:7587241323143620140:0]
2025-12-24T02:11:20.995104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323143620145:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:20.995141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:20.995176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.995292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241321301941334:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25752) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:20.995309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241321301941435:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:20.995315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241321301941334:2049] [node 2] ICP30 unregister session Session# [5:7587241321301941435:2049] VirtualId# [5:7587241321301941433:0]
2025-12-24T02:11:20.995355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:20.995361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241321301941334:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:20.995364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25752) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.001960Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:21.002059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323143620129:2050] [node 3] ICS01 socket: 71 reason# EndOfStream
2025-12-24T02:11:21.002071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323143620049:2050] [node 3] ICP30 unregister session Session# [6:7587241323143620129:2050] VirtualId# [6:7587241323143620126:0]
2025-12-24T02:11:21.002079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323143620129:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.002145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.002154Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:21.002174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.002242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241322815504502:2050] [node 3] ICS01 socket: 70 reason# EndOfStream
2025-12-24T02:11:21.002258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241322815504407:2050] [node 3] ICP30 unregister session Session# [4:7587241322815504502:2050] VirtualId# [4:7587241322815504492:0]
2025-12-24T02:11:21.002264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241322815504502:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.002304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:21.002326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.002364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.002390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241321301941423:2050] [node 3] ICS01 socket: 77 reason# EndOfStream
2025-12-24T02:11:21.002402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241321301941335:2050] [node 3] ICP30 unregister session Session# [5:7587241321301941423:2050] VirtualId# [5:7587241321301941416:0]
2025-12-24T02:11:21.002409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241321301941423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.002468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.002505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.004661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241321301941334:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:21.008800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:21.008913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241321301941419:2110] [node 4] ICS01 socket: 76 reason# EndOfStream
2025-12-24T02:11:21.008927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241321301941399:2110] [node 4] ICP30 unregister session Session# [5:7587241321301941419:2110] VirtualId# [5:7587241321301941415:0]
2025-12-24T02:11:21.008934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241321301941419:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.008988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.009002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:21.009086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241323143620150:2114] [node 4] ICS07 socket disconnect 45 reason# EndOfStream
2025-12-24T02:11:21.009095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323143620150:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.009042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.009140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.009164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241323143620150:2114] [node 4] ICS15 start handshake
2025-12-24T02:11:21.009473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241323143620123:2114] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1564) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.009487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323143620150:2114] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.009492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323143620123:2114] [node 4] ICP30 unregister session Session# [6:7587241323143620150:2114] VirtualId# [6:7587241323143620148:0]
2025-12-24T02:11:21.009533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.009547Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241323143620123:2114] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.009550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1564) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.015621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:21.015718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241323143620137:2095] [node 5] ICS01 socket: 97 reason# EndOfStream
2025-12-24T02:11:21.015731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241323143620096:2095] [node 5] ICP30 unregister session Session# [6:7587241323143620137:2095] VirtualId# [6:7587241323143620132:0]
2025-12-24T02:11:21.015739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241323143620137:2095] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.015807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:21.015842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:21.030312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325436772833:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:21.030324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325436772834:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.031867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327103360655:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.031879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327103360656:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.033692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241324158187565:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.033706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241324158187566:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.035328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325429637817:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.035338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325429637818:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.035343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325429637819:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.036948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241324583359517:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.036957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241324583359518:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.036961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241324583359519:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.038621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241326907640153:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.038630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241326907640154:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.038634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241326907640155:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.050274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241326907640215:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:21.051505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325436772897:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:21.051648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325436772898:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:21.052668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325436772903:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:21.053179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327103360722:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.053471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359580:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.053794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327103360725:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.054121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637877:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-12-24T02:11:21.054310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327103360728:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:21.058298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241324583359584:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:21.058483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241324583359585:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:21.064811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327103360730:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:21.066455Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241326907640215:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16110802ns) {Configure:104(16110302ns) {ProcessConfigured:135(11015820ns) {StartInitialHandshake:157(11015365ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(1130ns) {SwitchToState:179(496ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.071330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241324158187632:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:21.071509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241324158187633:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.071640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241324158187634:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.072553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187627:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.070977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325436772910:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54052
2025-12-24T02:11:21.071070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325436772912:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54052
2025-12-24T02:11:21.072640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187631:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
2025-12-24T02:11:21.072979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241325436772897:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21242626ns) {Configure:104(21242058ns) {ProcessConfigured:135(17171101ns) {StartInitialHandshake:157(17170636ns) {DropHandshakes:470(510ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(132ns) PrepareNewSessionHandshake:477(777ns) {SwitchToState:179(451ns)}}}}}
2025-12-24T02:11:21.073045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772910:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.073295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241324583359518:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16253863ns) {Configure:104(16252863ns) {ProcessConfigured:135(15930596ns) {StartInitialHandshake:157(15930069ns) {DropHandshakes:470(464ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(977ns) {SwitchToState:179(343ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.073331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325429637819:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16862298ns) {Configure:104(16861245ns) {ProcessConfigured:135(15342148ns) {StartInitialHandshake:157(15341541ns) {DropHandshakes:470(467ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(1177ns) {SwitchToState:179(897ns)}}}}}
2025-12-24T02:11:21.073523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325429637885:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:21.073656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325429637886:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:21.073924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637877:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.074605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637887:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-12-24T02:11:21.074805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325429637818:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18415016ns) {Configure:104(18414012ns) {ProcessConfigured:135(16677630ns) {StartInitialHandshake:157(16676988ns) {DropHandshakes:470(492ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(484ns) {SwitchToState:179(102ns)}}}}}
2025-12-24T02:11:21.074888Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325429637817:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18606825ns) {Configure:104(18605634ns) {ProcessConfigured:135(16591845ns) {StartInitialHandshake:157(16591321ns) {DropHandshakes:470(253ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:477(472ns) {SwitchToState:179(64ns)}}}}}
2025-12-24T02:11:21.074931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325429637818:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.075526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241326907640221:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:21.083011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187627:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.083128Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241324158187565:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27944862ns) {Configure:104(27944343ns) {ProcessConfigured:135(27652444ns) {StartInitialHandshake:157(27652012ns) {DropHandshakes:470(389ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:477(1166ns) {SwitchToState:179(432ns)}}}}}
2025-12-24T02:11:21.084308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360724:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.084329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327103360724:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.087124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241324583359585:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19575956ns) {Configure:104(19571494ns) {ProcessConfigured:135(18973802ns) {StartInitialHandshake:157(18973164ns) {DropHandshakes:470(1667ns) {DropIncomingHandshake:442(243ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(381ns) PrepareNewSessionHandshake:477(1738ns) {SwitchToState:179(252ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.088150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241324583359519:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29558103ns) {Configure:104(29556967ns) {ProcessConfigured:135(29178430ns) {StartInitialHandshake:157(29176866ns) {DropHandshakes:470(661ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(519ns) PrepareNewSessionHandshake:477(961ns) {SwitchToState:179(349ns)}}}}}
2025-12-24T02:11:21.088775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360719:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.088790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327103360719:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.086829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241324158187566:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(31312167ns) {Configure:104(31311392ns) {ProcessConfigured:135(31000487ns) {StartInitialHandshake:157(31000069ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1173ns) {SwitchToState:179(487ns)}}}}}
2025-12-24T02:11:21.088183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241324158187566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.086834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241326907640153:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29030784ns) {Configure:104(29030249ns) {ProcessConfigured:135(28719265ns) {StartInitialHandshake:157(28718789ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1587ns) {SwitchToState:179(906ns)}}}}}
2025-12-24T02:11:21.088883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241326907640154:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22366289ns) {Configure:104(22365705ns) {ProcessConfigured:135(22059140ns) {StartInitialHandshake:157(22058807ns) {DropHandshakes:470(566ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(1327ns) {SwitchToState:179(739ns)}}}}}
2025-12-24T02:11:21.090168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327103360719:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.091423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640225:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.091511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637898:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-12-24T02:11:21.092152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325429637877:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.092246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325429637818:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.092259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.092273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325429637818:2049] [node 2] ICP22 created new session: [4:7587241325429637899:2049]
2025-12-24T02:11:21.092280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325429637899:2049] [node 2] ICS09 handshake done sender: [4:7587241325429637877:2107] self: [4:7587241325429637897:0] peer: [2:7587241327103360723:0] socket: 61 qp: -1
2025-12-24T02:11:21.092293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325429637899:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.092310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.091471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.092851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359595:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.092879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241324583359519:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.093012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187638:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.093033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241324158187638:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.090706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327103360656:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.090726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.090735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327103360656:2049] [node 3] ICP22 created new session: [2:7587241327103360737:2049]
2025-12-24T02:11:21.090742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327103360737:2049] [node 3] ICS09 handshake done sender: [2:7587241327103360719:2108] self: [2:7587241327103360718:0] peer: [3:7587241324158187646:0] socket: 49 qp: -1
2025-12-24T02:11:21.090770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327103360737:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.090779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.090996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327103360724:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.091229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360725:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.091268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241327103360655:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.091469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327103360735:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8756
2025-12-24T02:11:21.094251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241324158187638:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.094411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241324158187633:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.094424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.094431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241324158187633:2109] [node 5] ICP22 created new session: [3:7587241324158187648:2109]
2025-12-24T02:11:21.094438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241324158187648:2109] [node 5] ICS09 handshake done sender: [3:7587241324158187638:2113] self: [3:7587241324158187637:0] peer: [5:7587241324583359597:0] socket: 67 qp: -1
2025-12-24T02:11:21.094446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241324158187648:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.094454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.094839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772896:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.094852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325436772896:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.095488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772912:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.095983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359580:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.096811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327103360722:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.096027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241324583359517:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.096833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.096844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327103360722:2109] [node 4] ICP22 created new session: [2:7587241327103360743:2109]
2025-12-24T02:11:21.096850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327103360743:2109] [node 4] ICS09 handshake done sender: [2:7587241327103360724:2110] self: [2:7587241327103360723:0] peer: [4:7587241325429637897:0] socket: 60 qp: -1
2025-12-24T02:11:21.096859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327103360743:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:21.096869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.098816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325436772896:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.099610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359599:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.099633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359600:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.099684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.099754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241326907640225:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.099792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640227:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.099805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772902:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.099817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325436772902:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.099816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327103360742:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8756
2025-12-24T02:11:21.099837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359600:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.099873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241324583359595:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.099922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241326907640155:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.099985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327103360725:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.100455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241326907640227:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.100485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241326907640221:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.100344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637887:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.100422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325429637819:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.103075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325436772833:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.102977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241324583359519:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.103102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.102999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.103110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325436772833:2048] [node 2] ICP22 created new session: [1:7587241325436772917:2048]
2025-12-24T02:11:21.103006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241324583359519:2050] [node 3] ICP22 created new session: [5:7587241324583359603:2050]
2025-12-24T02:11:21.103118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325436772917:2048] [node 2] ICS09 handshake done sender: [1:7587241325436772896:2109] self: [1:7587241325436772895:0] peer: [2:7587241327103360741:0] socket: 54 qp: -1
2025-12-24T02:11:21.103013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241324583359603:2050] [node 3] ICS09 handshake done sender: [5:7587241324583359595:2115] self: [5:7587241324583359597:0] peer: [3:7587241324158187637:0] socket: 76 qp: -1
2025-12-24T02:11:21.103124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325436772917:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:21.103019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241324583359603:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.103132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.103069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.103489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359599:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.103627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187651:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
2025-12-24T02:11:21.103648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640235:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.103825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637891:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.103841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325429637891:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.105536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325429637891:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.106036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359602:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.106334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325436772902:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.106345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241324583359518:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.106398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241327103360655:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.106425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.106436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327103360655:2048] [node 1] ICP22 created new session: [2:7587241327103360746:2048]
2025-12-24T02:11:21.106443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327103360746:2048] [node 1] ICS09 handshake done sender: [2:7587241327103360725:2111] self: [2:7587241327103360741:0] peer: [1:7587241325436772895:0] socket: 59 qp: -1
2025-12-24T02:11:21.106451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327103360746:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.106470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.106517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325429637886:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.106537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.106546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325429637886:2109] [node 6] ICP22 created new session: [4:7587241325429637904:2109]
2025-12-24T02:11:21.106552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325429637904:2109] [node 6] ICS09 handshake done sender: [4:7587241325429637891:2114] self: [4:7587241325429637890:0] peer: [6:7587241326907640234:0] socket: 72 qp: -1
2025-12-24T02:11:21.106560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325429637904:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:21.106568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.107682Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360735:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.107249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325436772898:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.107274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.107283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325436772898:2108] [node 5] ICP22 created new session: [1:7587241325436772921:2108]
2025-12-24T02:11:21.107289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325436772921:2108] [node 5] ICS09 handshake done sender: [1:7587241325436772902:2111] self: [1:7587241325436772901:0] peer: [5:7587241324583359598:0] socket: 56 qp: -1
2025-12-24T02:11:21.107296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325436772921:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.107305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.106564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187636:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.106566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187640:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.106578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241324158187640:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.106583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241324158187636:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.107431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241324583359584:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.106395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241326907640235:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.107937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241326907640215:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.107967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640236:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.109330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241326907640236:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.109378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241326907640154:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.109507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640239:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.109522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187631:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.109552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359592:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.109564Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241324583359592:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.109594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241324583359602:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.109642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360732:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.109658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327103360732:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.109714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241326907640227:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.110558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637889:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.110578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325429637889:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.111195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325429637889:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.111419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187651:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.111551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241324158187565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.111687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241324158187636:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.112033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187652:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
2025-12-24T02:11:21.112115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241324158187632:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.112153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.112161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241324158187632:2107] [node 4] ICP22 created new session: [3:7587241324158187657:2107]
2025-12-24T02:11:21.112167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241324158187657:2107] [node 4] ICS09 handshake done sender: [3:7587241324158187636:2112] self: [3:7587241324158187635:0] peer: [4:7587241325429637902:0] socket: 65 qp: -1
2025-12-24T02:11:21.112173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241324158187657:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:21.112181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.112475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187653:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
2025-12-24T02:11:21.112610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360734:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.112627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327103360734:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.112645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640240:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.112667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241326907640221:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.112676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.112684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241326907640221:2109] [node 4] ICP22 created new session: [6:7587241326907640241:2109]
2025-12-24T02:11:21.112695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241326907640241:2109] [node 4] ICS09 handshake done sender: [6:7587241326907640227:2114] self: [6:7587241326907640234:0] peer: [4:7587241325429637890:0] socket: 77 qp: -1
2025-12-24T02:11:21.112701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241326907640241:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:21.112727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.112748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359608:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.113253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325429637885:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.113269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.113274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325429637885:2108] [node 5] ICP22 created new session: [4:7587241325429637907:2108]
2025-12-24T02:11:21.113292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325429637907:2108] [node 5] ICS09 handshake done sender: [4:7587241325429637889:2113] self: [4:7587241325429637888:0] peer: [5:7587241324583359606:0] socket: 71 qp: -1
2025-12-24T02:11:21.113297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325429637907:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.113303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.113681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241324583359517:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241324583359580:2108] held: no
2025-12-24T02:11:21.113196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241327103360655:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:21.113922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241324583359580:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.114840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327103360732:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.114957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772907:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.114974Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325436772907:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.115695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325436772923:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54052
2025-12-24T02:11:21.116165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327103360728:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.116186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.116192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327103360728:2112] [node 5] ICP22 created new session: [2:7587241327103360751:2112]
2025-12-24T02:11:21.116200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327103360751:2112] [node 5] ICS09 handshake done sender: [2:7587241327103360732:2114] self: [2:7587241327103360731:0] peer: [5:7587241324583359605:0] socket: 87 qp: -1
2025-12-24T02:11:21.116206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327103360751:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:21.116215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.116385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325436772907:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.116554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325436772834:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.116572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.116581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325436772834:2049] [node 3] ICP22 created new session: [1:7587241325436772926:2049]
2025-12-24T02:11:21.116591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325436772926:2049] [node 3] ICS09 handshake done sender: [1:7587241325436772907:2113] self: [1:7587241325436772900:0] peer: [3:7587241324158187655:0] socket: 57 qp: -1
2025-12-24T02:11:21.116598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325436772926:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.116610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.116738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637909:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.116809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637910:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.117362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241324158187652:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.117709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241324158187627:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.117848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241326907640240:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.117976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241324158187640:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.118562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327103360750:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8756
2025-12-24T02:11:21.118063Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325436772928:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54052
2025-12-24T02:11:21.118164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772923:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.118184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241325436772834:2049] [node 3] ICP18 handshake (actor [1:7587241325436772923:2117]) is too old
2025-12-24T02:11:21.118641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327103360734:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.119154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241324583359517:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.119170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.119178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241324583359517:2048] [node 1] ICP22 created new session: [5:7587241324583359610:2048]
2025-12-24T02:11:21.119186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241324583359610:2048] [node 1] ICS09 handshake done sender: [5:7587241324583359580:2108] self: [5:7587241324583359598:0] peer: [1:7587241325436772901:0] socket: 58 qp: -1
2025-12-24T02:11:21.119194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241324583359610:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.119201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.120642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241324583359518:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241324583359600:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.120687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359609:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.120794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241324583359599:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.120889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241324583359584:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.120901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.120908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241324583359584:2109] [node 4] ICP22 created new session: [5:7587241324583359615:2109]
2025-12-24T02:11:21.120914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241324583359615:2109] [node 4] ICS09 handshake done sender: [5:7587241324583359599:2117] self: [5:7587241324583359606:0] peer: [4:7587241325429637888:0] socket: 80 qp: -1
2025-12-24T02:11:21.120921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241324583359615:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:21.120928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.122003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241326907640153:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.122265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241324583359592:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.122341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772928:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.122427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772905:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.122438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325436772905:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.122480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241324583359585:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.122513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.122520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241324583359585:2110] [node 6] ICP22 created new session: [5:7587241324583359617:2110]
2025-12-24T02:11:21.122526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241324583359617:2110] [node 6] ICS09 handshake done sender: [5:7587241324583359592:2114] self: [5:7587241324583359588:0] peer: [6:7587241326907640237:0] socket: 81 qp: -1
2025-12-24T02:11:21.122532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241324583359617:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.122539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.123127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325436772929:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54052
2025-12-24T02:11:21.123330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327103360730:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.123346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.123351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327103360730:2113] [node 6] ICP22 created new session: [2:7587241327103360754:2113]
2025-12-24T02:11:21.123357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327103360754:2113] [node 6] ICS09 handshake done sender: [2:7587241327103360734:2115] self: [2:7587241327103360733:0] peer: [6:7587241326907640238:0] socket: 93 qp: -1
2025-12-24T02:11:21.123361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327103360754:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:21.123368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.123967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637909:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.124059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325429637887:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.124086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325436772905:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.124173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640245:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.124168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325429637819:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241325429637887:2112] held: no
2025-12-24T02:11:21.124257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325429637819:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.124265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.124474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640248:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.124270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325429637819:2050] [node 3] ICP22 created new session: [4:7587241325429637912:2050]
2025-12-24T02:11:21.124275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325429637912:2050] [node 3] ICS09 handshake done sender: [4:7587241325429637887:2112] self: [4:7587241325429637902:0] peer: [3:7587241324158187635:0] socket: 69 qp: -1
2025-12-24T02:11:21.124281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325429637912:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.124286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.124244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772929:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.124258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325436772903:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.124270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.124277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325436772903:2110] [node 6] ICP22 created new session: [1:7587241325436772932:2110]
2025-12-24T02:11:21.124283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325436772932:2110] [node 6] ICS09 handshake done sender: [1:7587241325436772905:2112] self: [1:7587241325436772904:0] peer: [6:7587241326907640244:0] socket: 104 qp: -1
2025-12-24T02:11:21.124798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640246:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.124288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325436772932:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.124295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.124794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327103360750:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.124826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241327103360730:2113] [node 6] ICP18 handshake (actor [2:7587241327103360750:2118]) is too old
2025-12-24T02:11:21.125046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241324158187566:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.125064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.125071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241324158187566:2049] [node 2] ICP22 created new session: [3:7587241324158187662:2049]
2025-12-24T02:11:21.125077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241324158187662:2049] [node 2] ICS09 handshake done sender: [3:7587241324158187627:2108] self: [3:7587241324158187646:0] peer: [2:7587241327103360718:0] socket: 50 qp: -1
2025-12-24T02:11:21.125083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241324158187662:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.125091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.125424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241324158187634:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.125431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.125436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241324158187634:2110] [node 6] ICP22 created new session: [3:7587241324158187664:2110]
2025-12-24T02:11:21.125441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241324158187664:2110] [node 6] ICS09 handshake done sender: [3:7587241324158187640:2114] self: [3:7587241324158187639:0] peer: [6:7587241326907640233:0] socket: 66 qp: -1
2025-12-24T02:11:21.125458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241324158187664:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.125463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.125603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241324158187634:2110] [node 6] ICP18 handshake (actor [3:7587241324158187652:2118]) is too old
2025-12-24T02:11:21.125928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241324158187660:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41252
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.126967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637911:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-12-24T02:11:21.127144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241325429637885:2108] [node 5] ICP18 handshake (actor [4:7587241325429637909:2118]) is too old
2025-12-24T02:11:21.125810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241324583359612:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34109
2025-12-24T02:11:21.126536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241324583359600:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.127218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325429637911:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.127244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325429637817:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.127326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325436772913:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.127336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325436772913:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.127742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241324583359518:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.127755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.127761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241324583359518:2049] [node 2] ICP22 created new session: [5:7587241324583359619:2049]
2025-12-24T02:11:21.127766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241324583359619:2049] [node 2] ICS09 handshake done sender: [5:7587241324583359600:2118] self: [5:7587241324583359605:0] peer: [2:7587241327103360731:0] socket: 94 qp: -1
2025-12-24T02:11:21.127776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241324583359619:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.127784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.127851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241326907640247:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3685
2025-12-24T02:11:21.128346Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241324158187631:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.128429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241324158187565:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241324158187631:2111] held: no
2025-12-24T02:11:21.128506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241324158187565:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.128519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.128525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241324158187565:2048] [node 1] ICP22 created new session: [3:7587241324158187666:2048]
2025-12-24T02:11:21.128531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241324158187666:2048] [node 1] ICS09 handshake done sender: [3:7587241324158187631:2111] self: [3:7587241324158187655:0] peer: [1:7587241325436772900:0] socket: 64 qp: -1
2025-12-24T02:11:21.128537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241324158187666:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.128543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.135365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241326907640153:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241326907640240:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.136820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327103360756:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8756
2025-12-24T02:11:21.137266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241326907640215:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15749) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241326907640235:2117] held: no
2025-12-24T02:11:21.137370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241326907640236:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.137435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325436772913:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.137642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325429637915:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20548
2025-12-24T02:11:21.137644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325436772897:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.137665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.137673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325436772897:2107] [node 4] ICP22 created new session: [1:7587241325436772936:2107]
2025-12-24T02:11:21.137681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325436772936:2107] [node 4] ICS09 handshake done sender: [1:7587241325436772913:2116] self: [1:7587241325436772911:0] peer: [4:7587241325429637914:0] socket: 115 qp: -1
2025-12-24T02:11:21.137687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325436772936:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:21.137695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.137991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325429637911:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.138082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325429637817:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.138102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.138111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325429637817:2048] [node 1] ICP22 created new session: [4:7587241325429637916:2048]
2025-12-24T02:11:21.138129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325429637916:2048] [node 1] ICS09 handshake done sender: [4:7587241325429637911:2120] self: [4:7587241325429637914:0] peer: [1:7587241325436772911:0] socket: 119 qp: -1
2025-12-24T02:11:21.138141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325429637916:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.138148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.145782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241326907640154:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13346) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241326907640236:2118] held: no
2025-12-24T02:11:21.145884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241326907640154:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.145903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.145916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241326907640154:2049] [node 2] ICP22 created new session: [6:7587241326907640249:2049]
2025-12-24T02:11:21.145938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241326907640249:2049] [node 2] ICS09 handshake done sender: [6:7587241326907640236:2118] self: [6:7587241326907640238:0] peer: [2:7587241327103360733:0] socket: 102 qp: -1
2025-12-24T02:11:21.145954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241326907640249:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.145966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.146262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241326907640225:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.146845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241326907640235:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.146960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241326907640240:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.147344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241326907640155:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9377) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241326907640225:2112] held: no
2025-12-24T02:11:21.147413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241326907640155:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.147426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.147432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241326907640155:2050] [node 3] ICP22 created new session: [6:7587241326907640251:2050]
2025-12-24T02:11:21.147442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241326907640251:2050] [node 3] ICS09 handshake done sender: [6:7587241326907640225:2112] self: [6:7587241326907640233:0] peer: [3:7587241324158187639:0] socket: 74 qp: -1
2025-12-24T02:11:21.147461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241326907640251:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.147472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.147529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241326907640215:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.147546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.147552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241326907640215:2107] [node 5] ICP22 created new session: [6:7587241326907640253:2107]
2025-12-24T02:11:21.147558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241326907640253:2107] [node 5] ICS09 handshake done sender: [6:7587241326907640235:2117] self: [6:7587241326907640237:0] peer: [5:7587241324583359588:0] socket: 83 qp: -1
2025-12-24T02:11:21.147564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241326907640253:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:21.147571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.150801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241326907640153:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.150827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.150836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241326907640153:2048] [node 1] ICP22 created new session: [6:7587241326907640255:2048]
2025-12-24T02:11:21.150842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241326907640255:2048] [node 1] ICS09 handshake done sender: [6:7587241326907640240:2120] self: [6:7587241326907640244:0] peer: [1:7587241325436772904:0] socket: 108 qp: -1
2025-12-24T02:11:21.150852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241326907640255:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.150865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.160239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:21.160341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241324158187666:2048] [node 1] ICS01 socket: 64 reason# EndOfStream
2025-12-24T02:11:21.160355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241324158187565:2048] [node 1] ICP30 unregister session Session# [3:7587241324158187666:2048] VirtualId# [3:7587241324158187655:0]
2025-12-24T02:11:21.160363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241324158187666:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.160433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.160470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.161605Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:21.161693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:21.161771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241324583359610:2048] [node 1] ICS01 socket: 58 reason# EndOfStream
2025-12-24T02:11:21.161778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241324583359517:2048] [node 1] ICP30 unregister session Session# [5:7587241324583359610:2048] VirtualId# [5:7587241324583359598:0]
2025-12-24T02:11:21.161785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241324583359610:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.161847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.161882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.161722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241325429637916:2048] [node 1] ICS07 socket disconnect 119 reason# EndOfStream
2025-12-24T02:11:21.161733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325429637916:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.161805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.161835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241325429637916:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:21.162073Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:21.162157Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325429637817:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.162163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241326907640255:2048] [node 1] ICS01 socket: 108 reason# EndOfStream
2025-12-24T02:11:21.162174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241326907640153:2048] [node 1] ICP30 unregister session Session# [6:7587241326907640255:2048] VirtualId# [6:7587241326907640244:0]
2025-12-24T02:11:21.162174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325429637916:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.162180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241326907640255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.162187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.162180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325429637817:2048] [node 1] ICP30 unregister session Session# [4:7587241325429637916:2048] VirtualId# [4:7587241325429637914:0]
2025-12-24T02:11:21.162229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.162243Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241325429637817:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.162245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9427) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.162231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.162747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:21.162837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241327103360746:2048] [node 1] ICS07 socket disconnect 59 reason# EndOfStream
2025-12-24T02:11:21.162849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241327103360746:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.162907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.162939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241327103360746:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:21.171604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241325429637817:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:21.172212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:21.172310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241324158187662:2049] [node 2] ICS07 socket disconnect 50 reason# EndOfStream
2025-12-24T02:11:21.172322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241324158187662:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.172390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.172425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241324158187662:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:21.172864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241324158187566:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13346) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.172883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241324158187662:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.172895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241324158187566:2049] [node 2] ICP30 unregister session Session# [3:7587241324158187662:2049] VirtualId# [3:7587241324158187646:0]
2025-12-24T02:11:21.172936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.172947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241324158187566:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.172950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13346) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.173457Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:21.173542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325429637899:2049] [node 2] ICS01 socket: 61 reason# EndOfStream
2025-12-24T02:11:21.173554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325429637818:2049] [node 2] ICP30 unregister session Session# [4:7587241325429637899:2049] VirtualId# [4:7587241325429637897:0]
2025-12-24T02:11:21.173560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325429637899:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.173641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.173685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.173837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:21.173930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241326907640249:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-12-24T02:11:21.173943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241326907640154:2049] [node 2] ICP30 unregister session Session# [6:7587241326907640249:2049] VirtualId# [6:7587241326907640238:0]
2025-12-24T02:11:21.173950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241326907640249:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.173691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:21.173793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241324583359619:2049] [node 2] ICS01 socket: 94 reason# EndOfStream
2025-12-24T02:11:21.173800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241324583359518:2049] [node 2] ICP30 unregister session Session# [5:7587241324583359619:2049] VirtualId# [5:7587241324583359605:0]
2025-12-24T02:11:21.173810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241324583359619:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.173884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.173914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.174009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.174044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.184541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:21.184656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325429637912:2050] [node 3] ICS01 socket: 69 reason# EndOfStream
2025-12-24T02:11:21.184673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325429637819:2050] [node 3] ICP30 unregister session Session# [4:7587241325429637912:2050] VirtualId# [4:7587241325429637902:0]
2025-12-24T02:11:21.184681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325429637912:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.184695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:21.184754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.184787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241324583359603:2050] [node 3] ICS01 socket: 76 reason# EndOfStream
2025-12-24T02:11:21.184804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241324583359519:2050] [node 3] ICP30 unregister session Session# [5:7587241324583359603:2050] VirtualId# [5:7587241324583359597:0]
2025-12-24T02:11:21.184786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.184811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241324583359603:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.184874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.184870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:21.184906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.184951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241326907640251:2050] [node 3] ICS01 socket: 74 reason# EndOfStream
2025-12-24T02:11:21.184986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241326907640155:2050] [node 3] ICP30 unregister session Session# [6:7587241326907640251:2050] VirtualId# [6:7587241326907640233:0]
2025-12-24T02:11:21.184992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241326907640251:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.185054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.185085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.191757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:21.191853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241324583359615:2109] [node 4] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:11:21.191861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241324583359584:2109] [node 4] ICP30 unregister session Session# [5:7587241324583359615:2109] VirtualId# [5:7587241324583359606:0]
2025-12-24T02:11:21.191867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241324583359615:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.191929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.191967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.191932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:21.192024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241326907640241:2109] [node 4] ICS07 socket disconnect 77 reason# EndOfStream
2025-12-24T02:11:21.192030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241326907640241:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.192081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.192103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241326907640241:2109] [node 4] ICS15 start handshake
2025-12-24T02:11:21.192508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241326907640221:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.192527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241326907640241:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.192537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241326907640221:2109] [node 4] ICP30 unregister session Session# [6:7587241326907640241:2109] VirtualId# [6:7587241326907640234:0]
2025-12-24T02:11:21.192574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.192587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241326907640221:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.192590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17488) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.202337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241326907640221:2109] [node 4] ICP33 wake up from error state
2025-12-24T02:11:21.207800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:21.207918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241326907640253:2107] [node 5] ICS01 socket: 83 reason# EndOfStream
2025-12-24T02:11:21.207927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241326907640215:2107] [node 5] ICP30 unregister session Session# [6:7587241326907640253:2107] VirtualId# [6:7587241326907640237:0]
2025-12-24T02:11:21.207937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241326907640253:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.207990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:21.208023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:21.230357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324752490702:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.230357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324752490701:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:21.232121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241326537393464:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.232133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241326537393465:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.234027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327600147752:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.234036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327600147753:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.235949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325127201138:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.235962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325127201140:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.235992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325127201139:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.237817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241326412521126:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.237829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241326412521128:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.237864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241326412521127:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.239639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325873965350:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.239648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325873965351:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.239656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325873965352:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.250589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325873965412:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:21.251027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241326537393525:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:21.251880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324752490766:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:21.252025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324752490768:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.257202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325127201199:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:21.257308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325127201201:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:21.262313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327600147811:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:21.263096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241326412521196:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:21.263170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241326412521197:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:21.263280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327600147819:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.267038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241327600147752:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16003594ns) {Configure:104(16002624ns) {ProcessConfigured:135(10865064ns) {StartInitialHandshake:157(10864193ns) {DropHandshakes:470(254ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(1129ns) {SwitchToState:179(453ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.268255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521202:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.270947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241326537393529:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:21.271006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241326537393530:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.276955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241324752490766:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24834648ns) {Configure:104(24833952ns) {ProcessConfigured:135(24550409ns) {StartInitialHandshake:157(24549727ns) {DropHandshakes:470(724ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(301ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(1407ns) {SwitchToState:179(299ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.277155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393541:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
2025-12-24T02:11:21.277179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393542:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
2025-12-24T02:11:21.277190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393543:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.277362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327600147818:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:21.277425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325127201199:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20040926ns) {Configure:104(20040419ns) {ProcessConfigured:135(19851056ns) {StartInitialHandshake:157(19850664ns) {DropHandshakes:470(333ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(173ns) PrepareNewSessionHandshake:477(853ns) {SwitchToState:179(378ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.277451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324752490773:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:21.277713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147829:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
2025-12-24T02:11:21.277758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241327600147753:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26861640ns) {Configure:104(26860566ns) {ProcessConfigured:135(21688943ns) {StartInitialHandshake:157(21688192ns) {DropHandshakes:470(540ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:477(677ns) {SwitchToState:179(151ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.277856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147828:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
2025-12-24T02:11:21.278482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325127201138:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22302194ns) {Configure:104(22301696ns) {ProcessConfigured:135(21072430ns) {StartInitialHandshake:157(21071907ns) {DropHandshakes:470(224ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(35ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(821ns) {SwitchToState:179(29ns)}}}}}
2025-12-24T02:11:21.278503Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325127201139:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22190908ns) {Configure:104(22190255ns) {ProcessConfigured:135(21152823ns) {StartInitialHandshake:157(21152231ns) {DropHandshakes:470(476ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(426ns) PrepareNewSessionHandshake:477(706ns) {SwitchToState:179(68ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:21.278610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241324752490768:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26248824ns) {Configure:104(26247879ns) {ProcessConfigured:135(25926708ns) {StartInitialHandshake:157(25926459ns) {DropHandshakes:470(558ns) {DropIncomingHandshake:442(136ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:477(390ns) {SwitchToState:179(85ns)}}}}}
::ffff:127.1.0.0
2025-12-24T02:11:21.278618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324752490775:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54859
2025-12-24T02:11:21.278616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147830:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
2025-12-24T02:11:21.278689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324752490776:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54859
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.279411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.279549Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241326412521197:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16275392ns) {Configure:104(16274683ns) {ProcessConfigured:135(15954861ns) {StartInitialHandshake:157(15954328ns) {DropHandshakes:470(469ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(636ns) {SwitchToState:179(97ns)}}}}}
2025-12-24T02:11:21.279569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324752490777:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54859
2025-12-24T02:11:21.279604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324752490778:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54859
2025-12-24T02:11:21.274976Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241325873965412:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24304867ns) {Configure:104(24304098ns) {ProcessConfigured:135(23842640ns) {StartInitialHandshake:157(23842055ns) {DropHandshakes:470(741ns) {DropIncomingHandshake:442(153ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(1386ns) {SwitchToState:179(1318ns)}}}}}
2025-12-24T02:11:21.277379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325873965422:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:21.277385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965424:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.278631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965425:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.279156Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241325873965352:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17843083ns) {Configure:104(17842573ns) {ProcessConfigured:135(17556459ns) {StartInitialHandshake:157(17556173ns) {DropHandshakes:470(387ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(701ns) {SwitchToState:179(48ns)}}}}}
2025-12-24T02:11:21.279334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965429:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.282888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201216:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.283051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521205:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.290469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490777:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.290603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490776:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.291261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490775:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.291789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490778:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.292214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147829:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.292867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147830:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.293044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147824:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.293063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327600147824:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.291280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325873965424:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.291346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325873965351:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.292882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325127201207:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.292901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325127201207:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.292150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325873965429:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.292177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325873965352:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.292697Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325873965425:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.292727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325873965422:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.291369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393542:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.291399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393543:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.291477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393531:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.291480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241326537393531:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.291716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521206:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.292015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521206:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.293073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521202:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.294175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325127201207:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.294313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325127201201:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.294346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.294357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325127201201:2107] [node 6] ICP22 created new session: [4:7587241325127201219:2107]
2025-12-24T02:11:21.294364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325127201219:2107] [node 6] ICS09 handshake done sender: [4:7587241325127201207:2110] self: [4:7587241325127201206:0] peer: [6:7587241325873965433:0] socket: 74 qp: -1
2025-12-24T02:11:21.294377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325127201219:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:21.294392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241326537393531:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.294399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.294643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241326537393525:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.294663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.294670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241326537393525:2107] [node 6] ICP22 created new session: [2:7587241326537393550:2107]
2025-12-24T02:11:21.294676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241326537393550:2107] [node 6] ICS09 handshake done sender: [2:7587241326537393531:2110] self: [2:7587241326537393526:0] peer: [6:7587241325873965431:0] socket: 58 qp: -1
2025-12-24T02:11:21.294683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241326537393550:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:21.294707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.295111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393549:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.296174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147822:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.296192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327600147822:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.296021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201221:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.297227Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327600147824:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.297260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147838:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
2025-12-24T02:11:21.297355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327600147811:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.297375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.297383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327600147811:2105] [node 6] ICP22 created new session: [3:7587241327600147839:2105]
2025-12-24T02:11:21.297389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327600147839:2105] [node 6] ICS09 handshake done sender: [3:7587241327600147824:2112] self: [3:7587241327600147823:0] peer: [6:7587241325873965432:0] socket: 81 qp: -1
2025-12-24T02:11:21.297397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327600147839:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:21.297406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.298763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147834:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.295734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325873965351:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241325873965424:2113] held: no
2025-12-24T02:11:21.296313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965434:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.297542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325873965425:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.297943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325873965422:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.297955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.297962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325873965422:2111] [node 4] ICP22 created new session: [6:7587241325873965438:2111]
2025-12-24T02:11:21.297968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325873965438:2111] [node 4] ICS09 handshake done sender: [6:7587241325873965425:2114] self: [6:7587241325873965433:0] peer: [4:7587241325127201206:0] socket: 78 qp: -1
2025-12-24T02:11:21.297975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325873965438:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:21.297982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.298619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965436:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.299036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325127201216:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.299074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325127201138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.299127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241327600147752:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.298931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521205:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.298987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241326412521196:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.299371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147828:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.299393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241327600147753:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.299213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490774:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.299224Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324752490774:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.299791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490772:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.299795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324752490772:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.299980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324752490774:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.300199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324752490766:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.300208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.300214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324752490766:2108] [node 4] ICP22 created new session: [1:7587241324752490789:2108]
2025-12-24T02:11:21.300219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324752490789:2108] [node 4] ICS09 handshake done sender: [1:7587241324752490774:2112] self: [1:7587241324752490770:0] peer: [4:7587241325127201222:0] socket: 92 qp: -1
2025-12-24T02:11:21.300225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324752490789:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:21.300232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.300642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324752490772:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.300708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147844:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.300786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325127201212:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.300794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325127201212:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.301374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201223:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.299005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325873965424:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.299080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325873965351:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.299089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.299094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325873965351:2049] [node 2] ICP22 created new session: [6:7587241325873965440:2049]
2025-12-24T02:11:21.299100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325873965440:2049] [node 2] ICS09 handshake done sender: [6:7587241325873965424:2113] self: [6:7587241325873965431:0] peer: [2:7587241326537393526:0] socket: 67 qp: -1
2025-12-24T02:11:21.299106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325873965440:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.299112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.302950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325127201212:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.301515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.303135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241326412521205:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.303177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241326412521196:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.303189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.303194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241326412521196:2109] [node 4] ICP22 created new session: [5:7587241326412521212:2109]
2025-12-24T02:11:21.303200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241326412521212:2109] [node 4] ICS09 handshake done sender: [5:7587241326412521205:2116] self: [5:7587241326412521209:0] peer: [4:7587241325127201203:0] socket: 89 qp: -1
2025-12-24T02:11:21.303205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241326412521212:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:21.303213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.303800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965442:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.304874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325127201199:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.304890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.304896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325127201199:2106] [node 5] ICP22 created new session: [4:7587241325127201225:2106]
2025-12-24T02:11:21.304902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325127201225:2106] [node 5] ICS09 handshake done sender: [4:7587241325127201212:2112] self: [4:7587241325127201203:0] peer: [5:7587241326412521209:0] socket: 88 qp: -1
2025-12-24T02:11:21.304908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325127201225:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:21.304915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.305250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325127201216:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.305471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325873965442:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.305509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325873965412:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.305713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521204:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.305724Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241326412521204:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.306956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327600147834:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.306273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965444:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.307219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393535:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.307237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241326537393535:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.307214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965437:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.307214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325127201221:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.308324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325127201139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.308630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241327600147752:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.308656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.308664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327600147752:2048] [node 1] ICP22 created new session: [3:7587241327600147845:2048]
2025-12-24T02:11:21.308672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327600147845:2048] [node 1] ICS09 handshake done sender: [3:7587241327600147834:2118] self: [3:7587241327600147842:0] peer: [1:7587241324752490764:0] socket: 93 qp: -1
2025-12-24T02:11:21.308679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327600147845:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.308697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.306593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241326412521128:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.306930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521210:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.307403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241326412521204:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.307497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241326412521197:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.307511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.307525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241326412521197:2111] [node 6] ICP22 created new session: [5:7587241326412521216:2111]
2025-12-24T02:11:21.307531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241326412521216:2111] [node 6] ICS09 handshake done sender: [5:7587241326412521204:2115] self: [5:7587241326412521199:0] peer: [6:7587241325873965443:0] socket: 116 qp: -1
2025-12-24T02:11:21.307538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241326412521216:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:21.307555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.309849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521210:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.310150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393553:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
2025-12-24T02:11:21.309934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241326412521126:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.310654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201229:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
2025-12-24T02:11:21.307310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325873965429:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.310639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521219:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.311416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393553:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.311462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393541:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.311484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393549:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.311524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241326537393464:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.311800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327600147822:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.310571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324752490794:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54859
2025-12-24T02:11:21.310882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324752490702:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.310902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.310908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324752490702:2049] [node 3] ICP22 created new session: [1:7587241324752490795:2049]
2025-12-24T02:11:21.310936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324752490795:2049] [node 3] ICS09 handshake done sender: [1:7587241324752490772:2111] self: [1:7587241324752490764:0] peer: [3:7587241327600147842:0] socket: 90 qp: -1
2025-12-24T02:11:21.311846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241326537393535:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.310942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324752490795:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.310950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.310952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490779:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.310964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324752490779:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.311971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241326537393465:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.311994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.312001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241326537393465:2049] [node 3] ICP22 created new session: [2:7587241326537393558:2049]
2025-12-24T02:11:21.312008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241326537393558:2049] [node 3] ICS09 handshake done sender: [2:7587241326537393535:2112] self: [2:7587241326537393534:0] peer: [3:7587241327600147843:0] socket: 60 qp: -1
2025-12-24T02:11:21.312014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241326537393558:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.312021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.312299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393544:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.312314Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241326537393544:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.312355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327600147819:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.312381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.312404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327600147819:2109] [node 5] ICP22 created new session: [3:7587241327600147849:2109]
2025-12-24T02:11:21.312416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327600147849:2109] [node 5] ICS09 handshake done sender: [3:7587241327600147822:2111] self: [3:7587241327600147821:0] peer: [5:7587241326412521208:0] socket: 70 qp: -1
2025-12-24T02:11:21.312422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327600147849:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.312428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.312619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490794:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.312700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325127201138:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.312723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.312729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325127201138:2048] [node 1] ICP22 created new session: [4:7587241325127201230:2048]
2025-12-24T02:11:21.312735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325127201230:2048] [node 1] ICS09 handshake done sender: [4:7587241325127201216:2115] self: [4:7587241325127201222:0] peer: [1:7587241324752490770:0] socket: 94 qp: -1
2025-12-24T02:11:21.312740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325127201230:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.312750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.312903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241326412521202:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.312993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241326412521128:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.313021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.313030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241326412521128:2050] [node 3] ICP22 created new session: [5:7587241326412521220:2050]
2025-12-24T02:11:21.313037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241326412521220:2050] [node 3] ICS09 handshake done sender: [5:7587241326412521202:2113] self: [5:7587241326412521208:0] peer: [3:7587241327600147821:0] socket: 71 qp: -1
2025-12-24T02:11:21.313044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241326412521220:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.313053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.313101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325873965412:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9980) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241325873965442:2121] held: no
2025-12-24T02:11:21.313184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325873965352:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.313192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.313197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325873965352:2050] [node 3] ICP22 created new session: [6:7587241325873965446:2050]
2025-12-24T02:11:21.313202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325873965446:2050] [node 3] ICS09 handshake done sender: [6:7587241325873965429:2116] self: [6:7587241325873965432:0] peer: [3:7587241327600147823:0] socket: 87 qp: -1
2025-12-24T02:11:21.313209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325873965446:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.313216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.313388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325873965442:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.313658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241324752490766:2108] [node 4] ICP18 handshake (actor [1:7587241324752490794:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.313910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521222:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.314000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325873965412:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.314011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.314021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325873965412:2108] [node 5] ICP22 created new session: [6:7587241325873965448:2108]
2025-12-24T02:11:21.314027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325873965448:2108] [node 5] ICS09 handshake done sender: [6:7587241325873965442:2121] self: [6:7587241325873965443:0] peer: [5:7587241326412521199:0] socket: 117 qp: -1
2025-12-24T02:11:21.314032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325873965448:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.314049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.314245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965445:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.315076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241326537393544:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.315236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241326537393529:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.315255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.315261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241326537393529:2108] [node 4] ICP22 created new session: [2:7587241326537393562:2108]
2025-12-24T02:11:21.315266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241326537393562:2108] [node 4] ICS09 handshake done sender: [2:7587241326537393544:2116] self: [2:7587241326537393539:0] peer: [4:7587241325127201228:0] socket: 82 qp: -1
2025-12-24T02:11:21.315271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241326537393562:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:21.315577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241326412521222:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.315277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.315616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241326412521127:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.316205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490762:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.316218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324752490762:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.316203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325873965445:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.316233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325873965350:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.316589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521224:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.316870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324752490779:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.316969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241326412521210:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.317122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147838:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.317103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241326412521126:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.317122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.317129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241326412521126:2048] [node 1] ICP22 created new session: [5:7587241326412521225:2048]
2025-12-24T02:11:21.317137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241326412521225:2048] [node 1] ICS09 handshake done sender: [5:7587241326412521210:2118] self: [5:7587241326412521218:0] peer: [1:7587241324752490771:0] socket: 112 qp: -1
2025-12-24T02:11:21.317145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241326412521225:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.317152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.317275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241327600147753:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241327600147828:2113] held: no
2025-12-24T02:11:21.317130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324752490783:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.317140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324752490783:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.317789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324752490768:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.317808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.317815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324752490768:2109] [node 5] ICP22 created new session: [1:7587241324752490798:2109]
2025-12-24T02:11:21.317821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324752490798:2109] [node 5] ICS09 handshake done sender: [1:7587241324752490779:2117] self: [1:7587241324752490771:0] peer: [5:7587241326412521218:0] socket: 110 qp: -1
2025-12-24T02:11:21.317827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324752490798:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.317834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.318237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241326537393545:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.318253Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241326537393545:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.318263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325127201229:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.318289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325127201140:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.318306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241327600147811:2105] [node 6] ICP18 handshake (actor [3:7587241327600147838:2119]) is too old
2025-12-24T02:11:21.319483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327600147851:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20007
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.320059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201233:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
2025-12-24T02:11:21.320179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325127201221:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.320190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241326537393545:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.320280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325127201139:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.320297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.320304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325127201139:2049] [node 2] ICP22 created new session: [4:7587241325127201234:2049]
2025-12-24T02:11:21.320311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325127201234:2049] [node 2] ICS09 handshake done sender: [4:7587241325127201221:2116] self: [4:7587241325127201228:0] peer: [2:7587241326537393539:0] socket: 102 qp: -1
2025-12-24T02:11:21.320317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325127201234:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.320323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.320327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241326537393530:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.320345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.320353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241326537393530:2109] [node 5] ICP22 created new session: [2:7587241326537393565:2109]
2025-12-24T02:11:21.320361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241326537393565:2109] [node 5] ICS09 handshake done sender: [2:7587241326537393545:2117] self: [2:7587241326537393540:0] peer: [5:7587241326412521223:0] socket: 91 qp: -1
2025-12-24T02:11:21.320367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241326537393565:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.320402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.321325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325873965451:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43515
2025-12-24T02:11:21.321605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324752490762:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.321717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324752490701:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.321739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.321745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324752490701:2048] [node 2] ICP22 created new session: [1:7587241324752490802:2048]
2025-12-24T02:11:21.321752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324752490802:2048] [node 2] ICS09 handshake done sender: [1:7587241324752490762:2107] self: [1:7587241324752490761:0] peer: [2:7587241326537393555:0] socket: 54 qp: -1
2025-12-24T02:11:21.321758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324752490802:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:21.321763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.324317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241326412521127:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241326412521222:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.325203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324752490783:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.325396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325873965445:2123] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.325488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325873965350:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.325502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.325509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325873965350:2048] [node 1] ICP22 created new session: [6:7587241325873965452:2048]
2025-12-24T02:11:21.325516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325873965452:2048] [node 1] ICS09 handshake done sender: [6:7587241325873965445:2123] self: [6:7587241325873965450:0] peer: [1:7587241324752490782:0] socket: 125 qp: -1
2025-12-24T02:11:21.325523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325873965452:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.325530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.325656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241326412521227:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64550
2025-12-24T02:11:21.326030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241326412521222:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.326608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324752490773:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.326628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.326636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324752490773:2110] [node 6] ICP22 created new session: [1:7587241324752490804:2110]
2025-12-24T02:11:21.326643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324752490804:2110] [node 6] ICS09 handshake done sender: [1:7587241324752490783:2118] self: [1:7587241324752490782:0] peer: [6:7587241325873965450:0] socket: 69 qp: -1
2025-12-24T02:11:21.326649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324752490804:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.326656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.326183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241326412521127:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.326200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.326207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241326412521127:2049] [node 2] ICP22 created new session: [5:7587241326412521228:2049]
2025-12-24T02:11:21.326214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241326412521228:2049] [node 2] ICS09 handshake done sender: [5:7587241326412521222:2121] self: [5:7587241326412521223:0] peer: [2:7587241326537393540:0] socket: 107 qp: -1
2025-12-24T02:11:21.326221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241326412521228:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.326229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.326489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241326537393464:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19414) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241326537393541:2113] held: no
2025-12-24T02:11:21.327920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327600147828:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.328038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327600147833:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.328050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327600147833:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.328807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327600147833:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.328817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325127201236:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3373
2025-12-24T02:11:21.328881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241327600147753:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.328897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.328903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327600147753:2049] [node 2] ICP22 created new session: [3:7587241327600147853:2049]
2025-12-24T02:11:21.328908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327600147853:2049] [node 2] ICS09 handshake done sender: [3:7587241327600147828:2113] self: [3:7587241327600147843:0] peer: [2:7587241326537393534:0] socket: 66 qp: -1
2025-12-24T02:11:21.328914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327600147853:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.328944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.329009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325127201229:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.329214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327600147818:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.329230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.329234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327600147818:2108] [node 4] ICP22 created new session: [3:7587241327600147855:2108]
2025-12-24T02:11:21.329239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327600147855:2108] [node 4] ICS09 handshake done sender: [3:7587241327600147833:2117] self: [3:7587241327600147832:0] peer: [4:7587241325127201232:0] socket: 122 qp: -1
2025-12-24T02:11:21.329243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327600147855:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:21.329247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.329489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325127201140:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.329516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.329525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325127201140:2050] [node 3] ICP22 created new session: [4:7587241325127201237:2050]
2025-12-24T02:11:21.329532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325127201237:2050] [node 3] ICS09 handshake done sender: [4:7587241325127201229:2118] self: [4:7587241325127201232:0] peer: [3:7587241327600147832:0] socket: 124 qp: -1
2025-12-24T02:11:21.329537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325127201237:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.329552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.333690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241326537393567:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39934
2025-12-24T02:11:21.333810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241326537393541:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.333870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241326537393464:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.333894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.333907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241326537393464:2048] [node 1] ICP22 created new session: [2:7587241326537393568:2048]
2025-12-24T02:11:21.333913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241326537393568:2048] [node 1] ICS09 handshake done sender: [2:7587241326537393541:2113] self: [2:7587241326537393555:0] peer: [1:7587241324752490761:0] socket: 56 qp: -1
2025-12-24T02:11:21.333919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241326537393568:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.333930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.346953Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:21.347067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241326537393568:2048] [node 1] ICS01 socket: 56 reason# EndOfStream
2025-12-24T02:11:21.347082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241326537393464:2048] [node 1] ICP30 unregister session Session# [2:7587241326537393568:2048] VirtualId# [2:7587241326537393555:0]
2025-12-24T02:11:21.347089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241326537393568:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.347105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.347164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.347208Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:21.347289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241327600147845:2048] [node 1] ICS01 socket: 93 reason# EndOfStream
2025-12-24T02:11:21.347297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241327600147752:2048] [node 1] ICP30 unregister session Session# [3:7587241327600147845:2048] VirtualId# [3:7587241327600147842:0]
2025-12-24T02:11:21.347301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241327600147845:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.347339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.347367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.349003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:21.349093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241325127201230:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-12-24T02:11:21.349099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325127201230:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.349150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.349172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241325127201230:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:21.351266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:21.351364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241326412521225:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-12-24T02:11:21.351377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241326412521126:2048] [node 1] ICP30 unregister session Session# [5:7587241326412521225:2048] VirtualId# [5:7587241326412521218:0]
2025-12-24T02:11:21.351382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241326412521225:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.351495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:21.351539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325873965452:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-12-24T02:11:21.351542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325873965350:2048] [node 1] ICP30 unregister session Session# [6:7587241325873965452:2048] VirtualId# [6:7587241325873965450:0]
2025-12-24T02:11:21.351545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325873965452:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.351568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.351589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.351798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325127201138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19414) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.351813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325127201230:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.351819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325127201138:2048] [node 1] ICP30 unregister session Session# [4:7587241325127201230:2048] VirtualId# [4:7587241325127201222:0]
2025-12-24T02:11:21.351858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.351863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241325127201138:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.351866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19414) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.351417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.351444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.361534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:21.361610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241327600147853:2049] [node 2] ICS01 socket: 66 reason# EndOfStream
2025-12-24T02:11:21.361618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241327600147753:2049] [node 2] ICP30 unregister session Session# [3:7587241327600147853:2049] VirtualId# [3:7587241327600147843:0]
2025-12-24T02:11:21.361622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241327600147853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.361663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.361678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.362036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241325127201138:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:21.362944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:21.363026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241326412521228:2049] [node 2] ICS01 socket: 107 reason# EndOfStream
2025-12-24T02:11:21.363039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241326412521127:2049] [node 2] ICP30 unregister session Session# [5:7587241326412521228:2049] VirtualId# [5:7587241326412521223:0]
2025-12-24T02:11:21.363045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241326412521228:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.362525Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:21.362616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325873965440:2049] [node 2] ICS01 socket: 67 reason# EndOfStream
2025-12-24T02:11:21.362623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325873965351:2049] [node 2] ICP30 unregister session Session# [6:7587241325873965440:2049] VirtualId# [6:7587241325873965431:0]
2025-12-24T02:11:21.362629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325873965440:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.362720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.362758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.363096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.363153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.364604Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:21.364686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241325127201234:2049] [node 2] ICS07 socket disconnect 102 reason# EndOfStream
2025-12-24T02:11:21.364690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325127201234:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.364750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.364777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241325127201234:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:21.365164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325127201139:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.365177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325127201234:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.365209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325127201139:2049] [node 2] ICP30 unregister session Session# [4:7587241325127201234:2049] VirtualId# [4:7587241325127201228:0]
2025-12-24T02:11:21.365249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.365256Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241325127201139:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.365259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65179) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.375506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241325127201139:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:21.377432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:21.378177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:21.378249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241325127201237:2050] [node 3] ICS07 socket disconnect 124 reason# EndOfStream
2025-12-24T02:11:21.378258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325127201237:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.378291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.378313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241325127201237:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:21.378444Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:21.378559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241326412521220:2050] [node 3] ICS01 socket: 71 reason# EndOfStream
2025-12-24T02:11:21.378573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241326412521128:2050] [node 3] ICP30 unregister session Session# [5:7587241326412521220:2050] VirtualId# [5:7587241326412521208:0]
2025-12-24T02:11:21.378579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241326412521220:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.378632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.378657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.378888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325127201140:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10062) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.378907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325127201237:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.378914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325127201140:2050] [node 3] ICP30 unregister session Session# [4:7587241325127201237:2050] VirtualId# [4:7587241325127201232:0]
2025-12-24T02:11:21.378946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.378958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241325127201140:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.378960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10062) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.377552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325873965446:2050] [node 3] ICS01 socket: 87 reason# EndOfStream
2025-12-24T02:11:21.377564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325873965352:2050] [node 3] ICP30 unregister session Session# [6:7587241325873965446:2050] VirtualId# [6:7587241325873965432:0]
2025-12-24T02:11:21.377571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325873965446:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.377617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.377638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.392234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:21.392044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:21.392146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241326412521212:2109] [node 4] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:11:21.392156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241326412521196:2109] [node 4] ICP30 unregister session Session# [5:7587241326412521212:2109] VirtualId# [5:7587241326412521209:0]
2025-12-24T02:11:21.392163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241326412521212:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.392230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.392259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.392323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325873965438:2111] [node 4] ICS01 socket: 78 reason# EndOfStream
2025-12-24T02:11:21.392333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325873965422:2111] [node 4] ICP30 unregister session Session# [6:7587241325873965438:2111] VirtualId# [6:7587241325873965433:0]
2025-12-24T02:11:21.392340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325873965438:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.392381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.392406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:21.435965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324680085843:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:21.435980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324680085844:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.437869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327810898274:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.437882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327810898275:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.439562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241325628452585:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.439569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241325628452586:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.441573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325874232649:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.441589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325874232651:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.441614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325874232650:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.443749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328179895507:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.443762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328179895509:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.443858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328179895508:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.446234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325507385065:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.446258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325507385064:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.446266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325507385066:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.459073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325507385126:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.462389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324680085912:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:21.463107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385130:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.465045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324680085911:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:21.464948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452645:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.465708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325507385130:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.466045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325507385064:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.467026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895568:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.467057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898337:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
2025-12-24T02:11:21.469162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328179895568:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.469356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328179895570:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:21.469727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328179895570:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.469911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325507385128:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.469919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241325507385128:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.470572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895572:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.470606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325507385128:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.470814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328179895568:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.470976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328179895570:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.470994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.471002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328179895570:2109] [node 6] ICP22 created new session: [5:7587241328179895573:2109]
2025-12-24T02:11:21.471009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328179895573:2109] [node 6] ICS09 handshake done sender: [5:7587241328179895568:2108] self: [5:7587241328179895571:0] peer: [6:7587241325507385127:0] socket: 61 qp: -1
2025-12-24T02:11:21.471020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328179895573:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:21.471048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.471076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241324680085910:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.471603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241325507385126:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.471617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.471623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325507385126:2108] [node 5] ICP22 created new session: [6:7587241325507385133:2108]
2025-12-24T02:11:21.471628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325507385133:2108] [node 5] ICS09 handshake done sender: [6:7587241325507385128:2109] self: [6:7587241325507385127:0] peer: [5:7587241328179895571:0] socket: 60 qp: -1
2025-12-24T02:11:21.471633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325507385133:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.471652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.471139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898337:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.471196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241327810898274:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.471340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327810898341:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:21.479242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085914:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.479269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324680085914:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.479761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085908:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.479765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324680085908:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.483995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324680085914:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.484274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324680085908:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.486757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324680085919:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55334
2025-12-24T02:11:21.486793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324680085843:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.486807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.486816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324680085843:2048] [node 2] ICP22 created new session: [1:7587241324680085923:2048]
2025-12-24T02:11:21.486822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324680085923:2048] [node 2] ICS09 handshake done sender: [1:7587241324680085908:2110] self: [1:7587241324680085905:0] peer: [2:7587241327810898340:0] socket: 63 qp: -1
2025-12-24T02:11:21.486829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324680085923:2048] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.487027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385135:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.489029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241327810898341:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17586626ns) {Configure:104(17585703ns) {ProcessConfigured:135(17234531ns) {StartInitialHandshake:157(17234045ns) {DropHandshakes:470(515ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(795ns)}}}}}
2025-12-24T02:11:21.489169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898348:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
2025-12-24T02:11:21.489934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325507385130:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.489342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241325628452654:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:21.486839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.487230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241324680085910:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16049424ns) {Configure:104(16048935ns) {ProcessConfigured:135(15725353ns) {StartInitialHandshake:157(15724801ns) {DropHandshakes:470(370ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(553ns) {SwitchToState:179(49ns)}}}}}
2025-12-24T02:11:21.487245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324680085912:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.487251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.487255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324680085912:2109] [node 6] ICP22 created new session: [1:7587241324680085926:2109]
2025-12-24T02:11:21.487259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324680085926:2109] [node 6] ICS09 handshake done sender: [1:7587241324680085914:2111] self: [1:7587241324680085913:0] peer: [6:7587241325507385131:0] socket: 66 qp: -1
2025-12-24T02:11:21.487264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324680085926:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:21.487288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.490332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327810898337:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.489524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232710:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.490481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325874232712:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.490762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241327810898274:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.490776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.490781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327810898274:2048] [node 1] ICP22 created new session: [2:7587241327810898350:2048]
2025-12-24T02:11:21.490788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327810898350:2048] [node 1] ICS09 handshake done sender: [2:7587241327810898337:2108] self: [2:7587241327810898340:0] peer: [1:7587241324680085905:0] socket: 64 qp: -1
2025-12-24T02:11:21.490795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327810898350:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.490812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.494978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328179895578:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:21.495542Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241324680085911:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30294268ns) {Configure:104(30293281ns) {ProcessConfigured:135(29955509ns) {StartInitialHandshake:157(29955123ns) {DropHandshakes:470(548ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(921ns) {SwitchToState:179(221ns)}}}}}
2025-12-24T02:11:21.495707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241325507385126:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24061865ns) {HandleHandshakeStatus:329(24061211ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(200ns) SwitchToState:179(87ns) ProcessPendingSessionEvents:495(8748567ns) {ForwardSessionEventToSession:612(8734556ns) ForwardSessionEventToSession:612(2121ns) ForwardSessionEventToSession:612(927ns) ForwardSessionEventToSession:612(714ns) ForwardSessionEventToSession:612(649ns) ForwardSessionEventToSession:612(1007ns) ForwardSessionEventToSession:612(660ns) ForwardSessionEventToSession:612(911ns) ForwardSessionEventToSession:612(931ns) ForwardSessionEventToSession:612(861ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.499232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895580:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.500483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325507385064:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.500515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.500527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325507385064:2048] [node 1] ICP22 created new session: [6:7587241325507385137:2048]
2025-12-24T02:11:21.500534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325507385137:2048] [node 1] ICS09 handshake done sender: [6:7587241325507385130:2110] self: [6:7587241325507385131:0] peer: [1:7587241324680085913:0] socket: 67 qp: -1
2025-12-24T02:11:21.500543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325507385137:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.500570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.500925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327810898345:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:21.501783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328179895580:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.500330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452645:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.500382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241325628452585:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.502726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241327810898346:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.506871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328179895507:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.506359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232710:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.506449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232714:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.505079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241325628452651:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:21.505725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241325628452650:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:21.508760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325507385140:2112] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.507105Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241327810898275:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40258587ns) {Configure:104(40257690ns) {ProcessConfigured:135(39939596ns) {StartInitialHandshake:157(39939089ns) {DropHandshakes:470(439ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(390ns) PrepareNewSessionHandshake:477(667ns) {SwitchToState:179(454ns)}}}}}
2025-12-24T02:11:21.507965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325874232649:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18598611ns) {Configure:104(18597478ns) {ProcessConfigured:135(16360437ns) {StartInitialHandshake:157(16360090ns) {DropHandshakes:470(340ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(607ns) {SwitchToState:179(80ns)}}}}}
2025-12-24T02:11:21.510535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325874232649:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.510753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232723:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.510852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325874232650:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21444083ns) {Configure:104(21443313ns) {ProcessConfigured:135(19052574ns) {StartInitialHandshake:157(19051629ns) {DropHandshakes:470(390ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(733ns) {SwitchToState:179(186ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.513253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085925:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.513269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324680085925:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.514065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324680085925:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.514155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324680085910:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.514170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.514178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324680085910:2107] [node 4] ICP22 created new session: [1:7587241324680085932:2107]
2025-12-24T02:11:21.514184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324680085932:2107] [node 4] ICS09 handshake done sender: [1:7587241324680085925:2114] self: [1:7587241324680085920:0] peer: [4:7587241325874232724:0] socket: 78 qp: -1
2025-12-24T02:11:21.514190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324680085932:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:21.514209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.511463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452662:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.511525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452663:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.511539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452664:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.513961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452662:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.514603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898357:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.515400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232728:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.517057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325874232710:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.517196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325874232649:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.517214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.517223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325874232649:2048] [node 1] ICP22 created new session: [4:7587241325874232729:2048]
2025-12-24T02:11:21.517231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325874232729:2048] [node 1] ICS09 handshake done sender: [4:7587241325874232710:2108] self: [4:7587241325874232724:0] peer: [1:7587241324680085920:0] socket: 80 qp: -1
2025-12-24T02:11:21.517241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325874232729:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.517275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.514817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241328179895508:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22880885ns) {Configure:104(22880166ns) {ProcessConfigured:135(22584263ns) {StartInitialHandshake:157(22583722ns) {DropHandshakes:470(204ns) {DropIncomingHandshake:442(15ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:477(1209ns) {SwitchToState:179(496ns)}}}}}
2025-12-24T02:11:21.518590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241328179895578:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23548962ns) {Configure:104(23548329ns) {ProcessConfigured:135(23299217ns) {StartInitialHandshake:157(23298662ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(805ns) {SwitchToState:179(323ns)}}}}}
2025-12-24T02:11:21.518112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241324680085843:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(31229327ns) {HandleHandshakeStatus:329(31228252ns) {DropIncomingHandshake:442(184ns) DropOutgoingHandshake:457(297ns) SwitchToState:179(170ns) ProcessPendingSessionEvents:495(23240710ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23224984ns max# 23163839ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.519258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085929:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.519274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324680085929:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.519139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241325507385065:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19869048ns) {Configure:104(19868375ns) {ProcessConfigured:135(19371020ns) {StartInitialHandshake:157(19370672ns) {DropHandshakes:470(1211ns) {DropIncomingHandshake:442(100ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(440ns) {SwitchToState:179(85ns)}}}}}
2025-12-24T02:11:21.519353Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241325507385064:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(29322312ns) {HandleHandshakeStatus:329(29321436ns) {DropIncomingHandshake:442(860ns) DropOutgoingHandshake:457(78ns) SwitchToState:179(231ns) ProcessPendingSessionEvents:495(721ns)}}
2025-12-24T02:11:21.518904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898358:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
2025-12-24T02:11:21.519848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325874232715:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:21.519944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232723:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.521139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452664:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.521179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452663:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.522861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085916:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.522884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241324680085916:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.527322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241327810898274:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(36432144ns) {HandleHandshakeStatus:329(36431540ns) {DropIncomingHandshake:442(242ns) DropOutgoingHandshake:457(1413ns) SwitchToState:179(72ns) ProcessPendingSessionEvents:495(26364591ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26343286ns max# 26265021ns}}}
2025-12-24T02:11:21.527693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898358:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.527715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898357:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.528059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241325628452586:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49220559ns) {Configure:104(49219511ns) {ProcessConfigured:135(48861772ns) {StartInitialHandshake:157(48861117ns) {DropHandshakes:470(725ns) {DropIncomingHandshake:442(92ns) DropOutgoingHandshake:457(163ns)} GenerateSessionVirtualId:988(951ns) PrepareNewSessionHandshake:477(1281ns) {SwitchToState:179(877ns)}}}}}
2025-12-24T02:11:21.528240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241325628452586:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.530987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385149:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.531072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385150:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
2025-12-24T02:11:21.531479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325874232712:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(40250760ns) {Configure:104(30010089ns) {ProcessConfigured:135(29699542ns) {StartInitialHandshake:157(29698126ns) {DropHandshakes:470(403ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:477(1070ns) {SwitchToState:179(70ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.531691Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241328179895509:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39236989ns) {Configure:104(29919290ns) {ProcessConfigured:135(29659902ns) {StartInitialHandshake:157(29658830ns) {DropHandshakes:470(162ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(734ns) {SwitchToState:179(305ns)}}}}}
2025-12-24T02:11:21.531909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895590:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.528463Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241327810898345:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27389825ns) {Configure:104(27389350ns) {ProcessConfigured:135(27076128ns) {StartInitialHandshake:157(27075817ns) {DropHandshakes:470(507ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(573ns) {SwitchToState:179(241ns)}}}}}
2025-12-24T02:11:21.528527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898353:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.528538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327810898353:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.532801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327810898353:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.533340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324680085934:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55334
2025-12-24T02:11:21.533223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325507385150:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.533271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325507385065:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.533044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327810898275:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.533061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.533067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327810898275:2049] [node 3] ICP22 created new session: [2:7587241327810898364:2049]
2025-12-24T02:11:21.533074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327810898364:2049] [node 3] ICS09 handshake done sender: [2:7587241327810898353:2114] self: [2:7587241327810898338:0] peer: [3:7587241325628452668:0] socket: 92 qp: -1
2025-12-24T02:11:21.533085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327810898364:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.533098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.533429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324680085929:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.533562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324680085936:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55334
2025-12-24T02:11:21.533611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241324680085935:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55334
2025-12-24T02:11:21.533675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452669:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.533680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241324680085910:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:21.533694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324680085911:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.533704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.533711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324680085911:2108] [node 5] ICP22 created new session: [1:7587241324680085938:2108]
2025-12-24T02:11:21.533717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324680085938:2108] [node 5] ICS09 handshake done sender: [1:7587241324680085929:2115] self: [1:7587241324680085918:0] peer: [5:7587241328179895584:0] socket: 86 qp: -1
2025-12-24T02:11:21.533724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324680085938:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.533733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.536215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241325628452663:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.539074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325507385149:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.539131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325507385066:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.547030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898367:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
2025-12-24T02:11:21.547044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241327810898368:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63328
2025-12-24T02:11:21.547062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232734:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.547135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232714:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.547174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325874232650:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.547412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898347:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.547417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327810898347:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.548264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232734:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.548919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327810898347:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.548862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232737:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.548999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325874232714:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.549157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325874232650:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.549181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.549189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325874232650:2049] [node 2] ICP22 created new session: [4:7587241325874232738:2049]
2025-12-24T02:11:21.549194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325874232738:2049] [node 2] ICS09 handshake done sender: [4:7587241325874232714:2111] self: [4:7587241325874232735:0] peer: [2:7587241327810898342:0] socket: 82 qp: -1
2025-12-24T02:11:21.549200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325874232738:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.549219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.550805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241325874232715:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30638163ns) {Configure:104(30637402ns) {ProcessConfigured:135(30302120ns) {StartInitialHandshake:157(30301248ns) {DropHandshakes:470(760ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(1161ns) {SwitchToState:179(283ns)}}}}}
2025-12-24T02:11:21.555254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085934:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.555287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241324680085935:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.557075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241325628452651:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51834291ns) {Configure:104(51833111ns) {ProcessConfigured:135(51409775ns) {StartInitialHandshake:157(51409163ns) {DropHandshakes:470(772ns) {DropIncomingHandshake:442(213ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(400ns) PrepareNewSessionHandshake:477(1022ns) {SwitchToState:179(64ns)}}}}}
2025-12-24T02:11:21.557125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452657:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.557128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241325628452657:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.557929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.558029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241325628452650:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52179126ns) {Configure:104(52178573ns) {ProcessConfigured:135(51897970ns) {StartInitialHandshake:157(51897582ns) {DropHandshakes:470(406ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(175ns) PrepareNewSessionHandshake:477(753ns) {SwitchToState:179(200ns)}}}}}
2025-12-24T02:11:21.557308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241324680085916:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.558438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241324680085911:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24638810ns) {HandleHandshakeStatus:329(24638206ns) {DropIncomingHandshake:442(113ns) DropOutgoingHandshake:457(142ns) SwitchToState:179(271ns) ProcessPendingSessionEvents:495(24567844ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24548115ns max# 24478039ns}}}
2025-12-24T02:11:21.558478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241324680085911:2108] [node 5] ICP18 handshake (actor [1:7587241324680085935:2117]) is too old
2025-12-24T02:11:21.559115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241324680085844:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.559500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241325628452657:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.559650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.559663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241324680085844:2049] [node 3] ICP22 created new session: [1:7587241324680085942:2049]
2025-12-24T02:11:21.559670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241324680085942:2049] [node 3] ICS09 handshake done sender: [1:7587241324680085916:2112] self: [1:7587241324680085907:0] peer: [3:7587241325628452659:0] socket: 68 qp: -1
2025-12-24T02:11:21.559677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241324680085942:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.559688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.560851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241325628452654:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.560872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.560881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241325628452654:2110] [node 6] ICP22 created new session: [3:7587241325628452675:2110]
2025-12-24T02:11:21.560888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241325628452675:2110] [node 6] ICS09 handshake done sender: [3:7587241325628452657:2112] self: [3:7587241325628452655:0] peer: [6:7587241325507385152:0] socket: 88 qp: -1
2025-12-24T02:11:21.560895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241325628452675:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.560914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.561149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241325628452585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241325628452645:2107] held: no
2025-12-24T02:11:21.561565Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241325628452645:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.562185Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325507385066:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241325507385149:2116] held: no
2025-12-24T02:11:21.556406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898355:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.556425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327810898355:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.556482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327810898341:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.556510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.556519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327810898341:2109] [node 4] ICP22 created new session: [2:7587241327810898370:2109]
2025-12-24T02:11:21.556527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327810898370:2109] [node 4] ICS09 handshake done sender: [2:7587241327810898347:2112] self: [2:7587241327810898342:0] peer: [4:7587241325874232735:0] socket: 81 qp: -1
2025-12-24T02:11:21.556534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327810898370:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:21.556543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.559810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898367:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.559837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898368:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.562663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241325628452585:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.562680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.562687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241325628452585:2048] [node 1] ICP22 created new session: [3:7587241325628452679:2048]
2025-12-24T02:11:21.562788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241325628452679:2048] [node 1] ICS09 handshake done sender: [3:7587241325628452645:2107] self: [3:7587241325628452659:0] peer: [1:7587241324680085907:0] socket: 69 qp: -1
2025-12-24T02:11:21.562795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241325628452679:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.562803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.564724Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241328179895507:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241328179895580:2112] held: no
2025-12-24T02:11:21.563444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232743:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.565511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241325628452586:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.565534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.565543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241325628452586:2049] [node 2] ICP22 created new session: [3:7587241325628452681:2049]
2025-12-24T02:11:21.565550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241325628452681:2049] [node 2] ICS09 handshake done sender: [3:7587241325628452663:2114] self: [3:7587241325628452668:0] peer: [2:7587241327810898338:0] socket: 101 qp: -1
2025-12-24T02:11:21.565556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241325628452681:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.565564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.565295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241327810898275:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(32141635ns) {HandleHandshakeStatus:329(32140859ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(278ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(32039609ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32016635ns max# 31947235ns}}}
2025-12-24T02:11:21.565378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241327810898275:2049] [node 3] ICP18 handshake (actor [2:7587241327810898368:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.568564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.568610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895593:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.571147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328179895590:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.571147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328179895592:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.571191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328179895508:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.571206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328179895578:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.571398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232743:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.571449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232732:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.571468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325874232732:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.571526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325874232651:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.572403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325874232732:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.571067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241325628452683:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58720
2025-12-24T02:11:21.572406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241327810898361:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.572421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241327810898361:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.572725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325874232712:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.572742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.572749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325874232712:2109] [node 5] ICP22 created new session: [4:7587241325874232746:2109]
2025-12-24T02:11:21.572756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325874232746:2109] [node 5] ICS09 handshake done sender: [4:7587241325874232732:2117] self: [4:7587241325874232719:0] peer: [5:7587241328179895596:0] socket: 115 qp: -1
2025-12-24T02:11:21.572762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325874232746:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.572770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.572985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452683:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.580679Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325507385065:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241325507385150:2117] held: no
2025-12-24T02:11:21.579512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327810898355:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.579597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327810898346:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.579614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.579621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327810898346:2111] [node 6] ICP22 created new session: [2:7587241327810898374:2111]
2025-12-24T02:11:21.579627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327810898374:2111] [node 6] ICS09 handshake done sender: [2:7587241327810898355:2115] self: [2:7587241327810898354:0] peer: [6:7587241325507385151:0] socket: 110 qp: -1
2025-12-24T02:11:21.579633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327810898374:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:21.579642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.583006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452673:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.583024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241325628452673:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.583055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.595059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325874232748:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9506
2025-12-24T02:11:21.592386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385155:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
2025-12-24T02:11:21.593078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385153:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
2025-12-24T02:11:21.593133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385154:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
2025-12-24T02:11:21.598391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325507385153:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.597980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241327810898361:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.598615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241327810898345:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.598690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.598996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241325628452673:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.598873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241327810898345:2110] [node 5] ICP22 created new session: [2:7587241327810898377:2110]
2025-12-24T02:11:21.598887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241327810898377:2110] [node 5] ICS09 handshake done sender: [2:7587241327810898361:2118] self: [2:7587241327810898352:0] peer: [5:7587241328179895595:0] socket: 120 qp: -1
2025-12-24T02:11:21.598895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241327810898377:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:21.598913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.586735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328179895580:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.587052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328179895507:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.587068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.587079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328179895507:2048] [node 1] ICP22 created new session: [5:7587241328179895597:2048]
2025-12-24T02:11:21.587086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328179895597:2048] [node 1] ICS09 handshake done sender: [5:7587241328179895580:2112] self: [5:7587241328179895584:0] peer: [1:7587241324680085918:0] socket: 87 qp: -1
2025-12-24T02:11:21.587095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328179895597:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.587104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.591603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241328179895578:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241328179895590:2117] held: no
2025-12-24T02:11:21.599422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895600:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.599605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328179895592:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.599645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328179895508:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.599657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.599662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328179895508:2049] [node 2] ICP22 created new session: [5:7587241328179895601:2049]
2025-12-24T02:11:21.599668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328179895601:2049] [node 2] ICS09 handshake done sender: [5:7587241328179895592:2119] self: [5:7587241328179895595:0] peer: [2:7587241327810898352:0] socket: 102 qp: -1
2025-12-24T02:11:21.599674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328179895601:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.599680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.603275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895599:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.604206Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241325628452586:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(66779159ns) {HandleHandshakeStatus:329(66778581ns) {DropIncomingHandshake:442(761ns) DropOutgoingHandshake:457(2532ns) SwitchToState:179(148ns) ProcessPendingSessionEvents:495(16129426ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 105304ns max# 36968ns}}}
2025-12-24T02:11:21.598835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325874232740:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.598843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325874232740:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.599251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325874232743:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.599324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325874232651:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.599338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.599347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325874232651:2050] [node 3] ICP22 created new session: [4:7587241325874232749:2050]
2025-12-24T02:11:21.599357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325874232749:2050] [node 3] ICS09 handshake done sender: [4:7587241325874232743:2121] self: [4:7587241325874232744:0] peer: [3:7587241325628452661:0] socket: 94 qp: -1
2025-12-24T02:11:21.599363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325874232749:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.599366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241325874232712:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(26545371ns) {HandleHandshakeStatus:329(26544329ns) {DropIncomingHandshake:442(2900ns) DropOutgoingHandshake:457(183ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(16126633ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16106493ns max# 16042625ns}}}
2025-12-24T02:11:21.599371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.604031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325874232740:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.606902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325874232715:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.606925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.606933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325874232715:2110] [node 6] ICP22 created new session: [4:7587241325874232752:2110]
2025-12-24T02:11:21.606940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325874232752:2110] [node 6] ICS09 handshake done sender: [4:7587241325874232740:2120] self: [4:7587241325874232731:0] peer: [6:7587241325507385156:0] socket: 99 qp: -1
2025-12-24T02:11:21.606947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325874232752:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:21.606955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.607839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328179895590:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.598438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325507385140:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.599037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325507385150:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.602738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325507385149:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.603230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325507385065:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.603241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.603246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325507385065:2049] [node 2] ICP22 created new session: [6:7587241325507385157:2049]
2025-12-24T02:11:21.603251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325507385157:2049] [node 2] ICS09 handshake done sender: [6:7587241325507385150:2117] self: [6:7587241325507385151:0] peer: [2:7587241327810898354:0] socket: 113 qp: -1
2025-12-24T02:11:21.603256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325507385157:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.603262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.603551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325507385066:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.603559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.603564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325507385066:2050] [node 3] ICP22 created new session: [6:7587241325507385159:2050]
2025-12-24T02:11:21.603570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325507385159:2050] [node 3] ICS09 handshake done sender: [6:7587241325507385149:2116] self: [6:7587241325507385152:0] peer: [3:7587241325628452655:0] socket: 89 qp: -1
2025-12-24T02:11:21.603574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325507385159:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.603580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.608182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241325628452650:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.608203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.608208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241325628452650:2108] [node 4] ICP22 created new session: [3:7587241325628452686:2108]
2025-12-24T02:11:21.608212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241325628452686:2108] [node 4] ICS09 handshake done sender: [3:7587241325628452673:2120] self: [3:7587241325628452661:0] peer: [4:7587241325874232744:0] socket: 85 qp: -1
2025-12-24T02:11:21.608217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241325628452686:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:21.608223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.608335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328179895578:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.608344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.608348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328179895578:2111] [node 4] ICP22 created new session: [5:7587241328179895603:2111]
2025-12-24T02:11:21.608352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328179895603:2111] [node 4] ICS09 handshake done sender: [5:7587241328179895590:2117] self: [5:7587241328179895596:0] peer: [4:7587241325874232719:0] socket: 116 qp: -1
2025-12-24T02:11:21.608355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328179895603:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:21.608359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.615005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325507385161:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25076
2025-12-24T02:11:21.615709Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325507385153:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:21.615778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325507385140:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.615796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.615804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325507385140:2112] [node 4] ICP22 created new session: [6:7587241325507385162:2112]
2025-12-24T02:11:21.615812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325507385162:2112] [node 4] ICS09 handshake done sender: [6:7587241325507385153:2118] self: [6:7587241325507385156:0] peer: [4:7587241325874232731:0] socket: 124 qp: -1
2025-12-24T02:11:21.615820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325507385162:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:21.615829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.630801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328179895593:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.630861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328179895509:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.634885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241325628452670:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.634902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241325628452670:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.635817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241325628452670:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.635933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241325628452651:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.635962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.635970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241325628452651:2109] [node 5] ICP22 created new session: [3:7587241325628452689:2109]
2025-12-24T02:11:21.635978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241325628452689:2109] [node 5] ICS09 handshake done sender: [3:7587241325628452670:2118] self: [3:7587241325628452660:0] peer: [5:7587241328179895605:0] socket: 83 qp: -1
2025-12-24T02:11:21.635986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241325628452689:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:21.636012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.639600Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241328179895509:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24805) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241328179895593:2120] held: no
2025-12-24T02:11:21.639750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328179895606:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50251
2025-12-24T02:11:21.639936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328179895593:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.640023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328179895509:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.640039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.640044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328179895509:2050] [node 3] ICP22 created new session: [5:7587241328179895607:2050]
2025-12-24T02:11:21.640051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328179895607:2050] [node 3] ICS09 handshake done sender: [5:7587241328179895593:2120] self: [5:7587241328179895605:0] peer: [3:7587241325628452660:0] socket: 91 qp: -1
2025-12-24T02:11:21.640057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328179895607:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:21.640063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.656348Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:21.656448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241325628452679:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-12-24T02:11:21.656459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241325628452585:2048] [node 1] ICP30 unregister session Session# [3:7587241325628452679:2048] VirtualId# [3:7587241325628452659:0]
2025-12-24T02:11:21.656466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241325628452679:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.656507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.656530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.658806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:21.658915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241327810898350:2048] [node 1] ICS07 socket disconnect 64 reason# EndOfStream
2025-12-24T02:11:21.658927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241327810898350:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.658997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.659020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241327810898350:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:21.663019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241327810898274:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.663041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241327810898350:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.663059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241327810898274:2048] [node 1] ICP30 unregister session Session# [2:7587241327810898350:2048] VirtualId# [2:7587241327810898340:0]
2025-12-24T02:11:21.663110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.663119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241327810898274:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.663122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.667806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:21.667906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325874232729:2048] [node 1] ICS01 socket: 80 reason# EndOfStream
2025-12-24T02:11:21.667914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325874232649:2048] [node 1] ICP30 unregister session Session# [4:7587241325874232729:2048] VirtualId# [4:7587241325874232724:0]
2025-12-24T02:11:21.667922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325874232729:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.667983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.668011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.665650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:21.665760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241325507385137:2048] [node 1] ICS07 socket disconnect 67 reason# EndOfStream
2025-12-24T02:11:21.665766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325507385137:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.665836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.665864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241325507385137:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:21.669554Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325507385064:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.669572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325507385137:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.669579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325507385064:2048] [node 1] ICP30 unregister session Session# [6:7587241325507385137:2048] VirtualId# [6:7587241325507385131:0]
2025-12-24T02:11:21.669631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.669640Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241325507385064:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.669643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9944) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.670813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:21.670899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241328179895597:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-12-24T02:11:21.670906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328179895507:2048] [node 1] ICP30 unregister session Session# [5:7587241328179895597:2048] VirtualId# [5:7587241328179895584:0]
2025-12-24T02:11:21.670912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241328179895597:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.670976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:21.671004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:21.681698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:21.681816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241325507385064:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:21.681814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241325628452681:2049] [node 2] ICS07 socket disconnect 101 reason# EndOfStream
2025-12-24T02:11:21.681826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241325628452681:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.681883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.681903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241325628452681:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:21.691361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241325628452586:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24823) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:21.691379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241325628452681:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:21.691763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241325628452586:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24823) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.691773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241325628452681:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.691779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241325628452586:2049] [node 2] ICP30 unregister session Session# [3:7587241325628452681:2049] VirtualId# [3:7587241325628452668:0]
2025-12-24T02:11:21.691824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.691830Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241325628452586:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24823) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.691833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24823) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.695080Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:21.695183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325874232738:2049] [node 2] ICS01 socket: 82 reason# EndOfStream
2025-12-24T02:11:21.695190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325874232650:2049] [node 2] ICP30 unregister session Session# [4:7587241325874232738:2049] VirtualId# [4:7587241325874232735:0]
2025-12-24T02:11:21.695198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325874232738:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.695264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.695291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.695234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:21.695335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325507385157:2049] [node 2] ICS01 socket: 113 reason# EndOfStream
2025-12-24T02:11:21.695344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325507385065:2049] [node 2] ICP30 unregister session Session# [6:7587241325507385157:2049] VirtualId# [6:7587241325507385151:0]
2025-12-24T02:11:21.695350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325507385157:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.695420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.695473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.695574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:21.695654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241328179895601:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-12-24T02:11:21.695659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328179895508:2049] [node 2] ICP30 unregister session Session# [5:7587241328179895601:2049] VirtualId# [5:7587241328179895595:0]
2025-12-24T02:11:21.695665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241328179895601:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.695706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:21.695732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:21.714801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:21.714909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241328179895607:2050] [node 3] ICS01 socket: 91 reason# EndOfStream
2025-12-24T02:11:21.714920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328179895509:2050] [node 3] ICP30 unregister session Session# [5:7587241328179895607:2050] VirtualId# [5:7587241328179895605:0]
2025-12-24T02:11:21.714929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241328179895607:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.715014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.715043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.720289Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:21.720402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325507385159:2050] [node 3] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:11:21.720415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325507385066:2050] [node 3] ICP30 unregister session Session# [6:7587241325507385159:2050] VirtualId# [6:7587241325507385152:0]
2025-12-24T02:11:21.720423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325507385159:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.720504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:21.720549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:21.749278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:21.749395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241325507385162:2112] [node 4] ICS07 socket disconnect 124 reason# EndOfStream
2025-12-24T02:11:21.749402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325507385162:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.749469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:21.749496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241325507385162:2112] [node 4] ICS15 start handshake
2025-12-24T02:11:21.749974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325507385140:2112] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8741) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.749987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325507385162:2112] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.749994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325507385140:2112] [node 4] ICP30 unregister session Session# [6:7587241325507385162:2112] VirtualId# [6:7587241325507385156:0]
2025-12-24T02:11:21.750038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:21.750045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241325507385140:2112] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8741) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.750048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8741) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.760247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241325507385140:2112] [node 4] ICP33 wake up from error state
2025-12-24T02:11:21.779591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:21.779698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241325507385133:2108] [node 5] ICS07 socket disconnect 60 reason# EndOfStream
2025-12-24T02:11:21.779706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325507385133:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:21.779755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:21.779785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241325507385133:2108] [node 5] ICS15 start handshake
2025-12-24T02:11:21.780333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325507385126:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:21.780349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325507385133:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:21.780355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325507385126:2108] [node 5] ICP30 unregister session Session# [6:7587241325507385133:2108] VirtualId# [6:7587241325507385127:0]
2025-12-24T02:11:21.780401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:21.780409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241325507385126:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:21.780412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:21.820390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241325097207192:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.820408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241325097207193:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.820413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241325097207194:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.821475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325851970150:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.821484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325851970151:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.821489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325851970152:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.809490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325520036100:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:21.809499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325520036101:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.826833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241324657022292:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.826859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241324657022293:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:21.826836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327744312188:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.826848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327744312189:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.823340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325721922156:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:21.823350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325721922157:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:21.823356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325721922158:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:21.851138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325851970212:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:21.855066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241325097207254:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.856128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325721922217:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
2025-12-24T02:11:21.854953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241324657022353:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:21.855841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241324657022358:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.859106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207257:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.858926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312247:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:21.859696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325721922218:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
2025-12-24T02:11:21.859782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312247:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.860196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241327744312189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.859512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241325097207257:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.860877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922217:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.860393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022355:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.860406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241324657022355:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.861436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241324657022355:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.861603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312249:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:21.861649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325721922157:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.861793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327744312247:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.861860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241327744312189:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.861875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.861882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327744312189:2049] [node 2] ICP22 created new session: [3:7587241327744312250:2049]
2025-12-24T02:11:21.861889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241327744312250:2049] [node 2] ICS09 handshake done sender: [3:7587241327744312247:2106] self: [3:7587241327744312248:0] peer: [2:7587241324657022354:0] socket: 74 qp: -1
2025-12-24T02:11:21.861895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241327744312250:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.861915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.865077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325520036168:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:21.865244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325520036169:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.874916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312253:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:21.881352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241325520036175:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:21.879050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241325097207260:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.885222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325721922220:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
2025-12-24T02:11:21.883205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022357:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.883221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241324657022357:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.883645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241324657022293:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.883659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.883665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241324657022293:2049] [node 3] ICP22 created new session: [2:7587241324657022363:2049]
2025-12-24T02:11:21.883672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241324657022363:2049] [node 3] ICS09 handshake done sender: [2:7587241324657022355:2108] self: [2:7587241324657022354:0] peer: [3:7587241327744312248:0] socket: 70 qp: -1
2025-12-24T02:11:21.883681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241324657022363:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.883690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.884040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241324657022365:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:21.885019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241324657022372:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:21.885876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241324657022357:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.899451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241325097207260:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.899574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207261:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:21.900304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327744312260:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:21.900442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327744312261:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:21.900571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241327744312262:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:21.908519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241324657022353:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.908554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:21.908561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241324657022353:2107] [node 4] ICP22 created new session: [2:7587241324657022377:2107]
2025-12-24T02:11:21.908567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241324657022377:2107] [node 4] ICS09 handshake done sender: [2:7587241324657022357:2109] self: [2:7587241324657022356:0] peer: [4:7587241325721922219:0] socket: 71 qp: -1
2025-12-24T02:11:21.908574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241324657022377:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:21.908582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:21.910734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325851970214:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.910765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241325851970214:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.911082Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241325520036175:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29591199ns) {Configure:104(29590248ns) {ProcessConfigured:135(29195397ns) {StartInitialHandshake:157(29194534ns) {DropHandshakes:470(995ns) {DropIncomingHandshake:442(136ns) DropOutgoingHandshake:457(144ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1455ns) {SwitchToState:179(727ns)}}}}}
2025-12-24T02:11:21.923641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325721922217:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.924258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312253:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.924327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241327744312188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.927923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325851970214:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.931813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207263:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:21.931852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207264:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:21.932667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241325097207257:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.932750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241325097207260:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.932767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:21.932773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241325097207260:2111] [node 6] ICP22 created new session: [5:7587241325097207271:2111]
2025-12-24T02:11:21.932779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241325097207271:2111] [node 6] ICS09 handshake done sender: [5:7587241325097207257:2110] self: [5:7587241325097207262:0] peer: [6:7587241325851970213:0] socket: 75 qp: -1
2025-12-24T02:11:21.932787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241325097207271:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:21.932796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:21.934948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325721922224:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:21.935115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241325721922225:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.938971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241325851970212:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.938996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.939006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325851970212:2108] [node 5] ICP22 created new session: [6:7587241325851970217:2108]
2025-12-24T02:11:21.939014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325851970217:2108] [node 5] ICS09 handshake done sender: [6:7587241325851970214:2109] self: [6:7587241325851970213:0] peer: [5:7587241325097207262:0] socket: 66 qp: -1
2025-12-24T02:11:21.939025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325851970217:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:21.939035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.939438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325851970219:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:21.943159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241325097207261:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.943203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241325097207193:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.944013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036164:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.944029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325520036164:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.944437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325520036182:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34125
2025-12-24T02:11:21.944569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325520036179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34125
2025-12-24T02:11:21.944606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325520036180:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34125
2025-12-24T02:11:21.944639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325520036181:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34125
2025-12-24T02:11:21.950915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325721922157:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.950938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.950948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325721922157:2049] [node 2] ICP22 created new session: [4:7587241325721922228:2049]
2025-12-24T02:11:21.950956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241325721922228:2049] [node 2] ICS09 handshake done sender: [4:7587241325721922217:2108] self: [4:7587241325721922219:0] peer: [2:7587241324657022356:0] socket: 72 qp: -1
2025-12-24T02:11:21.950964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241325721922228:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.950986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.954969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241325851970230:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.959062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022370:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.959076Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241324657022370:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.959679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022358:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.961154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241324657022380:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-12-24T02:11:21.961193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241324657022381:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-12-24T02:11:21.961235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241324657022292:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.951087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325520036164:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:21.963110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241324657022370:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:21.962993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036181:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.963032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036180:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.963425Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036161:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.963432Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325520036161:2107] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.967206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207276:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.974995Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241325721922157:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23960029ns) {HandleHandshakeStatus:329(23958903ns) {DropIncomingHandshake:442(221ns) DropOutgoingHandshake:457(211ns) SwitchToState:179(64ns) ProcessPendingSessionEvents:495(16046277ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16024316ns max# 15945891ns}}}
2025-12-24T02:11:21.975076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922218:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.975773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325721922223:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
2025-12-24T02:11:21.976610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241325721922234:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.986864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241324657022365:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.986888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:21.986898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241324657022365:2111] [node 5] ICP22 created new session: [2:7587241324657022384:2111]
2025-12-24T02:11:21.986907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241324657022384:2111] [node 5] ICS09 handshake done sender: [2:7587241324657022370:2113] self: [2:7587241324657022369:0] peer: [5:7587241325097207275:0] socket: 88 qp: -1
2025-12-24T02:11:21.986916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241324657022384:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:21.986928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:21.987200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022380:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.990556Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241325097207261:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.987748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325520036101:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.987768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:21.987776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325520036101:2049] [node 3] ICP22 created new session: [1:7587241325520036184:2049]
2025-12-24T02:11:21.987783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325520036184:2049] [node 3] ICS09 handshake done sender: [1:7587241325520036164:2108] self: [1:7587241325520036163:0] peer: [3:7587241327744312271:0] socket: 82 qp: -1
2025-12-24T02:11:21.987790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325520036184:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:21.987797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:21.989136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036179:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.989404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241325520036187:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34125
2025-12-24T02:11:21.989415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241325520036101:2049] [node 3] ICP18 handshake (actor [1:7587241325520036179:2115]) is too old
2025-12-24T02:11:21.990807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325520036161:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:21.990982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241325097207193:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.992859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036182:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.990999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.991006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241325097207193:2049] [node 2] ICP22 created new session: [5:7587241325097207279:2049]
2025-12-24T02:11:21.991012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241325097207279:2049] [node 2] ICS09 handshake done sender: [5:7587241325097207261:2112] self: [5:7587241325097207275:0] peer: [2:7587241324657022369:0] socket: 90 qp: -1
2025-12-24T02:11:21.992878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325520036100:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:21.991017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241325097207279:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:21.992893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:21.991023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:21.992902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325520036100:2048] [node 2] ICP22 created new session: [1:7587241325520036191:2048]
2025-12-24T02:11:21.992908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241325520036191:2048] [node 2] ICS09 handshake done sender: [1:7587241325520036161:2107] self: [1:7587241325520036160:0] peer: [2:7587241324657022383:0] socket: 68 qp: -1
2025-12-24T02:11:21.992914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241325520036191:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:21.992921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:21.995081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325851970233:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:21.996755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325851970233:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.996784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325851970150:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:21.997449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241325851970235:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:21.997976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325851970233:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:21.998027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325851970150:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:21.998040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:21.998046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325851970150:2048] [node 1] ICP22 created new session: [6:7587241325851970236:2048]
2025-12-24T02:11:21.998052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241325851970236:2048] [node 1] ICS09 handshake done sender: [6:7587241325851970233:2116] self: [6:7587241325851970234:0] peer: [1:7587241325520036176:0] socket: 105 qp: -1
2025-12-24T02:11:21.998058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241325851970236:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:21.998065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:21.996877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036177:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:21.996888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325520036177:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:21.997791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325520036177:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:21.999740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022381:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.000501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922234:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.000528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922223:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.000556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325721922156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.000566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241325721922158:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.000815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241327744312188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241327744312253:2109] held: no
2025-12-24T02:11:22.000954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312264:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.000958Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327744312264:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:22.001861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312272:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:22.001966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241330016889539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.002352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325520036175:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.002371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.002377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325520036175:2113] [node 6] ICP22 created new session: [1:7587241329815003492:2113]
2025-12-24T02:11:22.002383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329815003492:2113] [node 6] ICS09 handshake done sender: [1:7587241325520036177:2114] self: [1:7587241325520036176:0] peer: [6:7587241325851970234:0] socket: 94 qp: -1
2025-12-24T02:11:22.002389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329815003492:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:22.002397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.002392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241330016889540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.002470Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241325097207263:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.002591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:22.002638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241325097207278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:22.002104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241325851970219:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.002229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325851970151:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.002826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241325097207194:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.002775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036171:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.002787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325520036171:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.003319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036187:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.003614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241324657022365:2111] [node 5] ICP18 handshake (actor [2:7587241324657022380:2116]) is too old
2025-12-24T02:11:22.003645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241324657022292:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241324657022358:2110] held: no
2025-12-24T02:11:22.003750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241324657022376:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.003754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241324657022376:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.006244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241330016889541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55612
2025-12-24T02:11:22.006234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312273:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:22.006283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241327744312274:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:22.006392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312266:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.006395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327744312266:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.006741Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327744312253:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.007497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325520036171:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.006776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241325097207277:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.007709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325520036168:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.007727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.007741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325520036168:2109] [node 4] ICP22 created new session: [1:7587241329815003496:2109]
2025-12-24T02:11:22.007748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329815003496:2109] [node 4] ICS09 handshake done sender: [1:7587241325520036171:2111] self: [1:7587241325520036170:0] peer: [4:7587241325721922241:0] socket: 108 qp: -1
2025-12-24T02:11:22.007754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329815003496:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:22.007762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.007868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325721922234:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.007885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241325097207254:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.007923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241325097207278:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.007969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241325097207192:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.008109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922233:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.008127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325721922233:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.008208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327744312264:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.008502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325721922156:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.008517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.008522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325721922156:2048] [node 1] ICP22 created new session: [4:7587241330016889542:2048]
2025-12-24T02:11:22.008529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241330016889542:2048] [node 1] ICS09 handshake done sender: [4:7587241325721922234:2117] self: [4:7587241325721922241:0] peer: [1:7587241325520036170:0] socket: 115 qp: -1
2025-12-24T02:11:22.008534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241330016889542:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.008540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.011080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241324657022386:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51278
2025-12-24T02:11:22.011357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241324657022358:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.011414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241324657022292:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.011428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.011434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241324657022292:2048] [node 1] ICP22 created new session: [2:7587241328951989685:2048]
2025-12-24T02:11:22.011441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241328951989685:2048] [node 1] ICS09 handshake done sender: [2:7587241324657022358:2110] self: [2:7587241324657022383:0] peer: [1:7587241325520036160:0] socket: 69 qp: -1
2025-12-24T02:11:22.011448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241328951989685:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.011454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.012072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241324657022376:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.012049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241325520036173:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.012059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241325520036173:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.012895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329392174581:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:22.012915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329392174582:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:22.013299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241325097207278:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.013394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312274:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.013409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241327744312188:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.013419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.013426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327744312188:2048] [node 1] ICP22 created new session: [3:7587241332039279575:2048]
2025-12-24T02:11:22.013432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241332039279575:2048] [node 1] ICS09 handshake done sender: [3:7587241327744312253:2109] self: [3:7587241327744312271:0] peer: [1:7587241325520036163:0] socket: 83 qp: -1
2025-12-24T02:11:22.013438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241332039279575:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.013444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.013636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327744312260:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.013641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.013646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327744312260:2111] [node 4] ICP22 created new session: [3:7587241332039279577:2111]
2025-12-24T02:11:22.013650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241332039279577:2111] [node 4] ICS09 handshake done sender: [3:7587241327744312264:2114] self: [3:7587241327744312263:0] peer: [4:7587241325721922242:0] socket: 91 qp: -1
2025-12-24T02:11:22.013656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241332039279577:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:22.013661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.013104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241325520036173:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.013185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241325520036169:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.013196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.013201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241325520036169:2110] [node 5] ICP22 created new session: [1:7587241329815003499:2110]
2025-12-24T02:11:22.013206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329815003499:2110] [node 5] ICS09 handshake done sender: [1:7587241325520036173:2112] self: [1:7587241325520036172:0] peer: [5:7587241329392174580:0] socket: 124 qp: -1
2025-12-24T02:11:22.013211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329815003499:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:22.013217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.011969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241330146937536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:22.014966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241324657022372:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.014983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.014989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241324657022372:2114] [node 6] ICP22 created new session: [2:7587241328951989687:2114]
2025-12-24T02:11:22.014996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241328951989687:2114] [node 6] ICS09 handshake done sender: [2:7587241324657022376:2115] self: [2:7587241324657022375:0] peer: [6:7587241330146937535:0] socket: 98 qp: -1
2025-12-24T02:11:22.015001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241328951989687:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:22.015007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.014871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241325851970219:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.018843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325721922223:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.019510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312273:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.019674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327744312266:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.019747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241325097207192:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.019765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.019772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241325097207192:2048] [node 1] ICP22 created new session: [5:7587241329392174583:2048]
2025-12-24T02:11:22.019779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329392174583:2048] [node 1] ICS09 handshake done sender: [5:7587241325097207278:2120] self: [5:7587241329392174580:0] peer: [1:7587241325520036172:0] socket: 126 qp: -1
2025-12-24T02:11:22.019786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329392174583:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.019795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.020650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241325721922158:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.020668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.020673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325721922158:2050] [node 3] ICP22 created new session: [4:7587241330016889544:2050]
2025-12-24T02:11:22.020678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241330016889544:2050] [node 3] ICS09 handshake done sender: [4:7587241325721922223:2113] self: [4:7587241325721922242:0] peer: [3:7587241327744312263:0] socket: 93 qp: -1
2025-12-24T02:11:22.020684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241330016889544:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.020692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.021721Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241325097207192:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:22.022042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241325097207263:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.022159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325851970151:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.022178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.022184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325851970151:2049] [node 2] ICP22 created new session: [6:7587241330146937537:2049]
2025-12-24T02:11:22.022191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241330146937537:2049] [node 2] ICS09 handshake done sender: [6:7587241325851970219:2110] self: [6:7587241330146937535:0] peer: [2:7587241324657022375:0] socket: 102 qp: -1
2025-12-24T02:11:22.022197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241330146937537:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.022204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.023445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241330016889540:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.023859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325721922233:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.023933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325721922224:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.023950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.023956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325721922224:2111] [node 5] ICP22 created new session: [4:7587241330016889549:2111]
2025-12-24T02:11:22.023965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241330016889549:2111] [node 5] ICS09 handshake done sender: [4:7587241325721922233:2116] self: [4:7587241325721922232:0] peer: [5:7587241329392174579:0] socket: 117 qp: -1
2025-12-24T02:11:22.023970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241330016889549:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:22.023976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.028394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241325097207194:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.028411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.028415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241325097207194:2050] [node 3] ICP22 created new session: [5:7587241329392174585:2050]
2025-12-24T02:11:22.028420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329392174585:2050] [node 3] ICS09 handshake done sender: [5:7587241325097207263:2113] self: [5:7587241329392174578:0] peer: [3:7587241327744312265:0] socket: 95 qp: -1
2025-12-24T02:11:22.028424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329392174585:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.028429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.027799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327744312261:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.027816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.027821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327744312261:2112] [node 5] ICP22 created new session: [3:7587241332039279579:2112]
2025-12-24T02:11:22.027827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241332039279579:2112] [node 5] ICS09 handshake done sender: [3:7587241327744312266:2115] self: [3:7587241327744312265:0] peer: [5:7587241329392174578:0] socket: 92 qp: -1
2025-12-24T02:11:22.027833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241332039279579:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:22.027838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.028115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241327744312260:2111] [node 4] ICP18 handshake (actor [3:7587241327744312274:2119]) is too old
2025-12-24T02:11:22.028187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241332039279573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28798
2025-12-24T02:11:22.032260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241327744312268:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.032269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241327744312268:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.032984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241327744312268:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.033064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241327744312262:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.033082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.033089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241327744312262:2113] [node 6] ICP22 created new session: [3:7587241332039279583:2113]
2025-12-24T02:11:22.033094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241332039279583:2113] [node 6] ICS09 handshake done sender: [3:7587241327744312268:2116] self: [3:7587241327744312267:0] peer: [6:7587241330146937541:0] socket: 106 qp: -1
2025-12-24T02:11:22.033101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241332039279583:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:22.033109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.031181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241330146937539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:22.031206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241330146937540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:22.032131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241330146937539:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.032161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325851970152:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.033593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241330146937542:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:22.033723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241330146937539:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.034588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325851970152:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.034602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.034608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325851970152:2050] [node 3] ICP22 created new session: [6:7587241330146937543:2050]
2025-12-24T02:11:22.034614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241330146937543:2050] [node 3] ICS09 handshake done sender: [6:7587241330146937539:2119] self: [6:7587241330146937541:0] peer: [3:7587241327744312267:0] socket: 118 qp: -1
2025-12-24T02:11:22.034619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241330146937543:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.034625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.034779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241325851970152:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:22.035408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241325097207254:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241325097207277:2119] held: no
2025-12-24T02:11:22.035548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241325721922236:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.035562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241325721922236:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.035622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329392174587:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46408
2025-12-24T02:11:22.036225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241325721922236:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.036289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241325721922225:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.036305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.036315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241325721922225:2112] [node 6] ICP22 created new session: [4:7587241330016889552:2112]
2025-12-24T02:11:22.036321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241330016889552:2112] [node 6] ICS09 handshake done sender: [4:7587241325721922236:2118] self: [4:7587241325721922235:0] peer: [6:7587241330146937545:0] socket: 99 qp: -1
2025-12-24T02:11:22.036326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241330016889552:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:22.036332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.035349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241330146937540:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.035403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241325851970230:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.037409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241325851970230:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241330146937540:2120] held: no
2025-12-24T02:11:22.037558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241330146937546:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:626
2025-12-24T02:11:22.037682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241330146937540:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.042082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241327744312261:2112] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:22.042306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241325097207277:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.042298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241325851970230:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.042318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.042327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241325851970230:2114] [node 4] ICP22 created new session: [6:7587241330146937547:2114]
2025-12-24T02:11:22.042333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241330146937547:2114] [node 4] ICS09 handshake done sender: [6:7587241330146937540:2120] self: [6:7587241330146937545:0] peer: [4:7587241325721922235:0] socket: 78 qp: -1
2025-12-24T02:11:22.042341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241330146937547:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:22.042351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.042421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241325097207254:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.042434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.042441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241325097207254:2108] [node 4] ICP22 created new session: [5:7587241329392174588:2108]
2025-12-24T02:11:22.042447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329392174588:2108] [node 4] ICS09 handshake done sender: [5:7587241325097207277:2119] self: [5:7587241329392174579:0] peer: [4:7587241325721922232:0] socket: 125 qp: -1
2025-12-24T02:11:22.042453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329392174588:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:22.042462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.059096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:22.059213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241328951989685:2048] [node 1] ICS01 socket: 69 reason# EndOfStream
2025-12-24T02:11:22.059222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241324657022292:2048] [node 1] ICP30 unregister session Session# [2:7587241328951989685:2048] VirtualId# [2:7587241324657022383:0]
2025-12-24T02:11:22.059230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241328951989685:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.059274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.059293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.063085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:22.063199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241332039279575:2048] [node 1] ICS01 socket: 83 reason# EndOfStream
2025-12-24T02:11:22.063208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241327744312188:2048] [node 1] ICP30 unregister session Session# [3:7587241332039279575:2048] VirtualId# [3:7587241327744312271:0]
2025-12-24T02:11:22.063215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241332039279575:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.063283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.063304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.075123Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:22.075229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241329392174583:2048] [node 1] ICS01 socket: 126 reason# EndOfStream
2025-12-24T02:11:22.075239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241325097207192:2048] [node 1] ICP30 unregister session Session# [5:7587241329392174583:2048] VirtualId# [5:7587241329392174580:0]
2025-12-24T02:11:22.075246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241329392174583:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.075314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.075339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.077929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:22.078027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241330016889542:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-12-24T02:11:22.078036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325721922156:2048] [node 1] ICP30 unregister session Session# [4:7587241330016889542:2048] VirtualId# [4:7587241325721922241:0]
2025-12-24T02:11:22.078042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241330016889542:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.078102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.078133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.078174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:22.078246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241325851970236:2048] [node 1] ICS01 socket: 105 reason# EndOfStream
2025-12-24T02:11:22.078251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325851970150:2048] [node 1] ICP30 unregister session Session# [6:7587241325851970236:2048] VirtualId# [6:7587241325851970234:0]
2025-12-24T02:11:22.078256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241325851970236:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.078286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.078309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.079295Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:22.079371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241327744312250:2049] [node 2] ICS01 socket: 74 reason# EndOfStream
2025-12-24T02:11:22.079378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241327744312189:2049] [node 2] ICP30 unregister session Session# [3:7587241327744312250:2049] VirtualId# [3:7587241327744312248:0]
2025-12-24T02:11:22.079382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241327744312250:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.079413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.079430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.088735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:22.088836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241325097207279:2049] [node 2] ICS01 socket: 90 reason# EndOfStream
2025-12-24T02:11:22.088845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241325097207193:2049] [node 2] ICP30 unregister session Session# [5:7587241325097207279:2049] VirtualId# [5:7587241325097207275:0]
2025-12-24T02:11:22.088850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241325097207279:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.088909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.088935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.090873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:22.090967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241330146937537:2049] [node 2] ICS01 socket: 102 reason# EndOfStream
2025-12-24T02:11:22.090975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325851970151:2049] [node 2] ICP30 unregister session Session# [6:7587241330146937537:2049] VirtualId# [6:7587241330146937535:0]
2025-12-24T02:11:22.090981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241330146937537:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.091035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.091060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.091147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:22.091214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241325721922228:2049] [node 2] ICS07 socket disconnect 72 reason# EndOfStream
2025-12-24T02:11:22.091218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241325721922228:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.091248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.091271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241325721922228:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:22.091793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241325721922157:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.091805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241325721922228:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.091811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241325721922157:2049] [node 2] ICP30 unregister session Session# [4:7587241325721922228:2049] VirtualId# [4:7587241325721922219:0]
2025-12-24T02:11:22.091848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.091855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241325721922157:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.091857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20168) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.105338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:22.105438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241329392174585:2050] [node 3] ICS01 socket: 95 reason# EndOfStream
2025-12-24T02:11:22.105450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241325097207194:2050] [node 3] ICP30 unregister session Session# [5:7587241329392174585:2050] VirtualId# [5:7587241329392174578:0]
2025-12-24T02:11:22.105457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241329392174585:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.105518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.105546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.105590Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:22.105659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241330146937543:2050] [node 3] ICS01 socket: 118 reason# EndOfStream
2025-12-24T02:11:22.105665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325851970152:2050] [node 3] ICP30 unregister session Session# [6:7587241330146937543:2050] VirtualId# [6:7587241330146937541:0]
2025-12-24T02:11:22.105670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241330146937543:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.105711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.105735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.118201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:22.118299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241330146937547:2114] [node 4] ICS01 socket: 78 reason# EndOfStream
2025-12-24T02:11:22.118309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241325851970230:2114] [node 4] ICP30 unregister session Session# [6:7587241330146937547:2114] VirtualId# [6:7587241330146937545:0]
2025-12-24T02:11:22.118316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241330146937547:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.118378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:22.118405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:22.141287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241330321919635:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:22.141300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241330321919636:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.147890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241330534165715:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.147898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241330534165716:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.146881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241332220247020:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.146896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241332220247021:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.162784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329830461903:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.162806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329830461904:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.162815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329830461905:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.160906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329570005616:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.160915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329570005617:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.160921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329570005618:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.158435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241330526840634:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.158443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241330526840635:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.158448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241330526840636:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.230506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840695:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.229716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329570005672:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:22.230222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005673:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.229664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241332220247085:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.231667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329570005679:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.230916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241332220247092:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:22.231500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461964:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.231376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241332220247093:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28182
2025-12-24T02:11:22.234682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241332220247097:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:22.232694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461965:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.228579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241330321919698:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:22.230888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241330321919706:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15128
2025-12-24T02:11:22.235579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241330321919709:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:22.235690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241330321919710:2113] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.238673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165778:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.235479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461965:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.235681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329830461967:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:22.236220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329830461967:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.239082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329570005681:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.239096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241329570005681:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.239502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241330526840695:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.239654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241330526840697:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:22.240090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241330526840697:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.238757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165779:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
2025-12-24T02:11:22.238942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461964:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.239287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329830461904:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.242996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919706:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.242879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165779:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.243249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241330534165716:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.243296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329570005675:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.243310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241329570005675:2104] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.243820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241330321919716:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15128
2025-12-24T02:11:22.244874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165778:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.244955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241330534165715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.244383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329570005673:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.244873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329570005616:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.246461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840699:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.246359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461970:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.243396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241332220247087:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.243413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241332220247087:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.245594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241332220247093:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.245627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241332220247084:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.245630Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241332220247084:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.246184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241332220247020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.246804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329570005675:2104] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.247044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241330526840695:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.247127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919716:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.247220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241329570005672:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.247232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.247238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329570005672:2101] [node 5] ICP22 created new session: [6:7587241329570005690:2101]
2025-12-24T02:11:22.247245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329570005690:2101] [node 5] ICS09 handshake done sender: [6:7587241329570005675:2104] self: [6:7587241329570005674:0] peer: [5:7587241330526840698:0] socket: 77 qp: -1
2025-12-24T02:11:22.247252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329570005690:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:22.247262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.252184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241330526840697:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.252207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.252215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241330526840697:2109] [node 6] ICP22 created new session: [5:7587241330526840700:2109]
2025-12-24T02:11:22.252223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241330526840700:2109] [node 6] ICS09 handshake done sender: [5:7587241330526840695:2108] self: [5:7587241330526840698:0] peer: [6:7587241329570005674:0] socket: 80 qp: -1
2025-12-24T02:11:22.252231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241330526840700:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:22.252254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.250794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919702:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.250817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241330321919702:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.251296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919700:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.251301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241330321919700:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.252335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919697:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.252348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241330321919697:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.252488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241330321919702:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.254889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241332220247087:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.255180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241332220247084:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.255264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241332220247085:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.255283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.255284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241332220247021:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.255289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241332220247085:2107] [node 4] ICP22 created new session: [2:7587241332220247104:2107]
2025-12-24T02:11:22.255296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241332220247104:2107] [node 4] ICS09 handshake done sender: [2:7587241332220247087:2110] self: [2:7587241332220247086:0] peer: [4:7587241329830461969:0] socket: 79 qp: -1
2025-12-24T02:11:22.255296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.255302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241332220247021:2049] [node 3] ICP22 created new session: [2:7587241332220247105:2049]
2025-12-24T02:11:22.255304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241332220247104:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:22.255307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241332220247105:2049] [node 3] ICS09 handshake done sender: [2:7587241332220247084:2109] self: [2:7587241332220247083:0] peer: [3:7587241330534165780:0] socket: 78 qp: -1
2025-12-24T02:11:22.255312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.255162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005692:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.255314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241332220247105:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:22.255322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.255588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241332220247103:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28182
2025-12-24T02:11:22.259951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241330321919700:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.260014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241330321919698:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.260026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.260035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241330321919698:2107] [node 6] ICP22 created new session: [1:7587241330321919720:2107]
2025-12-24T02:11:22.260043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241330321919720:2107] [node 6] ICS09 handshake done sender: [1:7587241330321919702:2110] self: [1:7587241330321919701:0] peer: [6:7587241329570005688:0] socket: 74 qp: -1
2025-12-24T02:11:22.260050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241330321919720:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:22.260072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.262954Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329570005681:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.262949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241330321919635:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.262975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.262983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241330321919635:2048] [node 2] ICP22 created new session: [1:7587241330321919724:2048]
2025-12-24T02:11:22.262990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241330321919724:2048] [node 2] ICS09 handshake done sender: [1:7587241330321919700:2109] self: [1:7587241330321919699:0] peer: [2:7587241332220247102:0] socket: 73 qp: -1
2025-12-24T02:11:22.262996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241330321919724:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:22.263004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.263381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241330321919697:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.267478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241329570005679:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.267499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.267509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329570005679:2106] [node 4] ICP22 created new session: [6:7587241329570005695:2106]
2025-12-24T02:11:22.267517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329570005695:2106] [node 4] ICS09 handshake done sender: [6:7587241329570005681:2107] self: [6:7587241329570005680:0] peer: [4:7587241329830461968:0] socket: 88 qp: -1
2025-12-24T02:11:22.267523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329570005695:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:22.267534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.267722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329570005673:2103] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.268304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329570005616:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.268317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.268324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329570005616:2048] [node 1] ICP22 created new session: [6:7587241329570005697:2048]
2025-12-24T02:11:22.268328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329570005697:2048] [node 1] ICS09 handshake done sender: [6:7587241329570005673:2103] self: [6:7587241329570005688:0] peer: [1:7587241330321919701:0] socket: 75 qp: -1
2025-12-24T02:11:22.268334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329570005697:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.268340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.268543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005694:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.267287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329830461964:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.267852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329830461904:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.267862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.267869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329830461904:2049] [node 2] ICP22 created new session: [4:7587241329830461971:2049]
2025-12-24T02:11:22.267875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329830461971:2049] [node 2] ICS09 handshake done sender: [4:7587241329830461964:2108] self: [4:7587241329830461969:0] peer: [2:7587241332220247086:0] socket: 85 qp: -1
2025-12-24T02:11:22.267880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329830461971:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.267912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.271533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241332220247093:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.271605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241332220247020:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.271618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.271627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241332220247020:2048] [node 1] ICP22 created new session: [2:7587241332220247110:2048]
2025-12-24T02:11:22.271634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241332220247110:2048] [node 1] ICS09 handshake done sender: [2:7587241332220247093:2112] self: [2:7587241332220247102:0] peer: [1:7587241330321919699:0] socket: 84 qp: -1
2025-12-24T02:11:22.271641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241332220247110:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.271649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.275057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461973:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.275003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329570005694:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.275320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329570005617:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.274888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165782:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
2025-12-24T02:11:22.278911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241332220247099:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.278925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241332220247099:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.279851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241332220247099:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.279942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241332220247097:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.279957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.279984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241332220247097:2114] [node 6] ICP22 created new session: [2:7587241332220247113:2114]
2025-12-24T02:11:22.279992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241332220247113:2114] [node 6] ICS09 handshake done sender: [2:7587241332220247099:2115] self: [2:7587241332220247098:0] peer: [6:7587241329570005699:0] socket: 104 qp: -1
2025-12-24T02:11:22.279999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241332220247113:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:22.280007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.282919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.283127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329570005694:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.283133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241330534165785:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:22.283198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241330534165779:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.283197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329570005617:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.283213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.283219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329570005617:2049] [node 2] ICP22 created new session: [6:7587241329570005701:2049]
2025-12-24T02:11:22.283227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329570005701:2049] [node 2] ICS09 handshake done sender: [6:7587241329570005694:2114] self: [6:7587241329570005699:0] peer: [2:7587241332220247098:0] socket: 111 qp: -1
2025-12-24T02:11:22.283234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329570005701:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.283244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.283369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241330321919636:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.283394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.283401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241330321919636:2049] [node 3] ICP22 created new session: [1:7587241330321919726:2049]
2025-12-24T02:11:22.283409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241330321919726:2049] [node 3] ICS09 handshake done sender: [1:7587241330321919697:2108] self: [1:7587241330321919696:0] peer: [3:7587241330534165781:0] socket: 72 qp: -1
2025-12-24T02:11:22.283416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241330321919726:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:22.283423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.283784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461974:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.283819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329830461965:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.283644Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165783:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.287065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840702:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.295737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329830461967:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.295759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.295767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329830461967:2110] [node 6] ICP22 created new session: [4:7587241329830461975:2110]
2025-12-24T02:11:22.295774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329830461975:2110] [node 6] ICS09 handshake done sender: [4:7587241329830461965:2109] self: [4:7587241329830461968:0] peer: [6:7587241329570005680:0] socket: 89 qp: -1
2025-12-24T02:11:22.295782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329830461975:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:22.295791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.297303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241330534165790:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:22.297452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241330534165791:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:22.297508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241330534165716:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.297517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.297521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241330534165716:2049] [node 2] ICP22 created new session: [3:7587241330534165792:2049]
2025-12-24T02:11:22.297527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241330534165792:2049] [node 2] ICS09 handshake done sender: [3:7587241330534165779:2109] self: [3:7587241330534165780:0] peer: [2:7587241332220247083:0] socket: 92 qp: -1
2025-12-24T02:11:22.297536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241330534165792:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.297543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.297881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241330534165778:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.299041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241330321919729:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15128
2025-12-24T02:11:22.298812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241330526840702:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.299178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241330526840635:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.306981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241330526840706:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:22.307041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840705:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.311005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329830461981:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:22.311036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241332220247095:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.311059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241332220247095:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.315284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241330321919730:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15128
2025-12-24T02:11:22.319062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241330526840705:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.319108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241330526840634:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.319253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919714:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.319264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241330321919714:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.320272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840717:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.320390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241330321919714:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.320612Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241330526840705:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.320701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241330526840634:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.320719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.320726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241330526840634:2048] [node 1] ICP22 created new session: [5:7587241330526840718:2048]
2025-12-24T02:11:22.320733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241330526840718:2048] [node 1] ICS09 handshake done sender: [5:7587241330526840705:2113] self: [5:7587241330526840716:0] peer: [1:7587241330321919711:0] socket: 117 qp: -1
2025-12-24T02:11:22.320741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241330526840718:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.320749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.321429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241330321919709:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.321445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.321472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241330321919709:2112] [node 5] ICP22 created new session: [1:7587241330321919732:2112]
2025-12-24T02:11:22.321480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241330321919732:2112] [node 5] ICS09 handshake done sender: [1:7587241330321919714:2115] self: [1:7587241330321919711:0] peer: [5:7587241330526840716:0] socket: 115 qp: -1
2025-12-24T02:11:22.321487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241330321919732:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:22.321493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.323264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241330534165715:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.323287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.323296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241330534165715:2048] [node 1] ICP22 created new session: [3:7587241330534165798:2048]
2025-12-24T02:11:22.323303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241330534165798:2048] [node 1] ICS09 handshake done sender: [3:7587241330534165778:2108] self: [3:7587241330534165781:0] peer: [1:7587241330321919696:0] socket: 87 qp: -1
2025-12-24T02:11:22.323312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241330534165798:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.323320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.323476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241330534165715:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.329967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461982:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.330083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461983:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.330989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461974:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.333146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241332220247095:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.333256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241332220247092:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.333270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.333279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241332220247092:2111] [node 5] ICP22 created new session: [2:7587241332220247116:2111]
2025-12-24T02:11:22.333287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241332220247116:2111] [node 5] ICS09 handshake done sender: [2:7587241332220247095:2113] self: [2:7587241332220247094:0] peer: [5:7587241330526840703:0] socket: 105 qp: -1
2025-12-24T02:11:22.333294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241332220247116:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:22.333303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.344139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461983:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.344556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165789:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.344566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241330534165789:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.344411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329830461903:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.344458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329830461905:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.345197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241330534165789:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.345269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241330534165785:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.345282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.345288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241330534165785:2112] [node 4] ICP22 created new session: [3:7587241330534165804:2112]
2025-12-24T02:11:22.345294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241330534165804:2112] [node 4] ICS09 handshake done sender: [3:7587241330534165789:2115] self: [3:7587241330534165788:0] peer: [4:7587241329830461991:0] socket: 90 qp: -1
2025-12-24T02:11:22.345300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241330534165804:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:22.345309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.345926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840720:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.345970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840721:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.345997Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840722:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.346245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241330526840720:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.346269Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241330526840702:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.346330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241330526840635:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.346343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.346347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241330526840635:2049] [node 2] ICP22 created new session: [5:7587241330526840723:2049]
2025-12-24T02:11:22.346351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241330526840723:2049] [node 2] ICS09 handshake done sender: [5:7587241330526840702:2111] self: [5:7587241330526840703:0] peer: [2:7587241332220247094:0] socket: 106 qp: -1
2025-12-24T02:11:22.346356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241330526840723:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.346361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.346516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241330526840636:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.346639Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165795:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.346646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241330534165795:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.347264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.346874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919729:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.347642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241330526840721:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.347678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241330526840706:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.348409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840727:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.348636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241330526840721:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.348679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241330526840706:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.348690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.348696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241330526840706:2112] [node 4] ICP22 created new session: [5:7587241330526840728:2112]
2025-12-24T02:11:22.348702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241330526840728:2112] [node 4] ICS09 handshake done sender: [5:7587241330526840721:2119] self: [5:7587241330526840726:0] peer: [4:7587241329830461984:0] socket: 133 qp: -1
2025-12-24T02:11:22.348708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241330526840728:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:22.348714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.347795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461985:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.347803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241329830461985:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.348486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329830461985:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.349503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329830461982:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.349513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241329830461981:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.349521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.349528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329830461981:2115] [node 5] ICP22 created new session: [4:7587241329830461994:2115]
2025-12-24T02:11:22.349532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329830461994:2115] [node 5] ICS09 handshake done sender: [4:7587241329830461985:2118] self: [4:7587241329830461984:0] peer: [5:7587241330526840726:0] socket: 130 qp: -1
2025-12-24T02:11:22.349538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329830461994:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:22.349545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.349821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241329830461981:2115] [node 5] ICP18 handshake (actor [4:7587241329830461982:2116]) is too old
2025-12-24T02:11:22.354727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241330321919713:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.354757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241330321919713:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.354880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329830461983:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.355163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329830461905:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.355178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.355187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329830461905:2050] [node 3] ICP22 created new session: [4:7587241329830461996:2050]
2025-12-24T02:11:22.355193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329830461996:2050] [node 3] ICS09 handshake done sender: [4:7587241329830461983:2117] self: [4:7587241329830461991:0] peer: [3:7587241330534165788:0] socket: 129 qp: -1
2025-12-24T02:11:22.355201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329830461996:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.355208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.363545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165806:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
2025-12-24T02:11:22.364095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241330534165795:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.366941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241332220247092:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(33579427ns) {HandleHandshakeStatus:329(33578349ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(264ns) SwitchToState:179(495ns) ProcessPendingSessionEvents:495(33492039ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 33473433ns max# 33406962ns}}}
2025-12-24T02:11:22.367498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005703:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.371645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241330321919713:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.371813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241330321919710:2113] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.371827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.371833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241330321919710:2113] [node 4] ICP22 created new session: [1:7587241330321919736:2113]
2025-12-24T02:11:22.371838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241330321919736:2113] [node 4] ICS09 handshake done sender: [1:7587241330321919713:2114] self: [1:7587241330321919712:0] peer: [4:7587241329830461990:0] socket: 110 qp: -1
2025-12-24T02:11:22.371843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241330321919736:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:22.371849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.371688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241330534165790:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.371706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.371713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241330534165790:2113] [node 5] ICP22 created new session: [3:7587241330534165809:2113]
2025-12-24T02:11:22.371719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241330534165809:2113] [node 5] ICS09 handshake done sender: [3:7587241330534165795:2116] self: [3:7587241330534165794:0] peer: [5:7587241330526840725:0] socket: 118 qp: -1
2025-12-24T02:11:22.371726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241330534165809:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:22.371733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.375025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241330534165808:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23824
2025-12-24T02:11:22.375747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329570005703:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.376170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329570005618:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.378981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241330534165785:2112] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:22.379042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165806:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.379074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241330534165797:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.379080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241330534165797:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.379948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241330534165797:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.379758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241330526840730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11363
2025-12-24T02:11:22.380035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241330534165791:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.380048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.380055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241330534165791:2114] [node 6] ICP22 created new session: [3:7587241330534165812:2114]
2025-12-24T02:11:22.380062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241330534165812:2114] [node 6] ICS09 handshake done sender: [3:7587241330534165797:2117] self: [3:7587241330534165796:0] peer: [6:7587241329570005704:0] socket: 125 qp: -1
2025-12-24T02:11:22.380068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241330534165812:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:22.380076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.380324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241330526840720:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.380472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329570005705:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34838
2025-12-24T02:11:22.380564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241330526840636:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.380579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.380585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241330526840636:2050] [node 3] ICP22 created new session: [5:7587241330526840731:2050]
2025-12-24T02:11:22.380591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241330526840731:2050] [node 3] ICS09 handshake done sender: [5:7587241330526840720:2118] self: [5:7587241330526840725:0] peer: [3:7587241330534165794:0] socket: 127 qp: -1
2025-12-24T02:11:22.380597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241330526840731:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.380605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.381486Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329570005703:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.381553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329570005618:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.381565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.381571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329570005618:2050] [node 3] ICP22 created new session: [6:7587241329570005706:2050]
2025-12-24T02:11:22.381577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329570005706:2050] [node 3] ICS09 handshake done sender: [6:7587241329570005703:2116] self: [6:7587241329570005704:0] peer: [3:7587241330534165796:0] socket: 126 qp: -1
2025-12-24T02:11:22.381583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329570005706:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.381589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.387665Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241329830461903:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241329830461974:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.394867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329830461998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44652
2025-12-24T02:11:22.395267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329830461974:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.398988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241330534165790:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(27183986ns) {HandleHandshakeStatus:329(27182957ns) {DropIncomingHandshake:442(128ns) DropOutgoingHandshake:457(310ns) SwitchToState:179(195ns) ProcessPendingSessionEvents:495(27087474ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27045190ns max# 26970367ns}}}
2025-12-24T02:11:22.399201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241330534165790:2113] [node 5] ICP18 handshake (actor [3:7587241330534165806:2118]) is too old
2025-12-24T02:11:22.403647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329830461903:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.403669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.403679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329830461903:2048] [node 1] ICP22 created new session: [4:7587241329830461999:2048]
2025-12-24T02:11:22.403685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329830461999:2048] [node 1] ICS09 handshake done sender: [4:7587241329830461974:2113] self: [4:7587241329830461990:0] peer: [1:7587241330321919712:0] socket: 112 qp: -1
2025-12-24T02:11:22.403692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329830461999:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.403700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.419114Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:22.419204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241332220247110:2048] [node 1] ICS01 socket: 84 reason# EndOfStream
2025-12-24T02:11:22.419213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241332220247020:2048] [node 1] ICP30 unregister session Session# [2:7587241332220247110:2048] VirtualId# [2:7587241332220247102:0]
2025-12-24T02:11:22.419220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241332220247110:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.419253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.419272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.426765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:22.426870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241330534165798:2048] [node 1] ICS01 socket: 87 reason# EndOfStream
2025-12-24T02:11:22.426879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241330534165715:2048] [node 1] ICP30 unregister session Session# [3:7587241330534165798:2048] VirtualId# [3:7587241330534165781:0]
2025-12-24T02:11:22.426887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241330534165798:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.426954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.426987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.439195Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:22.439260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241330526840718:2048] [node 1] ICS07 socket disconnect 117 reason# EndOfStream
2025-12-24T02:11:22.439269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241330526840718:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.438782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:22.438884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241329570005697:2048] [node 1] ICS07 socket disconnect 75 reason# EndOfStream
2025-12-24T02:11:22.438890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241329570005697:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.438954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.438979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241329570005697:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:22.441320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:22.441391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241329830461999:2048] [node 1] ICS01 socket: 112 reason# EndOfStream
2025-12-24T02:11:22.441398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241329830461903:2048] [node 1] ICP30 unregister session Session# [4:7587241329830461999:2048] VirtualId# [4:7587241329830461990:0]
2025-12-24T02:11:22.441403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241329830461999:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.441446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.441473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.439303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.439330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241330526840718:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:22.439622Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241330526840634:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.439630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241330526840718:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.439635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241330526840634:2048] [node 1] ICP30 unregister session Session# [5:7587241330526840718:2048] VirtualId# [5:7587241330526840716:0]
2025-12-24T02:11:22.439661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.439667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241330526840634:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.439670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.445957Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:22.446022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241330534165792:2049] [node 2] ICS07 socket disconnect 92 reason# EndOfStream
2025-12-24T02:11:22.446027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241330534165792:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.446055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.446069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241330534165792:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:22.444112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241329570005616:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.444132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241329570005697:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.444138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329570005616:2048] [node 1] ICP30 unregister session Session# [6:7587241329570005697:2048] VirtualId# [6:7587241329570005688:0]
2025-12-24T02:11:22.444181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.444187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241329570005616:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.444190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6203) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.448437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241330534165716:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:22.448450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241330534165792:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:22.448756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241330534165716:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.448767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241330534165792:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.448772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241330534165716:2049] [node 2] ICP30 unregister session Session# [3:7587241330534165792:2049] VirtualId# [3:7587241330534165780:0]
2025-12-24T02:11:22.448805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.448811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241330534165716:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.448813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.450261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241330526840634:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:22.451462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:22.451547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241329830461971:2049] [node 2] ICS07 socket disconnect 85 reason# EndOfStream
2025-12-24T02:11:22.451553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241329830461971:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.451601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.451619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241329830461971:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:22.452614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241329830461904:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.452623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241329830461971:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.452627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241329830461904:2049] [node 2] ICP30 unregister session Session# [4:7587241329830461971:2049] VirtualId# [4:7587241329830461969:0]
2025-12-24T02:11:22.452656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.452659Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241329830461904:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.452660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.451797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:22.451860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241329570005701:2049] [node 2] ICS07 socket disconnect 111 reason# EndOfStream
2025-12-24T02:11:22.451864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241329570005701:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.451889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.451909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241329570005701:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:22.452330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241329570005617:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.452336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241329570005701:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.452339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329570005617:2049] [node 2] ICP30 unregister session Session# [6:7587241329570005701:2049] VirtualId# [6:7587241329570005699:0]
2025-12-24T02:11:22.452372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.452377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241329570005617:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.452379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5742) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.454766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241329570005616:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:22.452112Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:22.452198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241330526840723:2049] [node 2] ICS01 socket: 106 reason# EndOfStream
2025-12-24T02:11:22.452212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241330526840635:2049] [node 2] ICP30 unregister session Session# [5:7587241330526840723:2049] VirtualId# [5:7587241330526840703:0]
2025-12-24T02:11:22.452217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241330526840723:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.452274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.452299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.462809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:22.462893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241329570005706:2050] [node 3] ICS01 socket: 126 reason# EndOfStream
2025-12-24T02:11:22.462901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329570005618:2050] [node 3] ICP30 unregister session Session# [6:7587241329570005706:2050] VirtualId# [6:7587241329570005704:0]
2025-12-24T02:11:22.462907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241329570005706:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.462959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.462982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.463156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241329570005617:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:22.463020Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:22.463077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241330526840731:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-12-24T02:11:22.463081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241330526840636:2050] [node 3] ICP30 unregister session Session# [5:7587241330526840731:2050] VirtualId# [5:7587241330526840725:0]
2025-12-24T02:11:22.463085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241330526840731:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.463109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.463129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.481572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:22.481663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241329570005695:2106] [node 4] ICS07 socket disconnect 88 reason# EndOfStream
2025-12-24T02:11:22.481668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241329570005695:2106] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.481717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:22.481744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241329570005695:2106] [node 4] ICS15 start handshake
2025-12-24T02:11:22.482111Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241329570005679:2106] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27822) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.482124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241329570005695:2106] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.482128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329570005679:2106] [node 4] ICP30 unregister session Session# [6:7587241329570005695:2106] VirtualId# [6:7587241329570005680:0]
2025-12-24T02:11:22.482153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:22.482159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241329570005679:2106] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.482161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27822) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:22.522893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241331531187048:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.522906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241331531187049:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.520122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241329598858521:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:22.520146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241329598858522:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.527414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241330350832155:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.527434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241330350832156:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.524063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329853492799:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.524070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329853492800:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.524075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329853492801:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.527174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241329798244948:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.527182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241329798244949:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.527186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241329798244950:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.554732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241331848598308:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.554738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241331848598309:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.554772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241331848598310:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.565496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241331848598342:2080] [node 4] ICP01 ready to work
2025-12-24T02:11:22.567059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241331531187084:2082] [node 4] ICP01 ready to work
2025-12-24T02:11:22.570862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241329598858586:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:22.583674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241329598858584:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:22.587041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241329598858585:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:22.587221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241331531187112:2084] [node 5] ICP01 ready to work
2025-12-24T02:11:22.589490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241329598858521:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18512740ns) {Configure:104(18512206ns) {ProcessConfigured:135(18160987ns) {StartInitialHandshake:157(18160536ns) {DropHandshakes:470(578ns) {DropIncomingHandshake:442(160ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(312ns) PrepareNewSessionHandshake:477(632ns) {SwitchToState:179(429ns)}}}}}
2025-12-24T02:11:22.587749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241331848598376:2084] [node 5] ICP01 ready to work
2025-12-24T02:11:22.588346Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241329798244950:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20347821ns) {Configure:104(20346534ns) {ProcessConfigured:135(19962734ns) {StartInitialHandshake:157(19962365ns) {DropHandshakes:470(466ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(313ns)}}}}}
2025-12-24T02:11:22.590808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241330350832219:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:22.590827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241330350832218:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:22.590947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241330350832221:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.595446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241329598858595:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8481
2025-12-24T02:11:22.596856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492864:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.596946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492865:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.598262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492864:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.606802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241331531187049:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19453457ns) {Configure:104(19452377ns) {ProcessConfigured:135(19109215ns) {StartInitialHandshake:157(19108801ns) {DropHandshakes:470(488ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(139ns) PrepareNewSessionHandshake:477(1497ns) {SwitchToState:179(234ns)}}}}}
2025-12-24T02:11:22.607523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187121:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
2025-12-24T02:11:22.607537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187122:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
2025-12-24T02:11:22.610791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241329798245001:2073] [node 6] ICP01 ready to work
2025-12-24T02:11:22.610825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241329798244995:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:22.615042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241330350832155:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24150840ns) {Configure:104(24150272ns) {ProcessConfigured:135(23781641ns) {StartInitialHandshake:157(23781251ns) {DropHandshakes:470(334ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(1400ns) {SwitchToState:179(748ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.615260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832225:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.615277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832226:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.615343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832227:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.616051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832228:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.616116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832229:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.616496Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241330350832221:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25462515ns) {Configure:104(25461454ns) {ProcessConfigured:135(25206334ns) {StartInitialHandshake:157(25205945ns) {DropHandshakes:470(350ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(948ns) {SwitchToState:179(694ns)}}}}}
2025-12-24T02:11:22.618771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241329598858522:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35032464ns) {Configure:104(35031681ns) {ProcessConfigured:135(34685240ns) {StartInitialHandshake:157(34684848ns) {DropHandshakes:470(382ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(135ns) PrepareNewSessionHandshake:477(919ns) {SwitchToState:179(528ns)}}}}}
2025-12-24T02:11:22.619879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329853492863:2078] [node 6] ICP01 ready to work
2025-12-24T02:11:22.619476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241331531187117:2086] [node 6] ICP01 ready to work
2025-12-24T02:11:22.619767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832228:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.620033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832226:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.620055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241330350832155:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.622715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241329853492862:2072] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.622867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241330350832156:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51487510ns) {Configure:104(51485973ns) {ProcessConfigured:135(51120202ns) {StartInitialHandshake:157(51119651ns) {DropHandshakes:470(302ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(471ns) PrepareNewSessionHandshake:477(1185ns) {SwitchToState:179(292ns)}}}}}
2025-12-24T02:11:22.622842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858590:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.622857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241329598858590:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.623308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241329598858601:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8481
2025-12-24T02:11:22.623333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241329598858603:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8481
2025-12-24T02:11:22.623344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241329598858602:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8481
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.627083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245026:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.627208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245025:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.627246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245027:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.631034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858595:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.631232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241331531187048:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47167613ns) {Configure:104(47167203ns) {ProcessConfigured:135(46924189ns) {StartInitialHandshake:157(46923770ns) {DropHandshakes:470(289ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1348ns) {SwitchToState:179(676ns)}}}}}
2025-12-24T02:11:22.631308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187121:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.630867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241331848598376:2084] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43002798ns) {Configure:104(43001862ns) {ProcessConfigured:135(42726666ns) {StartInitialHandshake:157(42726403ns) {DropHandshakes:470(238ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(143ns) PrepareNewSessionHandshake:477(1256ns) {SwitchToState:179(781ns)}}}}}
2025-12-24T02:11:22.631853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858602:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.630811Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241329798244995:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19848984ns) {Configure:104(19848041ns) {ProcessConfigured:135(19479361ns) {StartInitialHandshake:157(19478687ns) {DropHandshakes:470(734ns) {DropIncomingHandshake:442(137ns) DropOutgoingHandshake:457(146ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(1488ns) {SwitchToState:179(738ns)}}}}}
2025-12-24T02:11:22.631462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245025:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.631502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245026:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.631528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241329798244948:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.631539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241329798244950:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.634902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832229:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.635429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492876:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.635698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492877:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.638836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492865:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.638877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329853492799:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.638959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492876:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.638980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329853492801:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.637588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241331531187117:2086] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17956239ns) {Configure:104(17955562ns) {ProcessConfigured:135(17633755ns) {StartInitialHandshake:157(17633144ns) {DropHandshakes:470(418ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1049ns) {SwitchToState:179(593ns)}}}}}
2025-12-24T02:11:22.639100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187103:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.639105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241331531187103:2099] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.639127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858592:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.639138Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241329598858592:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.639443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241329598858590:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.639562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241329598858521:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.639578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.639586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241329598858521:2048] [node 2] ICP22 created new session: [1:7587241329598858607:2048]
2025-12-24T02:11:22.639593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329598858607:2048] [node 2] ICS09 handshake done sender: [1:7587241329598858590:2110] self: [1:7587241329598858587:0] peer: [2:7587241330350832239:0] socket: 86 qp: -1
2025-12-24T02:11:22.639600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329598858607:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:22.639610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.639737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492883:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.639827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241331531187103:2099] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.640032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329853492876:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.640103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329853492801:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.640119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.640125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329853492801:2050] [node 3] ICP22 created new session: [4:7587241329853492884:2050]
2025-12-24T02:11:22.640131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329853492884:2050] [node 3] ICS09 handshake done sender: [4:7587241329853492876:2115] self: [4:7587241329853492882:0] peer: [3:7587241331531187085:0] socket: 114 qp: -1
2025-12-24T02:11:22.640137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329853492884:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.640154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.640551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241331531187084:2082] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.640563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.640570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241331531187084:2082] [node 4] ICP22 created new session: [3:7587241331531187131:2082]
2025-12-24T02:11:22.640574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241331531187131:2082] [node 4] ICS09 handshake done sender: [3:7587241331531187103:2099] self: [3:7587241331531187085:0] peer: [4:7587241329853492882:0] socket: 100 qp: -1
2025-12-24T02:11:22.640579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241331531187131:2082] [node 4] ICS10 traffic start
2025-12-24T02:11:22.640594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.647156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858594:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.647170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241329598858594:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.647012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187120:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.647032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241331531187120:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.651311Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241331848598310:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20031671ns) {Configure:104(20031044ns) {ProcessConfigured:135(19693596ns) {StartInitialHandshake:157(19692938ns) {DropHandshakes:470(655ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(142ns) PrepareNewSessionHandshake:477(917ns) {SwitchToState:179(625ns)}}}}}
2025-12-24T02:11:22.650910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832225:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.654795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241329853492862:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31881069ns) {Configure:104(31880438ns) {ProcessConfigured:135(31569949ns) {StartInitialHandshake:157(31569335ns) {DropHandshakes:470(389ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(1330ns) {SwitchToState:179(774ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.659050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598386:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.659072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598387:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.658868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241330350832243:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59445
2025-12-24T02:11:22.662858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241329598858592:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.663392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241330350832226:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.663527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241330350832155:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.663544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.663551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241330350832155:2048] [node 1] ICP22 created new session: [2:7587241330350832244:2048]
2025-12-24T02:11:22.663557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241330350832244:2048] [node 1] ICS09 handshake done sender: [2:7587241330350832226:2112] self: [2:7587241330350832239:0] peer: [1:7587241329598858587:0] socket: 106 qp: -1
2025-12-24T02:11:22.663564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241330350832244:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.663583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.663040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241329598858585:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.663057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.663065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241329598858585:2107] [node 4] ICP22 created new session: [1:7587241329598858612:2107]
2025-12-24T02:11:22.663071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329598858612:2107] [node 4] ICS09 handshake done sender: [1:7587241329598858592:2111] self: [1:7587241329598858591:0] peer: [4:7587241329853492881:0] socket: 87 qp: -1
2025-12-24T02:11:22.663078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329598858612:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:22.663088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.663362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241329598858611:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8481
2025-12-24T02:11:22.664649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241329598858594:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.664811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241329598858584:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.664828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.664835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241329598858584:2108] [node 5] ICP22 created new session: [1:7587241329598858616:2108]
2025-12-24T02:11:22.664842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329598858616:2108] [node 5] ICS09 handshake done sender: [1:7587241329598858594:2112] self: [1:7587241329598858593:0] peer: [5:7587241329798245029:0] socket: 88 qp: -1
2025-12-24T02:11:22.664847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329598858616:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:22.664854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.666761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241331848598387:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.666808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241331848598308:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.666929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187134:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
2025-12-24T02:11:22.667086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858611:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.667125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858599:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.667127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241329598858599:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.667928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598389:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.666851Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245031:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.667753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187134:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.667778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241331531187048:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.672054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492888:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.672726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241331531187120:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.670967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241329598858599:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.671144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241329598858586:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.671163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.671169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241329598858586:2109] [node 6] ICP22 created new session: [1:7587241329598858620:2109]
2025-12-24T02:11:22.671176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329598858620:2109] [node 6] ICS09 handshake done sender: [1:7587241329598858599:2114] self: [1:7587241329598858589:0] peer: [6:7587241331848598388:0] socket: 127 qp: -1
2025-12-24T02:11:22.671182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329598858620:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:22.671190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.671193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241329598858600:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.671199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241329598858600:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.674878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241331531187112:2084] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.674903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.674913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241331531187112:2084] [node 5] ICP22 created new session: [3:7587241331531187137:2084]
2025-12-24T02:11:22.674920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241331531187137:2084] [node 5] ICS09 handshake done sender: [3:7587241331531187120:2112] self: [3:7587241331531187119:0] peer: [5:7587241329798245030:0] socket: 102 qp: -1
2025-12-24T02:11:22.674930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241331531187137:2084] [node 5] ICS10 traffic start
2025-12-24T02:11:22.674937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.675314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241331848598387:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.675163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241329598858600:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.675385Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329853492865:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.675602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329853492799:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.675619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.675625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329853492799:2048] [node 1] ICP22 created new session: [4:7587241329853492889:2048]
2025-12-24T02:11:22.675630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329853492889:2048] [node 1] ICS09 handshake done sender: [4:7587241329853492865:2111] self: [4:7587241329853492881:0] peer: [1:7587241329598858591:0] socket: 89 qp: -1
2025-12-24T02:11:22.675635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329853492889:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.675660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.675502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241331848598308:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.675518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.675524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241331848598308:2048] [node 1] ICP22 created new session: [6:7587241331848598390:2048]
2025-12-24T02:11:22.675529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241331848598390:2048] [node 1] ICS09 handshake done sender: [6:7587241331848598387:2116] self: [6:7587241331848598388:0] peer: [1:7587241329598858589:0] socket: 128 qp: -1
2025-12-24T02:11:22.675537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241331848598390:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.675544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.677389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241329598858522:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.677413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.677419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241329598858522:2049] [node 3] ICP22 created new session: [1:7587241329598858623:2049]
2025-12-24T02:11:22.677425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241329598858623:2049] [node 3] ICS09 handshake done sender: [1:7587241329598858600:2115] self: [1:7587241329598858588:0] peer: [3:7587241331531187135:0] socket: 130 qp: -1
2025-12-24T02:11:22.677431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241329598858623:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:22.677438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.680835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241331531187048:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241331531187134:2117] held: no
2025-12-24T02:11:22.680323Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241330350832155:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16686644ns) {HandleHandshakeStatus:329(16686029ns) {DropIncomingHandshake:442(226ns) DropOutgoingHandshake:457(1682ns) SwitchToState:179(204ns) ProcessPendingSessionEvents:495(7665310ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7642885ns max# 7575552ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.683211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598393:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.683000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241329798244948:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241329798245025:2114] held: no
2025-12-24T02:11:22.683733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241329798245025:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.692981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241331848598386:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.693024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241331848598342:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.693944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492877:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.693988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241329853492800:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.690976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241329798244950:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241329798245026:2115] held: no
2025-12-24T02:11:22.691131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241329798244948:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.691146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.691153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241329798244948:2048] [node 1] ICP22 created new session: [5:7587241329798245035:2048]
2025-12-24T02:11:22.691160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329798245035:2048] [node 1] ICS09 handshake done sender: [5:7587241329798245025:2114] self: [5:7587241329798245029:0] peer: [1:7587241329598858593:0] socket: 104 qp: -1
2025-12-24T02:11:22.691170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329798245035:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.691178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.698769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187139:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
2025-12-24T02:11:22.695112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598394:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.698738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492879:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.698768Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241329853492879:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.699184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241331848598394:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.699237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241331848598310:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.699917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832233:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.699938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241330350832233:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.703004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245027:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.702892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832227:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.702937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187129:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.702948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241331531187129:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.704078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241331531187129:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.704155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241331531187117:2086] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.704167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.704176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241331531187117:2086] [node 6] ICP22 created new session: [3:7587241331531187142:2086]
2025-12-24T02:11:22.704182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241331531187142:2086] [node 6] ICS09 handshake done sender: [3:7587241331531187129:2116] self: [3:7587241331531187127:0] peer: [6:7587241331848598396:0] socket: 96 qp: -1
2025-12-24T02:11:22.704189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241331531187142:2086] [node 6] ICS10 traffic start
2025-12-24T02:11:22.704199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.706897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241331531187134:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.707966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329853492879:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.708899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492894:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.709095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241331848598393:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.709130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241331848598309:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.709377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241329853492863:2078] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.709404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.709410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329853492863:2078] [node 6] ICP22 created new session: [4:7587241329853492895:2078]
2025-12-24T02:11:22.709417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329853492895:2078] [node 6] ICS09 handshake done sender: [4:7587241329853492879:2117] self: [4:7587241329853492878:0] peer: [6:7587241331848598395:0] socket: 91 qp: -1
2025-12-24T02:11:22.709425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329853492895:2078] [node 6] ICS10 traffic start
2025-12-24T02:11:22.709433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.708050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832231:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.708062Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241330350832231:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.707493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245032:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.707547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245033:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.707623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245034:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.707775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241329798244949:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.710891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241331531187048:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.710908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.710916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241331531187048:2048] [node 1] ICP22 created new session: [3:7587241331531187144:2048]
2025-12-24T02:11:22.710921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241331531187144:2048] [node 1] ICS09 handshake done sender: [3:7587241331531187134:2117] self: [3:7587241331531187135:0] peer: [1:7587241329598858588:0] socket: 133 qp: -1
2025-12-24T02:11:22.710927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241331531187144:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.710938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:22.711994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241331848598342:2080] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241331848598386:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.709278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241330350832231:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.709930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832237:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.709934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241330350832237:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.710459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241330350832219:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.710968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245033:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.711010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245034:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.714088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241329798245026:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.714230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.716588Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241331848598309:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241331848598393:2118] held: no
2025-12-24T02:11:22.710472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.710479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241330350832219:2108] [node 5] ICP22 created new session: [2:7587241330350832249:2108]
2025-12-24T02:11:22.710485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241330350832249:2108] [node 5] ICS09 handshake done sender: [2:7587241330350832231:2116] self: [2:7587241330350832230:0] peer: [5:7587241329798245038:0] socket: 111 qp: -1
2025-12-24T02:11:22.710491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241330350832249:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:22.710500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.714051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241330350832237:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.714220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241330350832221:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.714236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.714242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241330350832221:2109] [node 6] ICP22 created new session: [2:7587241330350832253:2109]
2025-12-24T02:11:22.714251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241330350832253:2109] [node 6] ICS09 handshake done sender: [2:7587241330350832237:2118] self: [2:7587241330350832223:0] peer: [6:7587241331848598397:0] socket: 95 qp: -1
2025-12-24T02:11:22.714257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241330350832253:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:22.714265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.716917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241330350832233:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.716979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241330350832218:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.716991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.716997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241330350832218:2107] [node 4] ICP22 created new session: [2:7587241330350832256:2107]
2025-12-24T02:11:22.717004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241330350832256:2107] [node 4] ICS09 handshake done sender: [2:7587241330350832233:2117] self: [2:7587241330350832232:0] peer: [4:7587241329853492892:0] socket: 112 qp: -1
2025-12-24T02:11:22.717010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241330350832256:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:22.717019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.720263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492894:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.720324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187146:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.720484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187147:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.720823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241329853492887:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.720832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241329853492887:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.721370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241329853492899:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47977
2025-12-24T02:11:22.721959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329853492887:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.722099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241329853492877:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.722184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241329853492862:2072] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.722199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.722205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329853492862:2072] [node 5] ICP22 created new session: [4:7587241329853492900:2072]
2025-12-24T02:11:22.722211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329853492900:2072] [node 5] ICS09 handshake done sender: [4:7587241329853492887:2119] self: [4:7587241329853492875:0] peer: [5:7587241329798245043:0] socket: 98 qp: -1
2025-12-24T02:11:22.722217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329853492900:2072] [node 5] ICS10 traffic start
2025-12-24T02:11:22.722223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.719755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241329798244949:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241329798245027:2116] held: no
2025-12-24T02:11:22.722637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241329853492800:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.722651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.719860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241329798244950:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.722658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241329853492800:2049] [node 2] ICP22 created new session: [4:7587241329853492902:2049]
2025-12-24T02:11:22.719871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.722664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241329853492902:2049] [node 2] ICS09 handshake done sender: [4:7587241329853492877:2116] self: [4:7587241329853492892:0] peer: [2:7587241330350832232:0] socket: 119 qp: -1
2025-12-24T02:11:22.719877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241329798244950:2050] [node 3] ICP22 created new session: [5:7587241329798245041:2050]
2025-12-24T02:11:22.722670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241329853492902:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.722676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.719882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329798245041:2050] [node 3] ICS09 handshake done sender: [5:7587241329798245026:2115] self: [5:7587241329798245030:0] peer: [3:7587241331531187119:0] socket: 117 qp: -1
2025-12-24T02:11:22.719891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329798245041:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.719899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.720136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241329798244995:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.720160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241329798245027:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.720686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241329798244949:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.720696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.720704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241329798244949:2049] [node 2] ICP22 created new session: [5:7587241329798245045:2049]
2025-12-24T02:11:22.720710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329798245045:2049] [node 2] ICS09 handshake done sender: [5:7587241329798245027:2116] self: [5:7587241329798245038:0] peer: [2:7587241330350832230:0] socket: 118 qp: -1
2025-12-24T02:11:22.720717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329798245045:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.720739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.724067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241329798245047:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62517
2025-12-24T02:11:22.722907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187146:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.722941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241331531187147:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.722970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241331531187049:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.722981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241331531187117:2086] [node 6] ICP18 handshake (actor [3:7587241331531187147:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.729391Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241331848598310:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241331848598394:2119] held: no
2025-12-24T02:11:22.729635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598399:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.729930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241331848598394:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.730035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241331848598310:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.730051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.730070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241331848598310:2050] [node 3] ICP22 created new session: [6:7587241331848598400:2050]
2025-12-24T02:11:22.730076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241331848598400:2050] [node 3] ICS09 handshake done sender: [6:7587241331848598394:2119] self: [6:7587241331848598396:0] peer: [3:7587241331531187127:0] socket: 125 qp: -1
2025-12-24T02:11:22.730085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241331848598400:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:22.730093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.731052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598402:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.733519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598403:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.733594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598404:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.735028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241331848598393:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.735137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241331848598309:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.735154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.735162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241331848598309:2049] [node 2] ICP22 created new session: [6:7587241331848598405:2049]
2025-12-24T02:11:22.735169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241331848598405:2049] [node 2] ICS09 handshake done sender: [6:7587241331848598393:2118] self: [6:7587241331848598397:0] peer: [2:7587241330350832223:0] socket: 115 qp: -1
2025-12-24T02:11:22.735187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241331848598405:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.735195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.734744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241329798244995:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241329798245033:2120] held: no
2025-12-24T02:11:22.736258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241329798245033:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.735139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241330350832240:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.735156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241330350832240:2119] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.738873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241331848598386:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.738889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241331848598403:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.738923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241331848598376:2084] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.739169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241331848598342:2080] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.739184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.739191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241331848598342:2080] [node 4] ICP22 created new session: [6:7587241331848598408:2080]
2025-12-24T02:11:22.739197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241331848598408:2080] [node 4] ICS09 handshake done sender: [6:7587241331848598386:2115] self: [6:7587241331848598395:0] peer: [4:7587241329853492878:0] socket: 99 qp: -1
2025-12-24T02:11:22.739203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241331848598408:2080] [node 4] ICS10 traffic start
2025-12-24T02:11:22.739211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.739038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241331531187149:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48441
2025-12-24T02:11:22.743934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241330350832240:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.744289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241330350832156:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.744309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.744317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241330350832156:2049] [node 3] ICP22 created new session: [2:7587241330350832259:2049]
2025-12-24T02:11:22.744323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241330350832259:2049] [node 3] ICS09 handshake done sender: [2:7587241330350832240:2119] self: [2:7587241330350832220:0] peer: [3:7587241331531187148:0] socket: 144 qp: -1
2025-12-24T02:11:22.744329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241330350832259:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:22.744335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:22.742878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241329798245024:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.742897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241329798245024:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.743588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241329798244995:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.743601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.743608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241329798244995:2067] [node 4] ICP22 created new session: [5:7587241329798245048:2067]
2025-12-24T02:11:22.743615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329798245048:2067] [node 4] ICS09 handshake done sender: [5:7587241329798245033:2120] self: [5:7587241329798245043:0] peer: [4:7587241329853492875:0] socket: 139 qp: -1
2025-12-24T02:11:22.743624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329798245048:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:22.743634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:22.747492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241331531187049:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241331531187146:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.776348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241331531187146:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.775694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241331848598410:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38251
2025-12-24T02:11:22.776365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241331848598403:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:22.776440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241331848598376:2084] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.776449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:22.776455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241331848598376:2084] [node 5] ICP22 created new session: [6:7587241331848598411:2084]
2025-12-24T02:11:22.776460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241331848598411:2084] [node 5] ICS09 handshake done sender: [6:7587241331848598403:2122] self: [6:7587241331848598407:0] peer: [5:7587241329798245023:0] socket: 78 qp: -1
2025-12-24T02:11:22.776467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241331848598411:2084] [node 5] ICS10 traffic start
2025-12-24T02:11:22.776474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:22.776941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241331531187049:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.776963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.776972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241331531187049:2049] [node 2] ICP22 created new session: [3:7587241331531187150:2049]
2025-12-24T02:11:22.776978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241331531187150:2049] [node 2] ICS09 handshake done sender: [3:7587241331531187146:2119] self: [3:7587241331531187148:0] peer: [2:7587241330350832220:0] socket: 105 qp: -1
2025-12-24T02:11:22.776985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241331531187150:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.776994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:22.775525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241329798245024:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.775831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241329798245001:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.775850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.775857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241329798245001:2073] [node 6] ICP22 created new session: [5:7587241329798245051:2073]
2025-12-24T02:11:22.775864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241329798245051:2073] [node 6] ICS09 handshake done sender: [5:7587241329798245024:2113] self: [5:7587241329798245023:0] peer: [6:7587241331848598407:0] socket: 116 qp: -1
2025-12-24T02:11:22.775870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241329798245051:2073] [node 6] ICS10 traffic start
2025-12-24T02:11:22.775878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.791110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:22.791197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241330350832244:2048] [node 1] ICS07 socket disconnect 106 reason# EndOfStream
2025-12-24T02:11:22.791204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241330350832244:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.791241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.791256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241330350832244:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:22.795102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:22.795202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241331531187144:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-12-24T02:11:22.795210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241331531187048:2048] [node 1] ICP30 unregister session Session# [3:7587241331531187144:2048] VirtualId# [3:7587241331531187135:0]
2025-12-24T02:11:22.795217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241331531187144:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.795266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.795288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.795638Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241330350832155:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:22.795652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241330350832244:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:22.795993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241330350832155:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:22.796003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241330350832244:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:22.796008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241330350832155:2048] [node 1] ICP30 unregister session Session# [2:7587241330350832244:2048] VirtualId# [2:7587241330350832239:0]
2025-12-24T02:11:22.796046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.796051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241330350832155:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.796053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8481) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:22.802812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:22.802908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241329853492889:2048] [node 1] ICS01 socket: 89 reason# EndOfStream
2025-12-24T02:11:22.802916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241329853492799:2048] [node 1] ICP30 unregister session Session# [4:7587241329853492889:2048] VirtualId# [4:7587241329853492881:0]
2025-12-24T02:11:22.802922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241329853492889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.802977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.803004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.803187Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:22.803253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241331848598390:2048] [node 1] ICS01 socket: 128 reason# EndOfStream
2025-12-24T02:11:22.803257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241331848598308:2048] [node 1] ICP30 unregister session Session# [6:7587241331848598390:2048] VirtualId# [6:7587241331848598388:0]
2025-12-24T02:11:22.803261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241331848598390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.803289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.803311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.803041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:22.803106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241329798245035:2048] [node 1] ICS01 socket: 104 reason# EndOfStream
2025-12-24T02:11:22.803110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241329798244948:2048] [node 1] ICP30 unregister session Session# [5:7587241329798245035:2048] VirtualId# [5:7587241329798245029:0]
2025-12-24T02:11:22.803115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241329798245035:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.803139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:22.803158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:22.814468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:22.814562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241331531187150:2049] [node 2] ICS01 socket: 105 reason# EndOfStream
2025-12-24T02:11:22.814570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241331531187049:2049] [node 2] ICP30 unregister session Session# [3:7587241331531187150:2049] VirtualId# [3:7587241331531187148:0]
2025-12-24T02:11:22.814577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241331531187150:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.814623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.814647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.819064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:22.819137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241331848598405:2049] [node 2] ICS01 socket: 115 reason# EndOfStream
2025-12-24T02:11:22.819143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241331848598309:2049] [node 2] ICP30 unregister session Session# [6:7587241331848598405:2049] VirtualId# [6:7587241331848598397:0]
2025-12-24T02:11:22.819148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241331848598405:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.819178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.819199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.821972Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:22.822066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241329853492902:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-12-24T02:11:22.822073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241329853492800:2049] [node 2] ICP30 unregister session Session# [4:7587241329853492902:2049] VirtualId# [4:7587241329853492892:0]
2025-12-24T02:11:22.822079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241329853492902:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.822131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.822157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.818828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:22.818929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241329798245045:2049] [node 2] ICS01 socket: 118 reason# EndOfStream
2025-12-24T02:11:22.818937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241329798244949:2049] [node 2] ICP30 unregister session Session# [5:7587241329798245045:2049] VirtualId# [5:7587241329798245038:0]
2025-12-24T02:11:22.818944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241329798245045:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.818996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:22.819022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:22.828856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:22.828919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241331848598400:2050] [node 3] ICS01 socket: 125 reason# EndOfStream
2025-12-24T02:11:22.828925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241331848598310:2050] [node 3] ICP30 unregister session Session# [6:7587241331848598400:2050] VirtualId# [6:7587241331848598396:0]
2025-12-24T02:11:22.828930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241331848598400:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.828959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.828981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.828614Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:22.828712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241329798245041:2050] [node 3] ICS01 socket: 117 reason# EndOfStream
2025-12-24T02:11:22.828721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241329798244950:2050] [node 3] ICP30 unregister session Session# [5:7587241329798245041:2050] VirtualId# [5:7587241329798245030:0]
2025-12-24T02:11:22.828729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241329798245041:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.828788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:22.828815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:22.849076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:22.849145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241331848598408:2080] [node 4] ICS01 socket: 99 reason# EndOfStream
2025-12-24T02:11:22.849150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241331848598342:2080] [node 4] ICP30 unregister session Session# [6:7587241331848598408:2080] VirtualId# [6:7587241331848598395:0]
2025-12-24T02:11:22.849156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241331848598408:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:22.849182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:22.849203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:22.887257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241331078387755:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:22.887274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241331078387756:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.897836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241329371837056:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.897843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241329371837057:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.896281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241328877808981:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.896293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241328877808982:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:22.901953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241331093231513:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.901972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241331093231514:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.901976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241331093231515:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.910887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328883408120:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.910895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328883408121:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.910911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328883408122:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.916228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329755915268:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:22.916236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329755915269:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:22.916241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329755915270:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:22.937882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328883408176:2102] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.938338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241328877809041:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.938788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241331093231574:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.938975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241328883408183:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.939409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241329371837115:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:22.939569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241328877809042:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.942610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241329371837118:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:22.942583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241329755915330:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:22.943639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241331078387822:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:22.944135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329755915329:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:22.942482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241331093231574:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.942654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241331093231576:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:22.943115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241331093231576:2109] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.948443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328883408178:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.948462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241328883408178:2103] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.950858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809042:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.951060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241328877809044:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:22.951603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241331078387825:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:22.953127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241328877809044:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.953213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809041:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.954864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837115:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.955286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241329371837056:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.955696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328883408178:2103] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:22.957322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329755915327:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.957348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241329755915327:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.955570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241331078387827:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:22.958841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329755915329:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.962024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241328883408176:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.962044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:22.962051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328883408176:2102] [node 4] ICP22 created new session: [5:7587241328883408195:2102]
2025-12-24T02:11:22.962059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328883408195:2102] [node 4] ICS09 handshake done sender: [5:7587241328883408178:2103] self: [5:7587241328883408177:0] peer: [4:7587241331093231577:0] socket: 86 qp: -1
2025-12-24T02:11:22.962066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328883408195:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:22.962076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.964594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241328877809053:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
2025-12-24T02:11:22.964912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241328877808981:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.965257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387816:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.965269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241331078387816:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.965839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241331078387816:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:22.966082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241331078387755:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.966099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.966106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241331078387755:2048] [node 2] ICP22 created new session: [1:7587241331078387834:2048]
2025-12-24T02:11:22.966112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241331078387834:2048] [node 2] ICS09 handshake done sender: [1:7587241331078387816:2107] self: [1:7587241331078387815:0] peer: [2:7587241328877809057:0] socket: 84 qp: -1
2025-12-24T02:11:22.966119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241331078387834:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:22.966138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.967007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241329371837127:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:22.967031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241329371837128:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:22.967163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241329371837118:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24484836ns) {Configure:104(24484274ns) {ProcessConfigured:135(23947109ns) {StartInitialHandshake:157(23946761ns) {DropHandshakes:470(597ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(131ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(899ns) {SwitchToState:179(192ns)}}}}}
2025-12-24T02:11:22.967601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387819:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.967622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241331078387819:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.968406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241329371837129:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:22.970721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241328877809047:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:22.970810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809053:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.969649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241329371837133:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:22.972663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241329371837136:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:22.973830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837128:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.973867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837129:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.973890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837127:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.974012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241329371837057:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.971286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328883408185:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.971802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241331078387837:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51792
2025-12-24T02:11:22.971882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241331078387838:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51792
2025-12-24T02:11:22.971302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241328883408185:2107] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.972465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329755915327:2105] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.976427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241328877809051:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:22.975043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241331078387822:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31296821ns) {Configure:104(31296304ns) {ProcessConfigured:135(31003540ns) {StartInitialHandshake:157(31003178ns) {DropHandshakes:470(528ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(993ns) {SwitchToState:179(440ns)}}}}}
2025-12-24T02:11:22.975129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387837:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.979156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329755915330:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:22.979196Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241329755915330:2102] [node 5] ICP35 event processing took too much time PendingConnection:279(20298007ns) {IncomingHandshake:261(20297425ns) {DropIncomingHandshake:442(72ns) PrepareNewSessionHandshake:477(545ns) {SwitchToState:179(107ns)} GenerateSessionVirtualId:988(367ns) IssueIncomingHandshakeReply:193(20291759ns) {SwitchToState:179(927ns)}}}
2025-12-24T02:11:22.979427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241329755915269:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.979453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:22.979464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329755915269:2049] [node 2] ICP22 created new session: [6:7587241329755915343:2049]
2025-12-24T02:11:22.979484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241329755915343:2049] [node 2] ICS09 handshake done sender: [6:7587241329755915327:2105] self: [6:7587241329755915326:0] peer: [2:7587241328877809048:0] socket: 90 qp: -1
2025-12-24T02:11:22.979497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241329755915343:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:22.979510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.983157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241331078387819:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:22.983307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241331078387756:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.983336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:22.983343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241331078387756:2049] [node 3] ICP22 created new session: [1:7587241331078387842:2049]
2025-12-24T02:11:22.983350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241331078387842:2049] [node 3] ICS09 handshake done sender: [1:7587241331078387819:2108] self: [1:7587241331078387818:0] peer: [3:7587241329371837122:0] socket: 89 qp: -1
2025-12-24T02:11:22.983357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241331078387842:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:22.983366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.984111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241328877809063:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
2025-12-24T02:11:22.984189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241329755915345:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:22.985806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387838:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.985847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241331078387755:2048] [node 2] ICP18 handshake (actor [1:7587241331078387838:2115]) is too old
2025-12-24T02:11:22.985986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241329755915345:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.986350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329755915268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:22.985697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241328877809041:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.986650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387831:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.986659Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241331078387831:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.983045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241329371837142:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:22.990793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241329371837133:2115] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21028414ns) {Configure:104(21027558ns) {ProcessConfigured:135(20708948ns) {StartInitialHandshake:157(20708441ns) {DropHandshakes:470(961ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(136ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1321ns) {SwitchToState:179(729ns)}}}}}
2025-12-24T02:11:22.990905Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241329371837056:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20682) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241329371837115:2106] held: no
2025-12-24T02:11:22.991239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241329371837115:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:22.990917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241328877809047:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19929013ns) {Configure:104(19928289ns) {ProcessConfigured:135(19625134ns) {StartInitialHandshake:157(19624884ns) {DropHandshakes:470(331ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:477(717ns) {SwitchToState:179(35ns)}}}}}
2025-12-24T02:11:22.992777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241328877809064:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
2025-12-24T02:11:22.994762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809052:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.994774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241328877809052:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:22.992161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241331093231578:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:22.994964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241328883408197:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-12-24T02:11:22.995060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241328883408185:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:22.995232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241328883408197:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:22.997032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241328883408183:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:22.997054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:22.997061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328883408183:2105] [node 6] ICP22 created new session: [5:7587241328883408204:2105]
2025-12-24T02:11:22.997068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241328883408204:2105] [node 6] ICS09 handshake done sender: [5:7587241328883408185:2107] self: [5:7587241328883408184:0] peer: [6:7587241329755915341:0] socket: 95 qp: -1
2025-12-24T02:11:22.997075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241328883408204:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:22.997084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.998163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241329755915330:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241329755915329:2106] held: no
2025-12-24T02:11:23.000761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241331078387831:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.000846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241331078387827:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.000856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.000866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241331078387827:2111] [node 6] ICP22 created new session: [1:7587241335373355142:2111]
2025-12-24T02:11:23.000872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241335373355142:2111] [node 6] ICS09 handshake done sender: [1:7587241331078387831:2113] self: [1:7587241331078387830:0] peer: [6:7587241329755915346:0] socket: 115 qp: -1
2025-12-24T02:11:23.000877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241335373355142:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:23.000895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:22.999248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241331093231514:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23716449ns) {Configure:104(23715804ns) {ProcessConfigured:135(23465931ns) {StartInitialHandshake:157(23465555ns) {DropHandshakes:470(365ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(1009ns) {SwitchToState:179(536ns)}}}}}
2025-12-24T02:11:22.999448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241331093231584:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:22.998737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241329371837056:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:22.998763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:22.998770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241329371837056:2048] [node 1] ICP22 created new session: [3:7587241329371837144:2048]
2025-12-24T02:11:22.998777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241329371837144:2048] [node 1] ICS09 handshake done sender: [3:7587241329371837115:2106] self: [3:7587241329371837122:0] peer: [1:7587241331078387818:0] socket: 91 qp: -1
2025-12-24T02:11:22.998786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241329371837144:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:22.998794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.001215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241328877808981:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.001235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.001241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241328877808981:2048] [node 1] ICP22 created new session: [2:7587241333172776362:2048]
2025-12-24T02:11:23.001247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333172776362:2048] [node 1] ICS09 handshake done sender: [2:7587241328877809041:2107] self: [2:7587241328877809057:0] peer: [1:7587241331078387815:0] socket: 85 qp: -1
2025-12-24T02:11:23.001253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333172776362:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.001269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.003915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241328877808981:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.004283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241328877809042:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.007869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375504:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-12-24T02:11:23.007878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375505:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-12-24T02:11:23.007929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375506:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.008242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241335373355144:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51792
2025-12-24T02:11:23.008586Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241335373355144:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.008350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333178375504:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.008438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328883408120:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.008492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333178375505:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.008506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328883408122:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.009429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333178375506:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.009494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241328883408121:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.009548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375509:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.010084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241331093231574:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.009990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882643:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.010133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882644:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.010168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882645:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.007135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241328877809044:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.010357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241331093231576:2109] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.007151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.007156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241328877809044:2109] [node 6] ICP22 created new session: [2:7587241333172776365:2109]
2025-12-24T02:11:23.010371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.007161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333172776365:2109] [node 6] ICS09 handshake done sender: [2:7587241328877809042:2108] self: [2:7587241328877809048:0] peer: [6:7587241329755915326:0] socket: 93 qp: -1
2025-12-24T02:11:23.007168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333172776365:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:23.007173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.007440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333172776364:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
2025-12-24T02:11:23.007963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333172776364:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.009199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241328877809052:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.009284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241328877808982:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.009299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.009305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241328877808982:2049] [node 3] ICP22 created new session: [2:7587241333172776370:2049]
2025-12-24T02:11:23.009311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333172776370:2049] [node 3] ICS09 handshake done sender: [2:7587241328877809052:2112] self: [2:7587241328877809049:0] peer: [3:7587241329371837140:0] socket: 110 qp: -1
2025-12-24T02:11:23.009317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333172776370:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.009322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.008907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837130:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.008919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241329371837130:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.009378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333666804443:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
2025-12-24T02:11:23.009785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241329371837129:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.009800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241329371837130:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.009866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241329371837057:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.009876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.009882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241329371837057:2049] [node 2] ICP22 created new session: [3:7587241333666804445:2049]
2025-12-24T02:11:23.009887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333666804445:2049] [node 2] ICS09 handshake done sender: [3:7587241329371837129:2113] self: [3:7587241329371837140:0] peer: [2:7587241328877809049:0] socket: 117 qp: -1
2025-12-24T02:11:23.009892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333666804445:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.009899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.010437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241329371837118:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.010449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.010455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241329371837118:2108] [node 5] ICP22 created new session: [3:7587241333666804447:2108]
2025-12-24T02:11:23.010460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333666804447:2108] [node 5] ICS09 handshake done sender: [3:7587241329371837130:2114] self: [3:7587241329371837119:0] peer: [5:7587241333178375508:0] socket: 118 qp: -1
2025-12-24T02:11:23.010464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333666804447:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:23.010470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.010378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241331093231576:2109] [node 5] ICP22 created new session: [4:7587241335388198883:2109]
2025-12-24T02:11:23.010383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241335388198883:2109] [node 5] ICS09 handshake done sender: [4:7587241331093231574:2108] self: [4:7587241331093231577:0] peer: [5:7587241328883408177:0] socket: 88 qp: -1
2025-12-24T02:11:23.010390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241335388198883:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.010414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.010359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333178375505:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.010572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328883408122:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.010581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.010598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328883408122:2050] [node 3] ICP22 created new session: [5:7587241333178375511:2050]
2025-12-24T02:11:23.010602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333178375511:2050] [node 3] ICS09 handshake done sender: [5:7587241333178375505:2117] self: [5:7587241333178375508:0] peer: [3:7587241329371837119:0] socket: 131 qp: -1
2025-12-24T02:11:23.010607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333178375511:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.010612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.014908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329755915345:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.015226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241329755915329:2106] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.015365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387829:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.015382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241331078387829:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.014950Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241331093231515:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43296665ns) {Configure:104(43295991ns) {ProcessConfigured:135(42936124ns) {StartInitialHandshake:157(42935699ns) {DropHandshakes:470(483ns) {DropIncomingHandshake:442(90ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(156ns) PrepareNewSessionHandshake:477(2185ns) {SwitchToState:179(1096ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.018880Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241331093231513:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43559042ns) {Configure:104(43558415ns) {ProcessConfigured:135(43244363ns) {StartInitialHandshake:157(43243537ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(1318ns) {SwitchToState:179(367ns)}}}}}
2025-12-24T02:11:23.018930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198882:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.018871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329755915268:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.018891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.018900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329755915268:2048] [node 1] ICP22 created new session: [6:7587241334050882647:2048]
2025-12-24T02:11:23.018906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241334050882647:2048] [node 1] ICS09 handshake done sender: [6:7587241329755915345:2113] self: [6:7587241329755915346:0] peer: [1:7587241331078387830:0] socket: 119 qp: -1
2025-12-24T02:11:23.018916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241334050882647:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.018924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.019271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241334050882649:2117] [node 4] ICP01 ready to work
2025-12-24T02:11:23.019347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329755915330:2102] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.019353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.019358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329755915330:2102] [node 5] ICP22 created new session: [6:7587241334050882650:2102]
2025-12-24T02:11:23.019362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241334050882650:2102] [node 5] ICS09 handshake done sender: [6:7587241329755915329:2106] self: [6:7587241329755915341:0] peer: [5:7587241328883408184:0] socket: 96 qp: -1
2025-12-24T02:11:23.019367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241334050882650:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:23.019372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.020797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241335373355146:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51792
2025-12-24T02:11:23.020328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809061:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.020341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241328877809061:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.021482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333666804449:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.023249Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241331078387827:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(22288440ns) {HandleHandshakeStatus:329(22287854ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(179ns) SwitchToState:179(167ns) ProcessPendingSessionEvents:495(19377955ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19360066ns max# 19298594ns}}}
2025-12-24T02:11:23.025725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198881:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.023014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333172776372:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56075
2025-12-24T02:11:23.026795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198891:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.027590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241335388198881:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.035044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241334050882645:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.035940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241331078387829:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.036066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241331078387825:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.036091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.036103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241331078387825:2110] [node 5] ICP22 created new session: [1:7587241335373355148:2110]
2025-12-24T02:11:23.036113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241335373355148:2110] [node 5] ICS09 handshake done sender: [1:7587241331078387829:2112] self: [1:7587241331078387828:0] peer: [5:7587241333178375507:0] socket: 108 qp: -1
2025-12-24T02:11:23.036122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241335373355148:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:23.036130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.036654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241328883408120:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20682) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241333178375504:2116] held: no
2025-12-24T02:11:23.037109Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241329371837118:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(26573614ns) {HandleHandshakeStatus:329(26573136ns) {DropIncomingHandshake:442(2934ns) DropOutgoingHandshake:457(111ns) SwitchToState:179(67ns) ProcessPendingSessionEvents:495(21446161ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21426346ns max# 21365559ns}}}
2025-12-24T02:11:23.039239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375513:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-12-24T02:11:23.039060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198892:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.039357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837138:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.039370Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241329371837138:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.039377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241328877809061:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.039745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241328877809051:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.039764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.039770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241328877809051:2111] [node 5] ICP22 created new session: [2:7587241333172776374:2111]
2025-12-24T02:11:23.039776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333172776374:2111] [node 5] ICS09 handshake done sender: [2:7587241328877809061:2115] self: [2:7587241328877809060:0] peer: [5:7587241333178375510:0] socket: 124 qp: -1
2025-12-24T02:11:23.039781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333172776374:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:23.039787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.047827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333178375504:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.047936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328883408120:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.047951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.047960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328883408120:2048] [node 1] ICP22 created new session: [5:7587241333178375514:2048]
2025-12-24T02:11:23.048028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333178375514:2048] [node 1] ICS09 handshake done sender: [5:7587241333178375504:2116] self: [5:7587241333178375507:0] peer: [1:7587241331078387828:0] socket: 125 qp: -1
2025-12-24T02:11:23.048049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333178375514:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.048105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.049962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241335388198882:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.050023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241335388198892:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.050095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241331093231514:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.050110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241331093231515:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.049648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241329755915270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.050733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241331093231513:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.050770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241331093231513:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23121244ns) {IncomingHandshake:261(23120590ns) {DropIncomingHandshake:442(153ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(290ns)} GenerateSessionVirtualId:988(336ns) IssueIncomingHandshakeReply:193(23113076ns) {SwitchToState:179(592ns)}}}
2025-12-24T02:11:23.050058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241331078387839:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.050083Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241331078387839:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.050933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241335388198891:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.051558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241329371837138:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.051747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241328883408121:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241333178375506:2118] held: no
2025-12-24T02:11:23.051937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333178375516:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18982
2025-12-24T02:11:23.052462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241329371837136:2116] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.052487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.052494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241329371837136:2116] [node 6] ICP22 created new session: [3:7587241333666804452:2116]
2025-12-24T02:11:23.052499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333666804452:2116] [node 6] ICS09 handshake done sender: [3:7587241329371837138:2117] self: [3:7587241329371837137:0] peer: [6:7587241334050882655:0] socket: 103 qp: -1
2025-12-24T02:11:23.052505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333666804452:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:23.052513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.052717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333178375506:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.051624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882656:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.051866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241334050882645:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.052080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241329755915270:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.052104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.052113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241329755915270:2050] [node 3] ICP22 created new session: [6:7587241334050882657:2050]
2025-12-24T02:11:23.052120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241334050882657:2050] [node 3] ICS09 handshake done sender: [6:7587241334050882645:2116] self: [6:7587241334050882655:0] peer: [3:7587241329371837137:0] socket: 139 qp: -1
2025-12-24T02:11:23.052141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241334050882657:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.052149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.054800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241328883408121:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.054817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.054823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241328883408121:2049] [node 2] ICP22 created new session: [5:7587241333178375517:2049]
2025-12-24T02:11:23.054829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333178375517:2049] [node 2] ICS09 handshake done sender: [5:7587241333178375506:2118] self: [5:7587241333178375510:0] peer: [2:7587241328877809060:0] socket: 132 qp: -1
2025-12-24T02:11:23.054834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333178375517:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.054842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.056893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241329371837143:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.056904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241329371837143:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.056696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241328877809065:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.056712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241328877809065:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.056402Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241331093231584:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(45390391ns) {Configure:104(45388530ns) {ProcessConfigured:135(45085459ns) {StartInitialHandshake:157(45084660ns) {DropHandshakes:470(476ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(721ns) {SwitchToState:179(80ns)}}}}}
2025-12-24T02:11:23.060196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241331078387839:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.060339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241331078387822:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.060352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.060359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241331078387822:2109] [node 4] ICP22 created new session: [1:7587241335373355151:2109]
2025-12-24T02:11:23.060365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241335373355151:2109] [node 4] ICS09 handshake done sender: [1:7587241331078387839:2116] self: [1:7587241331078387823:0] peer: [4:7587241335388198893:0] socket: 126 qp: -1
2025-12-24T02:11:23.060371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241335373355151:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:23.060379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.072080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241328877809065:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.072216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241328877809047:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.072228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.072237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241328877809047:2110] [node 4] ICP22 created new session: [2:7587241333172776377:2110]
2025-12-24T02:11:23.072244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333172776377:2110] [node 4] ICS09 handshake done sender: [2:7587241328877809065:2118] self: [2:7587241328877809059:0] peer: [4:7587241335388198894:0] socket: 130 qp: -1
2025-12-24T02:11:23.072251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333172776377:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:23.072263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.079653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333666804449:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.079676Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241329371837143:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.079729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241329371837133:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.079751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.079763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241329371837133:2115] [node 4] ICP22 created new session: [3:7587241333666804456:2115]
2025-12-24T02:11:23.079770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333666804456:2115] [node 4] ICS09 handshake done sender: [3:7587241329371837143:2119] self: [3:7587241329371837134:0] peer: [4:7587241335388198895:0] socket: 148 qp: -1
2025-12-24T02:11:23.079794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333666804456:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:23.079809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.080534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241331078387822:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(20087799ns) {HandleHandshakeStatus:329(20086924ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(122ns) ProcessPendingSessionEvents:495(20010698ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19984954ns max# 19918366ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.080500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198898:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.080811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241335388198881:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.080888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241331093231513:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.080901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.080907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241331093231513:2048] [node 1] ICP22 created new session: [4:7587241335388198901:2048]
2025-12-24T02:11:23.080913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241335388198901:2048] [node 1] ICS09 handshake done sender: [4:7587241335388198881:2113] self: [4:7587241335388198893:0] peer: [1:7587241331078387823:0] socket: 97 qp: -1
2025-12-24T02:11:23.080920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241335388198901:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.080926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.081474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198899:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.082000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241335388198900:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14389
2025-12-24T02:11:23.082067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241335388198882:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.082128Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241331093231515:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12830) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241335388198892:2118] held: no
2025-12-24T02:11:23.082208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241335388198892:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.082243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241331093231514:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.082255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.082266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241331093231514:2049] [node 2] ICP22 created new session: [4:7587241335388198903:2049]
2025-12-24T02:11:23.082273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241335388198903:2049] [node 2] ICS09 handshake done sender: [4:7587241335388198882:2114] self: [4:7587241335388198894:0] peer: [2:7587241328877809059:0] socket: 140 qp: -1
2025-12-24T02:11:23.082280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241335388198903:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.082286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.083000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241331093231515:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.083018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.083026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241331093231515:2050] [node 3] ICP22 created new session: [4:7587241335388198905:2050]
2025-12-24T02:11:23.083031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241335388198905:2050] [node 3] ICS09 handshake done sender: [4:7587241335388198892:2118] self: [4:7587241335388198895:0] peer: [3:7587241329371837134:0] socket: 149 qp: -1
2025-12-24T02:11:23.083038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241335388198905:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.083043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.090048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241331078387822:2109] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.090894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882659:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.091063Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241328877809047:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.094835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241334050882659:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.094902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241334050882649:2117] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.098993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241335388198896:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.099013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241335388198896:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.099791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241335388198896:2119] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.102860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241331093231584:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.102897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.102907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241331093231584:2111] [node 6] ICP22 created new session: [4:7587241335388198909:2111]
2025-12-24T02:11:23.102927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241335388198909:2111] [node 6] ICS09 handshake done sender: [4:7587241335388198896:2119] self: [4:7587241335388198885:0] peer: [6:7587241334050882660:0] socket: 128 qp: -1
2025-12-24T02:11:23.102934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241335388198909:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:23.102949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.103066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241334050882661:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33563
2025-12-24T02:11:23.106859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241334050882659:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.106892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241334050882649:2117] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13624) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241334050882659:2120] held: no
2025-12-24T02:11:23.107101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241334050882649:2117] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.107117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.107126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241334050882649:2117] [node 4] ICP22 created new session: [6:7587241334050882662:2117]
2025-12-24T02:11:23.107132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241334050882662:2117] [node 4] ICS09 handshake done sender: [6:7587241334050882659:2120] self: [6:7587241334050882660:0] peer: [4:7587241335388198885:0] socket: 141 qp: -1
2025-12-24T02:11:23.107140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241334050882662:2117] [node 4] ICS10 traffic start
2025-12-24T02:11:23.107151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.116710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:23.116797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241329371837144:2048] [node 1] ICS01 socket: 91 reason# EndOfStream
2025-12-24T02:11:23.116808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241329371837056:2048] [node 1] ICP30 unregister session Session# [3:7587241329371837144:2048] VirtualId# [3:7587241329371837122:0]
2025-12-24T02:11:23.116814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241329371837144:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.116847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.116862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.118238Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:23.118557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:23.118613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333178375514:2048] [node 1] ICS01 socket: 125 reason# EndOfStream
2025-12-24T02:11:23.118618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328883408120:2048] [node 1] ICP30 unregister session Session# [5:7587241333178375514:2048] VirtualId# [5:7587241333178375507:0]
2025-12-24T02:11:23.118622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333178375514:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.118652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.118675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.119404Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:23.119486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241334050882647:2048] [node 1] ICS01 socket: 119 reason# EndOfStream
2025-12-24T02:11:23.119490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329755915268:2048] [node 1] ICP30 unregister session Session# [6:7587241334050882647:2048] VirtualId# [6:7587241329755915346:0]
2025-12-24T02:11:23.119495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241334050882647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.119529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.119545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.119624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:23.119679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241335388198901:2048] [node 1] ICS01 socket: 97 reason# EndOfStream
2025-12-24T02:11:23.119683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241331093231513:2048] [node 1] ICP30 unregister session Session# [4:7587241335388198901:2048] VirtualId# [4:7587241335388198893:0]
2025-12-24T02:11:23.119686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241335388198901:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.119721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.119741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.130791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:23.130885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241333666804445:2049] [node 2] ICS01 socket: 117 reason# EndOfStream
2025-12-24T02:11:23.130899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241329371837057:2049] [node 2] ICP30 unregister session Session# [3:7587241333666804445:2049] VirtualId# [3:7587241329371837140:0]
2025-12-24T02:11:23.130908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241333666804445:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.130955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.130984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.133099Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:23.133179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241335388198903:2049] [node 2] ICS01 socket: 140 reason# EndOfStream
2025-12-24T02:11:23.133190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241331093231514:2049] [node 2] ICP30 unregister session Session# [4:7587241335388198903:2049] VirtualId# [4:7587241335388198894:0]
2025-12-24T02:11:23.133196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241335388198903:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.133241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.133282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.133345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:23.133396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333178375517:2049] [node 2] ICS01 socket: 132 reason# EndOfStream
2025-12-24T02:11:23.133399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328883408121:2049] [node 2] ICP30 unregister session Session# [5:7587241333178375517:2049] VirtualId# [5:7587241333178375510:0]
2025-12-24T02:11:23.133403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333178375517:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.133423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.133436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.134071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:23.134145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241329755915343:2049] [node 2] ICS07 socket disconnect 90 reason# EndOfStream
2025-12-24T02:11:23.134149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241329755915343:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.134195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.134218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241329755915343:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:23.134576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241329755915269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3035) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.134588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241329755915343:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.134594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329755915269:2049] [node 2] ICP30 unregister session Session# [6:7587241329755915343:2049] VirtualId# [6:7587241329755915326:0]
2025-12-24T02:11:23.134627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.134632Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241329755915269:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3035) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.134634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3035) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.143967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241329755915269:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:23.148324Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:23.148430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241334050882657:2050] [node 3] ICS01 socket: 139 reason# EndOfStream
2025-12-24T02:11:23.148439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241329755915270:2050] [node 3] ICP30 unregister session Session# [6:7587241334050882657:2050] VirtualId# [6:7587241334050882655:0]
2025-12-24T02:11:23.148446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241334050882657:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.148497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.148522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.152266Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:23.152353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333178375511:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-12-24T02:11:23.152360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328883408122:2050] [node 3] ICP30 unregister session Session# [5:7587241333178375511:2050] VirtualId# [5:7587241333178375508:0]
2025-12-24T02:11:23.152366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333178375511:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.152409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.152432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.157576Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:23.157666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241328883408195:2102] [node 4] ICS01 socket: 86 reason# EndOfStream
2025-12-24T02:11:23.157674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241328883408176:2102] [node 4] ICP30 unregister session Session# [5:7587241328883408195:2102] VirtualId# [5:7587241328883408177:0]
2025-12-24T02:11:23.157681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241328883408195:2102] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.157718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.157743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:23.161090Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:23.161179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241334050882662:2117] [node 4] ICS01 socket: 141 reason# EndOfStream
2025-12-24T02:11:23.161188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241334050882649:2117] [node 4] ICP30 unregister session Session# [6:7587241334050882662:2117] VirtualId# [6:7587241334050882660:0]
2025-12-24T02:11:23.161194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241334050882662:2117] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.161250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.161278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:23.205785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241336453102644:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:23.206379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241336453102645:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.208224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241333879718316:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.208237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241333879718317:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.210078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241336136994906:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.210086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241336136994907:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.213932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333269070700:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.213940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333269070701:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.213945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333269070702:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.217219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241336055238179:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.217228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241336055238180:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.217236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241336055238181:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.212037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336163506391:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.212046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336163506392:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.212050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336163506393:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.234751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241336453102708:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.233625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241336055238241:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:23.236726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241336453102716:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:23.236904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241336453102718:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.235563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506452:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.238852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136994966:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.247182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333269070763:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.247800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718381:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.246949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241336136994971:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.251200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070760:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.256607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241336453102722:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-12-24T02:11:23.257757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333269070769:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.258178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241336453102723:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-12-24T02:11:23.258238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070770:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.258483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238248:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.258860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136994982:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.259430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241336055238251:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:23.267094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241333879718316:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28213935ns) {Configure:104(28213122ns) {ProcessConfigured:135(27802271ns) {StartInitialHandshake:157(27801957ns) {DropHandshakes:470(591ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(147ns)} GenerateSessionVirtualId:988(556ns) PrepareNewSessionHandshake:477(862ns) {SwitchToState:179(816ns)}}}}}
2025-12-24T02:11:23.267101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241333879718382:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:23.271220Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241336163506392:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27598256ns) {Configure:104(27597548ns) {ProcessConfigured:135(27247627ns) {StartInitialHandshake:157(27247040ns) {DropHandshakes:470(970ns) {DropIncomingHandshake:442(422ns) DropOutgoingHandshake:457(87ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(677ns) {SwitchToState:179(483ns)}}}}}
2025-12-24T02:11:23.272701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241333879718380:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:23.272954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241336136994973:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.274835Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241333269070700:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22881657ns) {Configure:104(22880115ns) {ProcessConfigured:135(22544527ns) {StartInitialHandshake:157(22543750ns) {DropHandshakes:470(845ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(820ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.276380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241336453102725:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.276696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336163506458:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.277452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241336055238181:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17948307ns) {Configure:104(17947691ns) {ProcessConfigured:135(17578807ns) {StartInitialHandshake:157(17578226ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(248ns) PrepareNewSessionHandshake:477(1185ns) {SwitchToState:179(550ns)}}}}}
2025-12-24T02:11:23.277595Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241333269070702:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17452963ns) {Configure:104(17452008ns) {ProcessConfigured:135(17201772ns) {StartInitialHandshake:157(17201333ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(894ns) {SwitchToState:179(403ns)}}}}}
2025-12-24T02:11:23.277371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336163506459:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.276516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718391:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.276540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718392:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.276565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718393:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.277025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241333879718386:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:23.274732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241336136994972:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:23.284838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070760:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.284913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102723:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.287967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241336453102718:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28169865ns) {Configure:104(28168300ns) {ProcessConfigured:135(27820382ns) {StartInitialHandshake:157(27819507ns) {DropHandshakes:470(707ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(202ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1682ns) {SwitchToState:179(1635ns)}}}}}
2025-12-24T02:11:23.288464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102722:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.287863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506466:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.289154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718397:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.289377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336055238248:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.289037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241336453102728:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-12-24T02:11:23.289424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241336055238241:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.289579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718393:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.289608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718391:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.290521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238259:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.290735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336055238248:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.291021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506466:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.289067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070781:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.289652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070772:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.289658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241333269070772:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.289695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070770:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.290492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333269070772:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.291398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241333269070769:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.291423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.291430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333269070769:2111] [node 6] ICP22 created new session: [5:7587241333269070785:2111]
2025-12-24T02:11:23.291438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333269070785:2111] [node 6] ICS09 handshake done sender: [5:7587241333269070772:2113] self: [5:7587241333269070771:0] peer: [6:7587241336055238258:0] socket: 114 qp: -1
2025-12-24T02:11:23.291446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333269070785:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:23.291453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.291198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136994986:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.294760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718381:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.294796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241333879718316:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.294876Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241333879718380:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21983006ns) {Configure:104(21982356ns) {ProcessConfigured:135(21610656ns) {StartInitialHandshake:157(21609589ns) {DropHandshakes:470(845ns) {DropIncomingHandshake:442(162ns) DropOutgoingHandshake:457(144ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(952ns) {SwitchToState:179(576ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.295694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718397:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.295723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718392:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.295803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070781:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.295842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333269070701:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.294954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241336136994973:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21403852ns) {Configure:104(21402539ns) {ProcessConfigured:135(21119630ns) {StartInitialHandshake:157(21119262ns) {DropHandshakes:470(463ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(1063ns) {SwitchToState:179(19ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.298846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102725:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.298889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102705:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.298904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241336453102705:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.299463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102728:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.299548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241336453102705:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.299745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506472:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.300043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506472:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.299745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241333879718403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54550
2025-12-24T02:11:23.299841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718394:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.299850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241333879718394:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.300854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241333879718381:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.299727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994978:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.299745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241336136994978:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.302281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994966:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.302321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241336136994906:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.303209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241336453102729:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21363
2025-12-24T02:11:23.304325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506452:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.304373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336163506391:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.305198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070788:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.305730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333269070702:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.305761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241333269070702:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16022161ns) {IncomingHandshake:261(16021899ns) {DropIncomingHandshake:442(119ns) PrepareNewSessionHandshake:477(407ns) {SwitchToState:179(217ns)} GenerateSessionVirtualId:988(225ns) IssueIncomingHandshakeReply:193(16017223ns) {SwitchToState:179(256ns)}}}
2025-12-24T02:11:23.305618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241336055238241:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.305646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.305658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241336055238241:2108] [node 5] ICP22 created new session: [6:7587241336055238261:2108]
2025-12-24T02:11:23.305665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336055238261:2108] [node 5] ICS09 handshake done sender: [6:7587241336055238248:2111] self: [6:7587241336055238258:0] peer: [5:7587241333269070771:0] socket: 115 qp: -1
2025-12-24T02:11:23.305673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336055238261:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:23.305702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.306770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070789:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.307241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506475:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.307367Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333269070781:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.307573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336163506452:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.307640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336163506391:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.307659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.307667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336163506391:2048] [node 1] ICP22 created new session: [4:7587241336163506476:2048]
2025-12-24T02:11:23.307673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241336163506476:2048] [node 1] ICS09 handshake done sender: [4:7587241336163506452:2108] self: [4:7587241336163506474:0] peer: [1:7587241336453102712:0] socket: 94 qp: -1
2025-12-24T02:11:23.307680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241336163506476:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.307686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.305454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241333879718386:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28303978ns) {Configure:104(28302744ns) {ProcessConfigured:135(28054175ns) {StartInitialHandshake:157(28053394ns) {DropHandshakes:470(244ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(130ns) PrepareNewSessionHandshake:477(858ns) {SwitchToState:179(500ns)}}}}}
2025-12-24T02:11:23.305867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241333879718316:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.305883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.305891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241333879718316:2048] [node 1] ICP22 created new session: [2:7587241333879718406:2048]
2025-12-24T02:11:23.305896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333879718406:2048] [node 1] ICS09 handshake done sender: [2:7587241333879718381:2110] self: [2:7587241333879718399:0] peer: [1:7587241336453102704:0] socket: 98 qp: -1
2025-12-24T02:11:23.305901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333879718406:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.305916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.307168Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241333879718394:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.307732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241333879718382:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.307742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.307746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241333879718382:2108] [node 5] ICP22 created new session: [2:7587241333879718409:2108]
2025-12-24T02:11:23.307750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333879718409:2108] [node 5] ICS09 handshake done sender: [2:7587241333879718394:2116] self: [2:7587241333879718389:0] peer: [5:7587241333269070787:0] socket: 130 qp: -1
2025-12-24T02:11:23.307754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333879718409:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:23.307760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.303771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102729:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.304564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102713:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.304570Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241336453102713:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.305510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241336453102713:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.305801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102710:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.305809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241336453102710:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.309360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241336453102708:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.309381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.309389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241336453102708:2108] [node 4] ICP22 created new session: [1:7587241336453102735:2108]
2025-12-24T02:11:23.309396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241336453102735:2108] [node 4] ICS09 handshake done sender: [1:7587241336453102713:2110] self: [1:7587241336453102712:0] peer: [4:7587241336163506474:0] socket: 93 qp: -1
2025-12-24T02:11:23.309403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241336453102735:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:23.309425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.309610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241336453102644:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.309625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.309631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241336453102644:2048] [node 2] ICP22 created new session: [1:7587241336453102737:2048]
2025-12-24T02:11:23.309638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241336453102737:2048] [node 2] ICS09 handshake done sender: [1:7587241336453102705:2107] self: [1:7587241336453102704:0] peer: [2:7587241333879718399:0] socket: 90 qp: -1
2025-12-24T02:11:23.309644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241336453102737:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:23.309651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.312904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333269070770:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.313061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070790:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.313139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333269070702:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.313152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.313159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333269070702:2050] [node 3] ICP22 created new session: [5:7587241333269070791:2050]
2025-12-24T02:11:23.313165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333269070791:2050] [node 3] ICS09 handshake done sender: [5:7587241333269070770:2112] self: [5:7587241333269070782:0] peer: [3:7587241336136994975:0] socket: 111 qp: -1
2025-12-24T02:11:23.313170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333269070791:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.313177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.312953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238263:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.313452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336055238263:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.313481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241336055238179:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.310527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241336136994978:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.310903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994982:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.311668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241336136994907:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.311809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994986:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.311838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241336136994971:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.311846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.311851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241336136994971:2109] [node 5] ICP22 created new session: [3:7587241336136994993:2109]
2025-12-24T02:11:23.311856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241336136994993:2109] [node 5] ICS09 handshake done sender: [3:7587241336136994978:2113] self: [3:7587241336136994975:0] peer: [5:7587241333269070782:0] socket: 107 qp: -1
2025-12-24T02:11:23.311861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241336136994993:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.311876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.314464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136994995:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.310869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241336453102710:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.310971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241336453102645:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.310987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.310992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241336453102645:2049] [node 3] ICP22 created new session: [1:7587241336453102742:2049]
2025-12-24T02:11:23.310998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241336453102742:2049] [node 3] ICS09 handshake done sender: [1:7587241336453102710:2109] self: [1:7587241336453102709:0] peer: [3:7587241336136994988:0] socket: 92 qp: -1
2025-12-24T02:11:23.311004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241336453102742:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.311020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.316392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102724:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.316406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241336453102724:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.315086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241336055238241:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(24155696ns) {HandleHandshakeStatus:329(24154959ns) {DropIncomingHandshake:442(877ns) DropOutgoingHandshake:457(3725ns) SwitchToState:179(98ns) ProcessPendingSessionEvents:495(6404851ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 6381936ns max# 6298605ns}}}
2025-12-24T02:11:23.315117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241336055238241:2108] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.318470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241336136994966:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.318512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070790:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.319053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333269070700:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.320481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238265:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.322739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241336136994906:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241336136994966:2107] held: no
2025-12-24T02:11:23.322830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241336136994906:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.322840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.322847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241336136994906:2048] [node 1] ICP22 created new session: [3:7587241336136994996:2048]
2025-12-24T02:11:23.322854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241336136994996:2048] [node 1] ICS09 handshake done sender: [3:7587241336136994966:2107] self: [3:7587241336136994988:0] peer: [1:7587241336453102709:0] socket: 99 qp: -1
2025-12-24T02:11:23.322862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241336136994996:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.322880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.319181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241336453102721:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.319192Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241336453102721:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.323607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070793:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.323133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241333879718316:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19452719ns) {HandleHandshakeStatus:329(19451661ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(5394ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(14417689ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14395309ns max# 14318816ns}}}
2025-12-24T02:11:23.323846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718384:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.323854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241333879718384:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.324010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506479:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.324117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506480:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.324224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506479:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.324256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336163506392:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.324339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718400:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.324346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241333879718400:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.325750Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238266:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.326184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336055238263:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.326319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241336055238179:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.326343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.326350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241336055238179:2048] [node 1] ICP22 created new session: [6:7587241336055238267:2048]
2025-12-24T02:11:23.326356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336055238267:2048] [node 1] ICS09 handshake done sender: [6:7587241336055238263:2117] self: [6:7587241336055238264:0] peer: [1:7587241336453102720:0] socket: 151 qp: -1
2025-12-24T02:11:23.326362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336055238267:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.326377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.328170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333269070793:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.328237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333269070763:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.328692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506469:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.328704Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241336163506469:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.329586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336163506469:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.329789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241336163506459:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.329809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.329816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336163506459:2109] [node 5] ICP22 created new session: [4:7587241336163506483:2109]
2025-12-24T02:11:23.329822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241336163506483:2109] [node 5] ICS09 handshake done sender: [4:7587241336163506469:2115] self: [4:7587241336163506468:0] peer: [5:7587241333269070795:0] socket: 118 qp: -1
2025-12-24T02:11:23.329832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241336163506483:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.329838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.330979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136994998:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.330967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333269070701:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.330992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.331001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333269070701:2049] [node 2] ICP22 created new session: [5:7587241333269070796:2049]
2025-12-24T02:11:23.331007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333269070796:2049] [node 2] ICS09 handshake done sender: [5:7587241333269070781:2117] self: [5:7587241333269070787:0] peer: [2:7587241333879718389:0] socket: 131 qp: -1
2025-12-24T02:11:23.331013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333269070796:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.331043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.331093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506480:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.331121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336163506393:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.331324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994985:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.331331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241336136994985:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.331500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241333269070763:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11332) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241333269070793:2121] held: no
2025-12-24T02:11:23.332320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241336136994985:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.325974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241336453102724:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.327072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241336453102718:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.327087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.327093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241336453102718:2112] [node 6] ICP22 created new session: [1:7587241336453102745:2112]
2025-12-24T02:11:23.327098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241336453102745:2112] [node 6] ICS09 handshake done sender: [1:7587241336453102724:2116] self: [1:7587241336453102720:0] peer: [6:7587241336055238264:0] socket: 145 qp: -1
2025-12-24T02:11:23.327104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241336453102745:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:23.327111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.332376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506487:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.332428Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241336453102721:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.332602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336163506480:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.332459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241336136994972:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.332475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.332481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241336136994972:2108] [node 4] ICP22 created new session: [3:7587241336136995000:2108]
2025-12-24T02:11:23.332486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241336136995000:2108] [node 4] ICS09 handshake done sender: [3:7587241336136994985:2115] self: [3:7587241336136994984:0] peer: [4:7587241336163506486:0] socket: 150 qp: -1
2025-12-24T02:11:23.332491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241336136995000:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:23.332497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.332698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241336453102716:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.332721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.332729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241336453102716:2111] [node 5] ICP22 created new session: [1:7587241336453102748:2111]
2025-12-24T02:11:23.332735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241336453102748:2111] [node 5] ICS09 handshake done sender: [1:7587241336453102721:2113] self: [1:7587241336453102719:0] peer: [5:7587241333269070794:0] socket: 144 qp: -1
2025-12-24T02:11:23.332742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241336453102748:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:23.332748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.333023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241333879718404:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.333033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241333879718404:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.332704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336163506393:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.332717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.332722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336163506393:2050] [node 3] ICP22 created new session: [4:7587241336163506488:2050]
2025-12-24T02:11:23.332726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241336163506488:2050] [node 3] ICS09 handshake done sender: [4:7587241336163506480:2120] self: [4:7587241336163506486:0] peer: [3:7587241336136994984:0] socket: 110 qp: -1
2025-12-24T02:11:23.332731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241336163506488:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.332738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.333647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241333879718404:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.333739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241333879718386:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.333755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.333762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241333879718386:2109] [node 6] ICP22 created new session: [2:7587241333879718414:2109]
2025-12-24T02:11:23.333766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333879718414:2109] [node 6] ICS09 handshake done sender: [2:7587241333879718404:2120] self: [2:7587241333879718396:0] peer: [6:7587241336055238270:0] socket: 149 qp: -1
2025-12-24T02:11:23.333773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333879718414:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:23.333789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.332882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336055238265:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.332913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241336055238180:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.336669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241336055238180:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241336055238265:2118] held: no
2025-12-24T02:11:23.337047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241333879718384:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.337226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241333879718317:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.337244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.337251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241333879718317:2049] [node 3] ICP22 created new session: [2:7587241333879718418:2049]
2025-12-24T02:11:23.337256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333879718418:2049] [node 3] ICS09 handshake done sender: [2:7587241333879718384:2111] self: [2:7587241333879718383:0] peer: [3:7587241336136994991:0] socket: 100 qp: -1
2025-12-24T02:11:23.337261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333879718418:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.337269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.337517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241333879718400:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.337668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241333879718380:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.337683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.337688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241333879718380:2107] [node 4] ICP22 created new session: [2:7587241333879718420:2107]
2025-12-24T02:11:23.337879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241336136994907:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241336136994982:2114] held: no
2025-12-24T02:11:23.337692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241333879718420:2107] [node 4] ICS09 handshake done sender: [2:7587241333879718400:2118] self: [2:7587241333879718390:0] peer: [4:7587241336163506481:0] socket: 117 qp: -1
2025-12-24T02:11:23.337696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241333879718420:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:23.337705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.339582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070798:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.339609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333269070799:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49756
2025-12-24T02:11:23.339810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333269070790:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.339897Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333269070793:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.340731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238271:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.339982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333269070700:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.339997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.340004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333269070700:2048] [node 1] ICP22 created new session: [5:7587241333269070800:2048]
2025-12-24T02:11:23.340009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333269070800:2048] [node 1] ICS09 handshake done sender: [5:7587241333269070790:2120] self: [5:7587241333269070794:0] peer: [1:7587241336453102719:0] socket: 147 qp: -1
2025-12-24T02:11:23.340014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333269070800:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.340022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.340405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333269070763:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.340412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.340417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333269070763:2109] [node 4] ICP22 created new session: [5:7587241333269070802:2109]
2025-12-24T02:11:23.340421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241333269070802:2109] [node 4] ICS09 handshake done sender: [5:7587241333269070793:2121] self: [5:7587241333269070795:0] peer: [4:7587241336163506468:0] socket: 121 qp: -1
2025-12-24T02:11:23.340425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241333269070802:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:23.340431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.342884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336055238271:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.342890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238272:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.342926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241336055238251:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.343102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336055238272:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.343163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241336055238181:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.343209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238273:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.343292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241336163506392:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241336163506479:2119] held: no
2025-12-24T02:11:23.343307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336055238265:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.347170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994998:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.347533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241336136995004:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16484
2025-12-24T02:11:23.347724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241336136994982:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.348098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336163506490:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17452
2025-12-24T02:11:23.347857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241336136994907:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.347875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.347881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241336136994907:2049] [node 2] ICP22 created new session: [3:7587241336136995005:2049]
2025-12-24T02:11:23.347887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241336136995005:2049] [node 2] ICS09 handshake done sender: [3:7587241336136994982:2114] self: [3:7587241336136994991:0] peer: [2:7587241333879718383:0] socket: 104 qp: -1
2025-12-24T02:11:23.347894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241336136995005:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.347907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.348513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336163506479:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.348634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336163506392:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.348658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.348667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336163506392:2049] [node 2] ICP22 created new session: [4:7587241336163506491:2049]
2025-12-24T02:11:23.348673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241336163506491:2049] [node 2] ICS09 handshake done sender: [4:7587241336163506479:2119] self: [4:7587241336163506481:0] peer: [2:7587241333879718390:0] socket: 153 qp: -1
2025-12-24T02:11:23.348680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241336163506491:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.348690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.349205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336163506470:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.349215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241336163506470:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.349989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241336136994987:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.350006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241336136994987:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.350392Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241333269070701:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(42812279ns) {HandleHandshakeStatus:329(42811628ns) {DropIncomingHandshake:442(558ns) DropOutgoingHandshake:457(3154ns) SwitchToState:179(165ns) ProcessPendingSessionEvents:495(7254088ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7232177ns max# 7154352ns}}}
2025-12-24T02:11:23.350436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241333269070701:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.355508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336163506470:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.355663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241336163506458:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.355686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.355699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336163506458:2110] [node 6] ICP22 created new session: [4:7587241336163506494:2110]
2025-12-24T02:11:23.355705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241336163506494:2110] [node 6] ICS09 handshake done sender: [4:7587241336163506470:2116] self: [4:7587241336163506467:0] peer: [6:7587241336055238274:0] socket: 157 qp: -1
2025-12-24T02:11:23.355712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241336163506494:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.355723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.357095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241336055238180:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.357128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.357138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241336055238180:2049] [node 2] ICP22 created new session: [6:7587241336055238277:2049]
2025-12-24T02:11:23.357147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336055238277:2049] [node 2] ICS09 handshake done sender: [6:7587241336055238265:2118] self: [6:7587241336055238270:0] peer: [2:7587241333879718396:0] socket: 152 qp: -1
2025-12-24T02:11:23.357156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336055238277:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.357168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.357275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238276:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.357325Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241336055238251:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11332) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241336055238271:2120] held: no
2025-12-24T02:11:23.357456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336055238271:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.357519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241336055238251:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.357531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.357541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241336055238251:2112] [node 4] ICP22 created new session: [6:7587241336055238279:2112]
2025-12-24T02:11:23.357548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336055238279:2112] [node 4] ICS09 handshake done sender: [6:7587241336055238271:2120] self: [6:7587241336055238274:0] peer: [4:7587241336163506467:0] socket: 158 qp: -1
2025-12-24T02:11:23.357554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336055238279:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:23.357577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.358171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336055238281:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34835
2025-12-24T02:11:23.358318Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241336136994987:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.358431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241336136994973:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.358447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.358452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241336136994973:2110] [node 6] ICP22 created new session: [3:7587241336136995008:2110]
2025-12-24T02:11:23.358457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241336136995008:2110] [node 6] ICS09 handshake done sender: [3:7587241336136994987:2117] self: [3:7587241336136994983:0] peer: [6:7587241336055238275:0] socket: 125 qp: -1
2025-12-24T02:11:23.358461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241336136995008:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.358466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.359141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336055238272:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.359224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241336055238181:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.359243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.359249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241336055238181:2050] [node 3] ICP22 created new session: [6:7587241336055238282:2050]
2025-12-24T02:11:23.359255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336055238282:2050] [node 3] ICS09 handshake done sender: [6:7587241336055238272:2121] self: [6:7587241336055238275:0] peer: [3:7587241336136994983:0] socket: 161 qp: -1
2025-12-24T02:11:23.359265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336055238282:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.359271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.372723Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:23.372825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241333879718406:2048] [node 1] ICS07 socket disconnect 98 reason# EndOfStream
2025-12-24T02:11:23.372831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241333879718406:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.372888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.372915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241333879718406:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.375183Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:23.375272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241336136994996:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-12-24T02:11:23.375281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241336136994906:2048] [node 1] ICP30 unregister session Session# [3:7587241336136994996:2048] VirtualId# [3:7587241336136994988:0]
2025-12-24T02:11:23.375289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241336136994996:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.375348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.375371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.377201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:23.377274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336055238267:2048] [node 1] ICS01 socket: 151 reason# EndOfStream
2025-12-24T02:11:23.377281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241336055238179:2048] [node 1] ICP30 unregister session Session# [6:7587241336055238267:2048] VirtualId# [6:7587241336055238264:0]
2025-12-24T02:11:23.377287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336055238267:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.377337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.377408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.378726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:23.378802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241336163506476:2048] [node 1] ICS07 socket disconnect 94 reason# EndOfStream
2025-12-24T02:11:23.378811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241336163506476:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.378858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.378884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241336163506476:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.378773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:23.378861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333269070800:2048] [node 1] ICS01 socket: 147 reason# EndOfStream
2025-12-24T02:11:23.378869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333269070700:2048] [node 1] ICP30 unregister session Session# [5:7587241333269070800:2048] VirtualId# [5:7587241333269070794:0]
2025-12-24T02:11:23.378875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333269070800:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.378946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.378981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.379333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241336163506391:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.379346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241336163506476:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.379351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241336163506391:2048] [node 1] ICP30 unregister session Session# [4:7587241336163506476:2048] VirtualId# [4:7587241336163506474:0]
2025-12-24T02:11:23.379439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.379450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241336163506391:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.379454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29523) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.389746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241336163506391:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:23.394468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:23.394574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241336163506491:2049] [node 2] ICS01 socket: 153 reason# EndOfStream
2025-12-24T02:11:23.394589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241336163506392:2049] [node 2] ICP30 unregister session Session# [4:7587241336163506491:2049] VirtualId# [4:7587241336163506481:0]
2025-12-24T02:11:23.394598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241336163506491:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.394711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:23.394682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.394761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.394767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:23.394847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241336136995005:2049] [node 2] ICS01 socket: 104 reason# EndOfStream
2025-12-24T02:11:23.394853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241336136994907:2049] [node 2] ICP30 unregister session Session# [3:7587241336136995005:2049] VirtualId# [3:7587241336136994991:0]
2025-12-24T02:11:23.394860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241336136995005:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.394920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.394961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.395075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:23.395147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336055238277:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-12-24T02:11:23.395153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241336055238180:2049] [node 2] ICP30 unregister session Session# [6:7587241336055238277:2049] VirtualId# [6:7587241336055238270:0]
2025-12-24T02:11:23.395158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336055238277:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.395195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.395218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.394847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241333269070796:2049] [node 2] ICS07 socket disconnect 131 reason# EndOfStream
2025-12-24T02:11:23.394857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333269070796:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.394907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.394933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241333269070796:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:23.395948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241333269070701:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.395961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333269070796:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.395966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333269070701:2049] [node 2] ICP30 unregister session Session# [5:7587241333269070796:2049] VirtualId# [5:7587241333269070787:0]
2025-12-24T02:11:23.396021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.396028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241333269070701:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.396031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5845) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.405985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241333269070701:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:23.412810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:23.412608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:23.412714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241336163506488:2050] [node 3] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:11:23.412722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241336163506393:2050] [node 3] ICP30 unregister session Session# [4:7587241336163506488:2050] VirtualId# [4:7587241336163506486:0]
2025-12-24T02:11:23.412731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241336163506488:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.412811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.412853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.412911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241333269070791:2050] [node 3] ICS07 socket disconnect 111 reason# EndOfStream
2025-12-24T02:11:23.412923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241333269070791:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.412935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:23.412994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.413023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241333269070791:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:23.413073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336055238282:2050] [node 3] ICS01 socket: 161 reason# EndOfStream
2025-12-24T02:11:23.413087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241336055238181:2050] [node 3] ICP30 unregister session Session# [6:7587241336055238282:2050] VirtualId# [6:7587241336055238275:0]
2025-12-24T02:11:23.413093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336055238282:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.413148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.413174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.413481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241333269070702:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25664) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.413500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241333269070791:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.413507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333269070702:2050] [node 3] ICP30 unregister session Session# [5:7587241333269070791:2050] VirtualId# [5:7587241333269070782:0]
2025-12-24T02:11:23.413559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.413567Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241333269070702:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.413570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25664) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.423377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241333269070702:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:23.435262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:23.435364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336055238279:2112] [node 4] ICS01 socket: 158 reason# EndOfStream
2025-12-24T02:11:23.435372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241336055238251:2112] [node 4] ICP30 unregister session Session# [6:7587241336055238279:2112] VirtualId# [6:7587241336055238274:0]
2025-12-24T02:11:23.435379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336055238279:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.435432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.435458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:23.444334Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:23.444432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241336055238261:2108] [node 5] ICS07 socket disconnect 115 reason# EndOfStream
2025-12-24T02:11:23.444444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336055238261:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.444498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:23.444527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241336055238261:2108] [node 5] ICS15 start handshake
2025-12-24T02:11:23.444801Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241336055238241:2108] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23746) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.444810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336055238261:2108] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.444815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241336055238241:2108] [node 5] ICP30 unregister session Session# [6:7587241336055238261:2108] VirtualId# [6:7587241336055238258:0]
2025-12-24T02:11:23.444864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:23.444871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241336055238241:2108] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.444874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23746) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:23.462028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241333830787868:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:23.462038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241333830787869:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.471473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241334649485977:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.471494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241334649485978:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.476328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241333975902927:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.476341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241333975902928:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.477803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241333884603089:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.477812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241333884603090:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.477815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241333884603091:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.485832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241335234159942:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.485842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241335234159943:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.485845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241335234159944:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.489312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241335513903383:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.489329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241335513903384:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.489334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241335513903385:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.502880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241335513903444:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.503533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241333830787930:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58373
2025-12-24T02:11:23.503576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486037:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.503683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160003:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.506394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241334649486041:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.507255Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603149:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.510896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975902990:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
2025-12-24T02:11:23.511089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241334649486051:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:23.511195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241334649486052:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:23.518293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241333975902996:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:23.518435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241333975902997:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:23.518562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241333975902998:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:23.523297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241335234160008:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:23.523294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241333830787935:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:23.523434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241335234160009:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.523457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241333830787936:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.523452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241335513903452:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:23.523617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241333830787937:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.525011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241333830787938:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58373
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.525325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241333830787947:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58373
2025-12-24T02:11:23.528372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787947:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.528559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160003:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.528839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241333830787951:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58373
2025-12-24T02:11:23.531712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787930:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.530792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241333884603089:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18273644ns) {Configure:104(18273070ns) {ProcessConfigured:135(17928599ns) {StartInitialHandshake:157(17928268ns) {DropHandshakes:470(981ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(21ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1324ns) {SwitchToState:179(256ns)}}}}}
2025-12-24T02:11:23.531168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241333884603155:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:23.531328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241333884603156:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.532019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903006:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.532085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903007:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.532632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486053:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.532702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486054:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.533130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787938:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.532741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486059:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
2025-12-24T02:11:23.532776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486053:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.533229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486037:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.533258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241334649485977:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.533386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787928:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.533399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241333830787928:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.533401Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975902990:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.533435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903006:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.533523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486059:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.534058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241333830787928:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.534130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486063:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
2025-12-24T02:11:23.534194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241333830787868:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.534215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.534223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241333830787868:2048] [node 2] ICP22 created new session: [1:7587241333830787955:2048]
2025-12-24T02:11:23.534228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241333830787955:2048] [node 2] ICS09 handshake done sender: [1:7587241333830787928:2106] self: [1:7587241333830787927:0] peer: [2:7587241334649486061:0] socket: 97 qp: -1
2025-12-24T02:11:23.534236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241333830787955:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:23.534244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.534351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241334649486037:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.534434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241334649485977:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.534452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.534465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241334649485977:2048] [node 1] ICP22 created new session: [2:7587241334649486064:2048]
2025-12-24T02:11:23.534471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241334649486064:2048] [node 1] ICS09 handshake done sender: [2:7587241334649486037:2107] self: [2:7587241334649486061:0] peer: [1:7587241333830787927:0] socket: 99 qp: -1
2025-12-24T02:11:23.534481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241334649486064:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.534490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.534815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241333975902928:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.534839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241333975902927:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.539103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787940:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.539115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241333830787940:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:23.540396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241333830787940:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.540456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903011:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
2025-12-24T02:11:23.540535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241333830787869:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.540559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.540571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241333830787869:2049] [node 3] ICP22 created new session: [1:7587241333830787959:2049]
2025-12-24T02:11:23.540583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241333830787959:2049] [node 3] ICS09 handshake done sender: [1:7587241333830787940:2113] self: [1:7587241333830787939:0] peer: [3:7587241333975903010:0] socket: 115 qp: -1
2025-12-24T02:11:23.540591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241333830787959:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.540599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.542969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241333975903006:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.543560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241333975902927:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.543582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.543592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241333975902927:2048] [node 1] ICP22 created new session: [3:7587241333975903012:2048]
2025-12-24T02:11:23.543604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333975903012:2048] [node 1] ICS09 handshake done sender: [3:7587241333975903006:2116] self: [3:7587241333975903010:0] peer: [1:7587241333830787939:0] socket: 123 qp: -1
2025-12-24T02:11:23.543613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333975903012:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.543624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.543741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486043:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.543758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241334649486043:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.544649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241334649486043:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.547592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603157:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.547697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603167:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.549531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241334649485978:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.549576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.549584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241334649485978:2049] [node 3] ICP22 created new session: [2:7587241334649486068:2049]
2025-12-24T02:11:23.549593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241334649486068:2049] [node 3] ICS09 handshake done sender: [2:7587241334649486043:2110] self: [2:7587241334649486042:0] peer: [3:7587241333975903009:0] socket: 106 qp: -1
2025-12-24T02:11:23.549601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241334649486068:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.549615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.550069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903461:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.552859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160022:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.553024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160023:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.554946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241333975902927:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.555320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241333830787962:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58373
2025-12-24T02:11:23.555355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787951:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.555440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241335513903461:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.555470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241335513903383:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.555317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160022:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.555356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241335234159942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.555831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787944:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.555839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241333830787944:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.556192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787946:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.556195Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241333830787946:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:23.556957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
::ffff:127.1.0.0
2025-12-24T02:11:23.557014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.557037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241333830787944:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.557305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241333830787946:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.557456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241333830787936:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.557496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.557483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241335513903461:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.557511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241333830787936:2109] [node 5] ICP22 created new session: [1:7587241333830787966:2109]
2025-12-24T02:11:23.557522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241333830787966:2109] [node 5] ICS09 handshake done sender: [1:7587241333830787944:2115] self: [1:7587241333830787943:0] peer: [5:7587241335234160024:0] socket: 111 qp: -1
2025-12-24T02:11:23.557530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241333830787966:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.557541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.557712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241335513903383:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.557729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.557735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241335513903383:2048] [node 1] ICP22 created new session: [6:7587241335513903464:2048]
2025-12-24T02:11:23.557741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241335513903464:2048] [node 1] ICS09 handshake done sender: [6:7587241335513903461:2115] self: [6:7587241335513903462:0] peer: [1:7587241333830787945:0] socket: 136 qp: -1
2025-12-24T02:11:23.557748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241335513903464:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.557755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.560939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486054:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.562841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603149:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.562887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603157:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.562913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603167:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.566458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241333884603090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.566487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241333884603089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.566502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241333884603091:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.566712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241334649486070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36116
2025-12-24T02:11:23.566776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241335234160022:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.566866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241335234159942:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.566885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.566890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241335234159942:2048] [node 1] ICP22 created new session: [5:7587241335234160026:2048]
2025-12-24T02:11:23.566896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241335234160026:2048] [node 1] ICS09 handshake done sender: [5:7587241335234160022:2116] self: [5:7587241335234160024:0] peer: [1:7587241333830787943:0] socket: 134 qp: -1
2025-12-24T02:11:23.566902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241335234160026:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.566910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.566872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486045:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.566884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241334649486045:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.567002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241333830787937:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.567034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.567049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241333830787937:2110] [node 6] ICP22 created new session: [1:7587241333830787968:2110]
2025-12-24T02:11:23.567057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241333830787968:2110] [node 6] ICS09 handshake done sender: [1:7587241333830787946:2116] self: [1:7587241333830787945:0] peer: [6:7587241335513903462:0] socket: 135 qp: -1
2025-12-24T02:11:23.567063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241333830787968:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.567077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.567114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903467:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.567587Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486070:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.567838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787942:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.567848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241333830787942:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.568833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241334649486045:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.568730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241333830787962:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.569192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241333830787942:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.569460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160023:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.569561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241334649486041:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.569604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.569620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241334649486041:2108] [node 4] ICP22 created new session: [2:7587241334649486075:2108]
2025-12-24T02:11:23.569616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241333830787935:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.569631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.569636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241333830787935:2108] [node 4] ICP22 created new session: [1:7587241333830787972:2108]
2025-12-24T02:11:23.569640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241333830787972:2108] [node 4] ICS09 handshake done sender: [1:7587241333830787942:2114] self: [1:7587241333830787941:0] peer: [4:7587241333884603171:0] socket: 116 qp: -1
2025-12-24T02:11:23.569646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241333830787972:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:23.569651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.569627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241334649486075:2108] [node 4] ICS09 handshake done sender: [2:7587241334649486045:2111] self: [2:7587241334649486044:0] peer: [4:7587241333884603170:0] socket: 107 qp: -1
2025-12-24T02:11:23.569634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241334649486075:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:23.569646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.569746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603174:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.569612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241333975902928:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241333975902990:2108] held: no
2025-12-24T02:11:23.569762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903000:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.569767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241333975903000:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.570412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903007:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.570478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903014:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
2025-12-24T02:11:23.570554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241335513903467:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.570594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241335513903384:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.570989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486058:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.571005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241334649486058:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.571026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903015:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
2025-12-24T02:11:23.571498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241334649486058:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.571511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241333975902990:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.572139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241334649486052:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.572159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.572176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241334649486052:2113] [node 6] ICP22 created new session: [2:7587241334649486078:2113]
2025-12-24T02:11:23.572188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241334649486078:2113] [node 6] ICS09 handshake done sender: [2:7587241334649486058:2117] self: [2:7587241334649486057:0] peer: [6:7587241335513903468:0] socket: 149 qp: -1
2025-12-24T02:11:23.572195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241334649486078:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:23.572204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.572256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241333884603157:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.572259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241333975903016:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26978
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.572334Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241333884603089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241333884603157:2112] held: no
2025-12-24T02:11:23.572337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903469:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.572396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241333975903000:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.572435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241333884603089:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.572449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.572456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241333884603089:2048] [node 1] ICP22 created new session: [4:7587241333884603175:2048]
2025-12-24T02:11:23.572462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241333884603175:2048] [node 1] ICS09 handshake done sender: [4:7587241333884603157:2112] self: [4:7587241333884603171:0] peer: [1:7587241333830787941:0] socket: 122 qp: -1
2025-12-24T02:11:23.572468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241333884603175:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.572475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.572800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241333975902928:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.572820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.572826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241333975902928:2049] [node 2] ICP22 created new session: [3:7587241333975903020:2049]
2025-12-24T02:11:23.572832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333975903020:2049] [node 2] ICS09 handshake done sender: [3:7587241333975902990:2108] self: [3:7587241333975903009:0] peer: [2:7587241334649486042:0] socket: 110 qp: -1
2025-12-24T02:11:23.572838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333975903020:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.572843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.572827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241335513903467:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.572898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241333884603090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5261) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241333884603149:2107] held: no
2025-12-24T02:11:23.573089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241333975902996:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.573102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.573107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241333975902996:2109] [node 4] ICP22 created new session: [3:7587241333975903022:2109]
2025-12-24T02:11:23.573111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333975903022:2109] [node 4] ICS09 handshake done sender: [3:7587241333975903000:2113] self: [3:7587241333975902999:0] peer: [4:7587241333884603172:0] socket: 128 qp: -1
2025-12-24T02:11:23.573116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333975903022:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:23.573121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.573541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603173:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.573801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903016:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.573849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903015:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.573877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241335513903384:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.573898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.573905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241335513903384:2049] [node 2] ICP22 created new session: [6:7587241335513903471:2049]
2025-12-24T02:11:23.573912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241335513903471:2049] [node 2] ICS09 handshake done sender: [6:7587241335513903467:2117] self: [6:7587241335513903468:0] peer: [2:7587241334649486057:0] socket: 150 qp: -1
2025-12-24T02:11:23.573920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241335513903471:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.573932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.574112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241335513903384:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.574407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241333884603149:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.574494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241333884603090:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.574508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.574515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241333884603090:2049] [node 2] ICP22 created new session: [4:7587241333884603180:2049]
2025-12-24T02:11:23.574521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241333884603180:2049] [node 2] ICS09 handshake done sender: [4:7587241333884603149:2107] self: [4:7587241333884603170:0] peer: [2:7587241334649486044:0] socket: 108 qp: -1
2025-12-24T02:11:23.574526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241333884603180:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.574532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.574729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903473:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.574908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241335234159943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.574987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160028:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.574998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160029:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.576459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903470:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.576640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603177:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.576879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241333884603167:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.576967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241333884603091:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.576984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.576990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241333884603091:2050] [node 3] ICP22 created new session: [4:7587241333884603184:2050]
2025-12-24T02:11:23.576996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241333884603184:2050] [node 3] ICS09 handshake done sender: [4:7587241333884603167:2117] self: [4:7587241333884603172:0] peer: [3:7587241333975902999:0] socket: 137 qp: -1
2025-12-24T02:11:23.577002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241333884603184:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.577013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.579682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603182:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.579738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241333884603183:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48172
2025-12-24T02:11:23.579836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241335513903473:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.579877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241335513903452:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.579991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603182:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.580025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603165:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.580035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241333884603165:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.580529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241333884603165:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.580631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241333884603156:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.580651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.580661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241333884603156:2110] [node 6] ICP22 created new session: [4:7587241333884603188:2110]
2025-12-24T02:11:23.580666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241333884603188:2110] [node 6] ICS09 handshake done sender: [4:7587241333884603165:2116] self: [4:7587241333884603164:0] peer: [6:7587241335513903475:0] socket: 154 qp: -1
2025-12-24T02:11:23.580671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241333884603188:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.580676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.580730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903476:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.580902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241335513903473:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.580964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241335513903452:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.581007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.581019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241335513903452:2110] [node 4] ICP22 created new session: [6:7587241335513903477:2110]
2025-12-24T02:11:23.581025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241335513903477:2110] [node 4] ICS09 handshake done sender: [6:7587241335513903473:2120] self: [6:7587241335513903475:0] peer: [4:7587241333884603164:0] socket: 155 qp: -1
2025-12-24T02:11:23.581031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241335513903477:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:23.581040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.582882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241334649486056:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.582905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241334649486056:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.583662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241333975902996:2109] [node 4] ICP18 handshake (actor [3:7587241333975903015:2120]) is too old
2025-12-24T02:11:23.583692Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241334649486056:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.583881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160032:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.584048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241335234160023:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.584293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241335513903470:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.584326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241335513903385:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.584406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903004:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.584418Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241333975903004:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.584451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603183:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.584149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241335234159943:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.584164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.584169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241335234159943:2049] [node 2] ICP22 created new session: [5:7587241335234160033:2049]
2025-12-24T02:11:23.584175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241335234160033:2049] [node 2] ICS09 handshake done sender: [5:7587241335234160023:2117] self: [5:7587241335234160031:0] peer: [2:7587241334649486055:0] socket: 142 qp: -1
2025-12-24T02:11:23.584180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241335234160033:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.584186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.584426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160028:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.584444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160029:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.584461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241335234159944:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.584471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241335234160008:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.585035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241333975903002:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.585046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241333975903002:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.585276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241334649486051:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.585305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.585314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241334649486051:2112] [node 5] ICP22 created new session: [2:7587241334649486081:2112]
2025-12-24T02:11:23.585325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241334649486081:2112] [node 5] ICS09 handshake done sender: [2:7587241334649486056:2116] self: [2:7587241334649486055:0] peer: [5:7587241335234160031:0] socket: 105 qp: -1
2025-12-24T02:11:23.585338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241334649486081:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:23.585348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.585420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241333884603163:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.585434Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241333884603163:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.586176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241333975903004:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.586609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903480:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.586288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241333975902998:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.586308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.586319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241333975902998:2111] [node 6] ICP22 created new session: [3:7587241333975903028:2111]
2025-12-24T02:11:23.586325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333975903028:2111] [node 6] ICS09 handshake done sender: [3:7587241333975903004:2115] self: [3:7587241333975903003:0] peer: [6:7587241335513903479:0] socket: 139 qp: -1
2025-12-24T02:11:23.586331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333975903028:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:23.586337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.587108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160037:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.587226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241333975903002:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.587383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241333975902997:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.587403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.587410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241333975902997:2110] [node 5] ICP22 created new session: [3:7587241333975903030:2110]
2025-12-24T02:11:23.587415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241333975903030:2110] [node 5] ICS09 handshake done sender: [3:7587241333975903002:2114] self: [3:7587241333975903001:0] peer: [5:7587241335234160035:0] socket: 96 qp: -1
2025-12-24T02:11:23.587422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241333975903030:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:23.587430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.587605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241333884603163:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.588093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241335234160028:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.588221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241335513903470:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.588247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241333884603155:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.588276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.588283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241333884603155:2108] [node 5] ICP22 created new session: [4:7587241333884603192:2108]
2025-12-24T02:11:23.588290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241333884603192:2108] [node 5] ICS09 handshake done sender: [4:7587241333884603163:2115] self: [4:7587241333884603162:0] peer: [5:7587241335234160036:0] socket: 153 qp: -1
2025-12-24T02:11:23.588301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241333884603192:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:23.588311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.588277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241335234159944:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.588298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.588306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241335234159944:2050] [node 3] ICP22 created new session: [5:7587241335234160039:2050]
2025-12-24T02:11:23.588314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241335234160039:2050] [node 3] ICS09 handshake done sender: [5:7587241335234160028:2119] self: [5:7587241335234160035:0] peer: [3:7587241333975903001:0] socket: 158 qp: -1
2025-12-24T02:11:23.588322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241335234160039:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.588329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.588365Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241335513903385:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25193) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241335513903470:2119] held: no
2025-12-24T02:11:23.588462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241335513903385:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.588470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.588475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241335513903385:2050] [node 3] ICP22 created new session: [6:7587241335513903482:2050]
2025-12-24T02:11:23.588481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241335513903482:2050] [node 3] ICS09 handshake done sender: [6:7587241335513903470:2119] self: [6:7587241335513903479:0] peer: [3:7587241333975903003:0] socket: 112 qp: -1
2025-12-24T02:11:23.588487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241335513903482:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.588493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.589040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903481:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.590211Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241335234160008:2109] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11452) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241335234160029:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.592053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241335234160041:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29460
2025-12-24T02:11:23.592332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241335234160029:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.592380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241335513903481:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.592448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241335513903444:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.592418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241335234160008:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.592432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.592438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241335234160008:2109] [node 4] ICP22 created new session: [5:7587241335234160042:2109]
2025-12-24T02:11:23.592444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241335234160042:2109] [node 4] ICS09 handshake done sender: [5:7587241335234160029:2120] self: [5:7587241335234160036:0] peer: [4:7587241333884603162:0] socket: 159 qp: -1
2025-12-24T02:11:23.592451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241335234160042:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:23.592458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.592857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241335234160017:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.592864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241335234160017:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.593379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241335234160017:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.593477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241335513903485:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14621
2025-12-24T02:11:23.593679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241335513903481:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.593749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241335513903444:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.593766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.593772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241335513903444:2107] [node 5] ICP22 created new session: [6:7587241335513903486:2107]
2025-12-24T02:11:23.593779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241335513903486:2107] [node 5] ICS09 handshake done sender: [6:7587241335513903481:2123] self: [6:7587241335513903484:0] peer: [5:7587241335234160016:0] socket: 166 qp: -1
2025-12-24T02:11:23.593784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241335513903486:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:23.593792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.593468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241335234160009:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.593485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.593490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241335234160009:2110] [node 6] ICP22 created new session: [5:7587241335234160045:2110]
2025-12-24T02:11:23.593496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241335234160045:2110] [node 6] ICS09 handshake done sender: [5:7587241335234160017:2115] self: [5:7587241335234160016:0] peer: [6:7587241335513903484:0] socket: 165 qp: -1
2025-12-24T02:11:23.593501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241335234160045:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.593509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.600269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:23.600381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241334649486064:2048] [node 1] ICS01 socket: 99 reason# EndOfStream
2025-12-24T02:11:23.600395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241334649485977:2048] [node 1] ICP30 unregister session Session# [2:7587241334649486064:2048] VirtualId# [2:7587241334649486061:0]
2025-12-24T02:11:23.600403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241334649486064:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.600467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.600494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.603853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:23.603936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241333884603175:2048] [node 1] ICS01 socket: 122 reason# EndOfStream
2025-12-24T02:11:23.603943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241333884603089:2048] [node 1] ICP30 unregister session Session# [4:7587241333884603175:2048] VirtualId# [4:7587241333884603171:0]
2025-12-24T02:11:23.603948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241333884603175:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.603993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.604028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.604218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:23.604302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241335513903464:2048] [node 1] ICS01 socket: 136 reason# EndOfStream
2025-12-24T02:11:23.604313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241335513903383:2048] [node 1] ICP30 unregister session Session# [6:7587241335513903464:2048] VirtualId# [6:7587241335513903462:0]
2025-12-24T02:11:23.604319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241335513903464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.604368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.604391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.602976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:23.603091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241333975903012:2048] [node 1] ICS01 socket: 123 reason# EndOfStream
2025-12-24T02:11:23.603101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241333975902927:2048] [node 1] ICP30 unregister session Session# [3:7587241333975903012:2048] VirtualId# [3:7587241333975903010:0]
2025-12-24T02:11:23.603108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241333975903012:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.603165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.603193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.610796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:23.610908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241335234160026:2048] [node 1] ICS07 socket disconnect 134 reason# EndOfStream
2025-12-24T02:11:23.610919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241335234160026:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.610978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.611000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241335234160026:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.613816Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:23.613947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241333975903020:2049] [node 2] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:11:23.613961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241333975902928:2049] [node 2] ICP30 unregister session Session# [3:7587241333975903020:2049] VirtualId# [3:7587241333975903009:0]
2025-12-24T02:11:23.613968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241333975903020:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.614011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.614045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.615040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:23.615135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241333884603180:2049] [node 2] ICS01 socket: 108 reason# EndOfStream
2025-12-24T02:11:23.615146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241333884603090:2049] [node 2] ICP30 unregister session Session# [4:7587241333884603180:2049] VirtualId# [4:7587241333884603170:0]
2025-12-24T02:11:23.615151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241333884603180:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.615189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.615216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.614897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241335234159942:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1508) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.614924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241335234160026:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.614928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241335234159942:2048] [node 1] ICP30 unregister session Session# [5:7587241335234160026:2048] VirtualId# [5:7587241335234160024:0]
2025-12-24T02:11:23.614968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.614973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241335234159942:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.614976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.615377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:23.615461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241335513903471:2049] [node 2] ICS01 socket: 150 reason# EndOfStream
2025-12-24T02:11:23.615472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241335513903384:2049] [node 2] ICP30 unregister session Session# [6:7587241335513903471:2049] VirtualId# [6:7587241335513903468:0]
2025-12-24T02:11:23.615494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241335513903471:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.615539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.615585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.615598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:23.615662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241335234160033:2049] [node 2] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:11:23.615666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241335234159943:2049] [node 2] ICP30 unregister session Session# [5:7587241335234160033:2049] VirtualId# [5:7587241335234160031:0]
2025-12-24T02:11:23.615671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241335234160033:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.615700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.615719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.625145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241335234159942:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:23.626898Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:23.627077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241335513903482:2050] [node 3] ICS01 socket: 112 reason# EndOfStream
2025-12-24T02:11:23.627091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241335513903385:2050] [node 3] ICP30 unregister session Session# [6:7587241335513903482:2050] VirtualId# [6:7587241335513903479:0]
2025-12-24T02:11:23.627098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241335513903482:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.627084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:23.627158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.627172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241333884603184:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-12-24T02:11:23.627181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241333884603091:2050] [node 3] ICP30 unregister session Session# [4:7587241333884603184:2050] VirtualId# [4:7587241333884603172:0]
2025-12-24T02:11:23.627189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241333884603184:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.627235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.627264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.627232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.627290Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:23.627383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241335234160039:2050] [node 3] ICS07 socket disconnect 158 reason# EndOfStream
2025-12-24T02:11:23.627387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241335234160039:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.627409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.627427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241335234160039:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:23.627984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241335234159944:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25193) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.627996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241335234160039:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.628000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241335234159944:2050] [node 3] ICP30 unregister session Session# [5:7587241335234160039:2050] VirtualId# [5:7587241335234160035:0]
2025-12-24T02:11:23.628034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.628039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241335234159944:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.628041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25193) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.641318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241335234159944:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:23.647103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:23.647215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241335513903477:2110] [node 4] ICS07 socket disconnect 155 reason# EndOfStream
2025-12-24T02:11:23.647222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241335513903477:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.647295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.647327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241335513903477:2110] [node 4] ICS15 start handshake
2025-12-24T02:11:23.647789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241335513903452:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11452) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.647805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241335513903477:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.647811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241335513903452:2110] [node 4] ICP30 unregister session Session# [6:7587241335513903477:2110] VirtualId# [6:7587241335513903475:0]
2025-12-24T02:11:23.647856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:23.647862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241335513903452:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.647865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11452) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:23.666905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332806705850:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:23.666920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332806705851:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.668254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335932440138:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.668268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335932440139:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.669769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241334894984859:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.669782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241334894984860:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.669787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241334894984861:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.666621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335705104262:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.666637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335705104263:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.671469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241336583027324:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.671496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241336583027325:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.671501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241336583027326:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.674519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241333967460424:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.674543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241333967460425:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.674549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241333967460426:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.685300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241333967460486:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.686298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027385:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.687004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332806705908:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:23.687128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332806705909:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.689407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984920:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.689192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241336583027385:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.689359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241336583027387:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:23.689805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241336583027387:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.688910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440182:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
2025-12-24T02:11:23.689019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104319:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.689117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335932440185:2093] [node 5] ICP01 ready to work
2025-12-24T02:11:23.690858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241333967460488:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.690874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241333967460488:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.691515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241333967460488:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.691624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241333967460486:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.691642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.691649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241333967460486:2108] [node 5] ICP22 created new session: [6:7587241333967460491:2108]
2025-12-24T02:11:23.691656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241333967460491:2108] [node 5] ICS09 handshake done sender: [6:7587241333967460488:2109] self: [6:7587241333967460487:0] peer: [5:7587241336583027388:0] socket: 102 qp: -1
2025-12-24T02:11:23.691663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241333967460491:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:23.691674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.689460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332806705922:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:23.698813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335705104332:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:23.698986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335705104333:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.699659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335932440196:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:23.699957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335932440204:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:23.702729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335705104330:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.707143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241334894984926:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.708536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440206:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
2025-12-24T02:11:23.714832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241334894984924:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.720725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241335705104332:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21595479ns) {Configure:104(21594819ns) {ProcessConfigured:135(21262399ns) {StartInitialHandshake:157(21261355ns) {DropHandshakes:470(564ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(1167ns) {SwitchToState:179(407ns)}}}}}
2025-12-24T02:11:23.720533Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332806705927:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51824
2025-12-24T02:11:23.720581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332806705930:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51824
2025-12-24T02:11:23.720606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332806705931:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51824
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.722845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241335705104333:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19988121ns) {Configure:104(19987358ns) {ProcessConfigured:135(19693481ns) {StartInitialHandshake:157(19692720ns) {DropHandshakes:470(877ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(1070ns) {SwitchToState:179(453ns)}}}}}
2025-12-24T02:11:23.723039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104338:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.723052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104339:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.723064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104340:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.724863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104319:2104] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.724889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241335705104262:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.723646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241333967460502:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:23.724744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705931:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.727273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104339:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.727563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104340:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.727469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705911:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.727495Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332806705911:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.727602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241336583027393:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:23.730873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440182:2091] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.730930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241335932440138:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.732302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440220:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
2025-12-24T02:11:23.731081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705913:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.731095Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332806705913:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.732116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332806705913:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.732276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332806705851:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.732292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.732303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332806705851:2049] [node 3] ICP22 created new session: [1:7587241332806705934:2049]
2025-12-24T02:11:23.732311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241332806705934:2049] [node 3] ICS09 handshake done sender: [1:7587241332806705913:2106] self: [1:7587241332806705912:0] peer: [3:7587241335932440219:0] socket: 106 qp: -1
2025-12-24T02:11:23.732320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241332806705934:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.732330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.734867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241336583027324:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38749481ns) {Configure:104(38748270ns) {ProcessConfigured:135(38500226ns) {StartInitialHandshake:157(38499614ns) {DropHandshakes:470(462ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(148ns) PrepareNewSessionHandshake:477(1897ns) {SwitchToState:179(1121ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:23.736028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.736978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460506:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.737206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984920:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.737240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984937:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.737263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241334894984859:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.736406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332806705911:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.736539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332806705850:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.736567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.736574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332806705850:2048] [node 2] ICP22 created new session: [1:7587241332806705938:2048]
2025-12-24T02:11:23.736582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241332806705938:2048] [node 2] ICS09 handshake done sender: [1:7587241332806705911:2105] self: [1:7587241332806705910:0] peer: [2:7587241335705104345:0] socket: 105 qp: -1
2025-12-24T02:11:23.736591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241332806705938:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:23.736600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.738314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705915:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.738320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332806705915:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.738809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705930:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.738958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027391:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.738744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440221:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
2025-12-24T02:11:23.738867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335932440182:2091] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.739281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104338:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.739091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984936:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.739323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984941:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.739448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241333967460506:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.739502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241333967460424:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.739846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027397:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.740591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027392:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.740580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440220:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.740645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241335932440138:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.740666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.740674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335932440138:2048] [node 1] ICP22 created new session: [3:7587241335932440224:2048]
2025-12-24T02:11:23.740686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241335932440224:2048] [node 1] ICS09 handshake done sender: [3:7587241335932440182:2091] self: [3:7587241335932440219:0] peer: [1:7587241332806705912:0] socket: 109 qp: -1
2025-12-24T02:11:23.740692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241335932440224:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.740700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.741508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241334894984920:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.741641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241334894984859:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241334894984920:2108] held: no
2025-12-24T02:11:23.741748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241334894984859:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.741784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.741792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241334894984859:2048] [node 1] ICP22 created new session: [4:7587241334894984942:2048]
2025-12-24T02:11:23.741799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241334894984942:2048] [node 1] ICS09 handshake done sender: [4:7587241334894984920:2108] self: [4:7587241334894984940:0] peer: [1:7587241332806705914:0] socket: 110 qp: -1
2025-12-24T02:11:23.741813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241334894984942:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.741821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.741861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984936:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.741942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241334894984861:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.742209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440210:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.742170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.742223Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241335932440210:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.742245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241336583027397:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.742275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241336583027385:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.739266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332806705915:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.739446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332806705908:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.739462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.739469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332806705908:2102] [node 4] ICP22 created new session: [1:7587241332806705943:2102]
2025-12-24T02:11:23.739474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241332806705943:2102] [node 4] ICS09 handshake done sender: [1:7587241332806705915:2107] self: [1:7587241332806705914:0] peer: [4:7587241334894984940:0] socket: 107 qp: -1
2025-12-24T02:11:23.739481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241332806705943:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:23.739510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.739818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705924:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.739822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332806705924:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.740601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332806705945:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51824
2025-12-24T02:11:23.740623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332806705924:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.740733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332806705922:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.740754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.740761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332806705922:2109] [node 6] ICP22 created new session: [1:7587241332806705947:2109]
2025-12-24T02:11:23.740767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241332806705947:2109] [node 6] ICS09 handshake done sender: [1:7587241332806705924:2110] self: [1:7587241332806705923:0] peer: [6:7587241333967460507:0] socket: 147 qp: -1
2025-12-24T02:11:23.740780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241332806705947:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:23.740793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.742757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440221:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.742905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241336583027387:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.742942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.742950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241336583027387:2109] [node 6] ICP22 created new session: [5:7587241336583027405:2109]
2025-12-24T02:11:23.742963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241336583027405:2109] [node 6] ICS09 handshake done sender: [5:7587241336583027385:2108] self: [5:7587241336583027388:0] peer: [6:7587241333967460487:0] socket: 103 qp: -1
2025-12-24T02:11:23.742971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241336583027405:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:23.742990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.742985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335932440210:2112] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.743253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440206:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.743276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335932440204:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.743290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.743297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335932440204:2102] [node 4] ICP22 created new session: [3:7587241335932440230:2102]
2025-12-24T02:11:23.743156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241333967460424:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241333967460506:2115] held: no
2025-12-24T02:11:23.743303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241335932440230:2102] [node 4] ICS09 handshake done sender: [3:7587241335932440210:2112] self: [3:7587241335932440209:0] peer: [4:7587241334894984944:0] socket: 126 qp: -1
2025-12-24T02:11:23.743310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241335932440230:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:23.743571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335705104349:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1318
2025-12-24T02:11:23.743317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.743606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335705104319:2104] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.743556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440223:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.743614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460509:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.744351Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984946:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.745232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241335932440139:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.744513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241334894984936:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.744564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241334894984861:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.744575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.744581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241334894984861:2050] [node 3] ICP22 created new session: [4:7587241334894984947:2050]
2025-12-24T02:11:23.744588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241334894984947:2050] [node 3] ICS09 handshake done sender: [4:7587241334894984936:2115] self: [4:7587241334894984944:0] peer: [3:7587241335932440209:0] socket: 127 qp: -1
2025-12-24T02:11:23.744594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241334894984947:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.744600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.743032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705927:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.743061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241332806705850:2048] [node 2] ICP18 handshake (actor [1:7587241332806705927:2113]) is too old
2025-12-24T02:11:23.747463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460510:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
2025-12-24T02:11:23.747773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241336583027391:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.747812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241333967460509:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.747806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241336583027324:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.747893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460508:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.748146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705917:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.748160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332806705917:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.748125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241333967460510:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.748174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241333967460506:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.748417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241333967460502:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.748445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241333967460426:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.748598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241333967460424:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.748614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.748621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241333967460424:2048] [node 1] ICP22 created new session: [6:7587241333967460514:2048]
2025-12-24T02:11:23.748627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241333967460514:2048] [node 1] ICS09 handshake done sender: [6:7587241333967460506:2115] self: [6:7587241333967460507:0] peer: [1:7587241332806705923:0] socket: 148 qp: -1
2025-12-24T02:11:23.748633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241333967460514:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.748644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.748198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241335705104262:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.748224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.748232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335705104262:2048] [node 1] ICP22 created new session: [2:7587241335705104351:2048]
2025-12-24T02:11:23.748238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241335705104351:2048] [node 1] ICS09 handshake done sender: [2:7587241335705104319:2104] self: [2:7587241335705104345:0] peer: [1:7587241332806705910:0] socket: 108 qp: -1
2025-12-24T02:11:23.748244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241335705104351:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.748264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.748153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241336583027326:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.748404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027408:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.749038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332806705917:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.749171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332806705909:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.749185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.749192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332806705909:2103] [node 5] ICP22 created new session: [1:7587241332806705950:2103]
2025-12-24T02:11:23.749206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241332806705950:2103] [node 5] ICS09 handshake done sender: [1:7587241332806705917:2108] self: [1:7587241332806705916:0] peer: [5:7587241336583027407:0] socket: 141 qp: -1
2025-12-24T02:11:23.749212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241332806705950:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:23.749220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.749729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104324:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.749746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335705104324:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.749720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984932:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.749736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241334894984932:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.749817Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241335705104262:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.749720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440187:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.749732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440212:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.749743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241335932440212:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.749743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241335932440187:2094] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.750449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984937:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:23.750486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241334894984860:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.750586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335932440212:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.751785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335932440196:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.751806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.751813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335932440196:2096] [node 6] ICP22 created new session: [3:7587241335932440234:2096]
2025-12-24T02:11:23.751820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241335932440234:2096] [node 6] ICS09 handshake done sender: [3:7587241335932440212:2113] self: [3:7587241335932440211:0] peer: [6:7587241333967460513:0] socket: 158 qp: -1
2025-12-24T02:11:23.751827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241335932440234:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:23.751835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.752743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984950:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.752827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241334894984932:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.752859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027410:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.753106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241334894984926:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.753121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.753129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241334894984926:2110] [node 6] ICP22 created new session: [4:7587241334894984953:2110]
2025-12-24T02:11:23.753218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241334894984953:2110] [node 6] ICS09 handshake done sender: [4:7587241334894984932:2114] self: [4:7587241334894984931:0] peer: [6:7587241333967460512:0] socket: 132 qp: -1
2025-12-24T02:11:23.753229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241334894984953:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.753247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.753438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460517:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
2025-12-24T02:11:23.753476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335932440187:2094] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.753509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460516:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
2025-12-24T02:11:23.753497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104337:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.753513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335705104337:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.754045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335705104324:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.754091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335705104337:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.754063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335932440185:2093] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.754081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.754090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335932440185:2093] [node 5] ICP22 created new session: [3:7587241335932440238:2093]
2025-12-24T02:11:23.754096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241335932440238:2093] [node 5] ICS09 handshake done sender: [3:7587241335932440187:2094] self: [3:7587241335932440186:0] peer: [5:7587241336583027409:0] socket: 111 qp: -1
2025-12-24T02:11:23.754107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241335932440238:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:23.754116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.754294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241336583027408:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.754341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984950:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.754371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241334894984926:2110] [node 6] ICP18 handshake (actor [4:7587241334894984950:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.754793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335932440237:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.754891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460518:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.754938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984955:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
2025-12-24T02:11:23.755006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335705104263:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.755032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.755043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335705104263:2049] [node 3] ICP22 created new session: [2:7587241335705104356:2049]
2025-12-24T02:11:23.755049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241335705104356:2049] [node 3] ICS09 handshake done sender: [2:7587241335705104324:2106] self: [2:7587241335705104322:0] peer: [3:7587241335932440232:0] socket: 116 qp: -1
2025-12-24T02:11:23.755059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241335705104356:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:23.755067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.755069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241333967460509:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.755140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241333967460502:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.755154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.755161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241333967460502:2112] [node 4] ICP22 created new session: [6:7587241333967460519:2112]
2025-12-24T02:11:23.755135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241336583027397:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:23.755169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241333967460519:2112] [node 4] ICS09 handshake done sender: [6:7587241333967460509:2117] self: [6:7587241333967460512:0] peer: [4:7587241334894984931:0] socket: 159 qp: -1
2025-12-24T02:11:23.755175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241333967460519:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:23.755184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.755421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335932440206:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.755507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335705104332:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.755526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.755532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335705104332:2108] [node 4] ICP22 created new session: [2:7587241335705104358:2108]
2025-12-24T02:11:23.755538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241335705104358:2108] [node 4] ICS09 handshake done sender: [2:7587241335705104337:2113] self: [2:7587241335705104334:0] peer: [4:7587241334894984951:0] socket: 137 qp: -1
2025-12-24T02:11:23.755544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241335705104358:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:23.755550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.757092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241336583027324:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.757113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:23.757120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241336583027324:2048] [node 1] ICP22 created new session: [5:7587241336583027412:2048]
2025-12-24T02:11:23.757126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241336583027412:2048] [node 1] ICS09 handshake done sender: [5:7587241336583027397:2115] self: [5:7587241336583027407:0] peer: [1:7587241332806705916:0] socket: 142 qp: -1
2025-12-24T02:11:23.757134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241336583027412:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:23.757140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:23.756818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241335932440139:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.756838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.756844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335932440139:2049] [node 2] ICP22 created new session: [3:7587241335932440240:2049]
2025-12-24T02:11:23.756849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241335932440240:2049] [node 2] ICS09 handshake done sender: [3:7587241335932440206:2110] self: [3:7587241335932440232:0] peer: [2:7587241335705104322:0] socket: 119 qp: -1
2025-12-24T02:11:23.756856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241335932440240:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.756863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.757309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241333967460426:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18319) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241333967460510:2118] held: no
2025-12-24T02:11:23.757433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241333967460510:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.759323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241333967460426:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.759447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241334894984937:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.759392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.759403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241333967460426:2050] [node 3] ICP22 created new session: [6:7587241333967460521:2050]
2025-12-24T02:11:23.759409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241333967460521:2050] [node 3] ICS09 handshake done sender: [6:7587241333967460510:2118] self: [6:7587241333967460513:0] peer: [3:7587241335932440211:0] socket: 162 qp: -1
2025-12-24T02:11:23.759418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241333967460521:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.759424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:23.759839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241333967460516:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.760149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241333967460425:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.760258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104343:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.760273Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335705104343:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.760569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332806705945:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.760701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241332806705909:2103] [node 5] ICP18 handshake (actor [1:7587241332806705945:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.761649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241336583027393:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.761736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241334894984860:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.761757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.761765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241334894984860:2049] [node 2] ICP22 created new session: [4:7587241334894984956:2049]
2025-12-24T02:11:23.761773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241334894984956:2049] [node 2] ICS09 handshake done sender: [4:7587241334894984937:2116] self: [4:7587241334894984951:0] peer: [2:7587241335705104334:0] socket: 146 qp: -1
2025-12-24T02:11:23.761780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241334894984956:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.761788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.763114Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241334894984860:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.762942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027411:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.763177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241336583027391:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:23.763269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241336583027326:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.763279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:23.763284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241336583027326:2050] [node 3] ICP22 created new session: [5:7587241336583027416:2050]
2025-12-24T02:11:23.763290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241336583027416:2050] [node 3] ICS09 handshake done sender: [5:7587241336583027391:2111] self: [5:7587241336583027409:0] peer: [3:7587241335932440186:0] socket: 113 qp: -1
2025-12-24T02:11:23.763296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241336583027416:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:23.763316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.764166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335932440223:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.764200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241335932440185:2093] [node 5] ICP18 handshake (actor [3:7587241335932440223:2120]) is too old
2025-12-24T02:11:23.764127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241334894984958:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12574
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.765139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027418:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.765105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984939:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.765113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241334894984939:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.766001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241334894984939:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:23.766161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241334894984924:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.766177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.766190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241334894984924:2109] [node 5] ICP22 created new session: [4:7587241334894984960:2109]
2025-12-24T02:11:23.766200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241334894984960:2109] [node 5] ICS09 handshake done sender: [4:7587241334894984939:2117] self: [4:7587241334894984938:0] peer: [5:7587241336583027414:0] socket: 118 qp: -1
2025-12-24T02:11:23.766209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241334894984960:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.766221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.767180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241333967460524:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54521
2025-12-24T02:11:23.767341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335705104343:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:23.767438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335705104333:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.767452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:23.767564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335705104333:2110] [node 6] ICP22 created new session: [2:7587241335705104362:2110]
2025-12-24T02:11:23.767573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241335705104362:2110] [node 6] ICS09 handshake done sender: [2:7587241335705104343:2118] self: [2:7587241335705104335:0] peer: [6:7587241333967460523:0] socket: 114 qp: -1
2025-12-24T02:11:23.767578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241335705104362:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:23.767585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:23.768144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241333967460516:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.769186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241333967460425:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.769205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.769212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241333967460425:2049] [node 2] ICP22 created new session: [6:7587241333967460525:2049]
2025-12-24T02:11:23.769218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241333967460525:2049] [node 2] ICS09 handshake done sender: [6:7587241333967460516:2119] self: [6:7587241333967460523:0] peer: [2:7587241335705104335:0] socket: 166 qp: -1
2025-12-24T02:11:23.769225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241333967460525:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.769233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.769398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241333967460425:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:23.768576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104349:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.772392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241336583027418:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.772475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241336583027325:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.772635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027420:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.773038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241334894984958:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.773074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241334894984924:2109] [node 5] ICP18 handshake (actor [4:7587241334894984958:2122]) is too old
2025-12-24T02:11:23.772895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335705104342:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.772904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335705104342:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:23.773349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335705104342:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.773524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241336583027421:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13878
2025-12-24T02:11:23.775242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241336583027418:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:23.775362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241336583027393:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7729) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241336583027408:2118] held: no
2025-12-24T02:11:23.775587Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241336583027408:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:23.775648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241336583027325:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.775664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:23.775673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241336583027325:2049] [node 2] ICP22 created new session: [5:7587241336583027422:2049]
2025-12-24T02:11:23.775689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241336583027422:2049] [node 2] ICS09 handshake done sender: [5:7587241336583027418:2121] self: [5:7587241336583027419:0] peer: [2:7587241335705104341:0] socket: 131 qp: -1
2025-12-24T02:11:23.775696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241336583027422:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:23.775706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:23.776569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241336583027393:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:23.776582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:23.776588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241336583027393:2110] [node 4] ICP22 created new session: [5:7587241336583027424:2110]
2025-12-24T02:11:23.776593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241336583027424:2110] [node 4] ICS09 handshake done sender: [5:7587241336583027408:2118] self: [5:7587241336583027414:0] peer: [4:7587241334894984938:0] socket: 122 qp: -1
2025-12-24T02:11:23.776599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241336583027424:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:23.776610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:23.779006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335705104330:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:23.779033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:23.779042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335705104330:2109] [node 5] ICP22 created new session: [2:7587241335705104366:2109]
2025-12-24T02:11:23.779049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241335705104366:2109] [node 5] ICS09 handshake done sender: [2:7587241335705104342:2117] self: [2:7587241335705104341:0] peer: [5:7587241336583027419:0] socket: 123 qp: -1
2025-12-24T02:11:23.779056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241335705104366:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:23.779065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:23.794412Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:23.794526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241335705104351:2048] [node 1] ICS07 socket disconnect 108 reason# EndOfStream
2025-12-24T02:11:23.794541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241335705104351:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.794602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.795531Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:23.795621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241335932440224:2048] [node 1] ICS07 socket disconnect 109 reason# EndOfStream
2025-12-24T02:11:23.795633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241335932440224:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.795680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.795705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241335932440224:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.797952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241335932440138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:23.797969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241335932440224:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.794627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241335705104351:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.798336Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241335705104262:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:23.798349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241335705104351:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.800690Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:23.800750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241336583027412:2048] [node 1] ICS07 socket disconnect 142 reason# EndOfStream
2025-12-24T02:11:23.800755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241336583027412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.800795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.800810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241336583027412:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:23.801094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241336583027324:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.801103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241336583027412:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.801106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241336583027324:2048] [node 1] ICP30 unregister session Session# [5:7587241336583027412:2048] VirtualId# [5:7587241336583027407:0]
2025-12-24T02:11:23.801140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.801146Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241336583027324:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.801148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.800632Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:23.800738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241333967460514:2048] [node 1] ICS01 socket: 148 reason# EndOfStream
2025-12-24T02:11:23.800747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241333967460424:2048] [node 1] ICP30 unregister session Session# [6:7587241333967460514:2048] VirtualId# [6:7587241333967460507:0]
2025-12-24T02:11:23.800754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241333967460514:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.800816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.800845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.802766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:23.802847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241334894984942:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:11:23.802861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241334894984859:2048] [node 1] ICP30 unregister session Session# [4:7587241334894984942:2048] VirtualId# [4:7587241334894984940:0]
2025-12-24T02:11:23.802867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241334894984942:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.802911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:23.802934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.802869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241335932440138:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.802885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241335932440224:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.802889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241335932440138:2048] [node 1] ICP30 unregister session Session# [3:7587241335932440224:2048] VirtualId# [3:7587241335932440219:0]
2025-12-24T02:11:23.802927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:23.802932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241335932440138:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.802935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.807150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:23.807239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241335932440240:2049] [node 2] ICS01 socket: 119 reason# EndOfStream
2025-12-24T02:11:23.807249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241335932440139:2049] [node 2] ICP30 unregister session Session# [3:7587241335932440240:2049] VirtualId# [3:7587241335932440232:0]
2025-12-24T02:11:23.807256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241335932440240:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.807311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.807336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.809162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:23.809274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241336583027422:2049] [node 2] ICS01 socket: 131 reason# EndOfStream
2025-12-24T02:11:23.809288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241336583027325:2049] [node 2] ICP30 unregister session Session# [5:7587241336583027422:2049] VirtualId# [5:7587241336583027419:0]
2025-12-24T02:11:23.809295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241336583027422:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.809365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.809396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.809419Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:23.809519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241333967460525:2049] [node 2] ICS07 socket disconnect 166 reason# EndOfStream
2025-12-24T02:11:23.809529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241333967460525:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.809570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.809589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241333967460525:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:23.810003Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241333967460425:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.810016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241333967460525:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.810021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241333967460425:2049] [node 2] ICP30 unregister session Session# [6:7587241333967460525:2049] VirtualId# [6:7587241333967460523:0]
2025-12-24T02:11:23.810067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.810073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241333967460425:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.810075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9733) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.810469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241336583027324:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:23.810745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:23.810816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241334894984956:2049] [node 2] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:11:23.810821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241334894984860:2049] [node 2] ICP30 unregister session Session# [4:7587241334894984956:2049] VirtualId# [4:7587241334894984951:0]
2025-12-24T02:11:23.810825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241334894984956:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.810867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:23.810894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:23.819694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241333967460425:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:23.822235Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:23.822341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241336583027416:2050] [node 3] ICS07 socket disconnect 113 reason# EndOfStream
2025-12-24T02:11:23.822348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241336583027416:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.822409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.822438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241336583027416:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:23.822473Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:23.822565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241333967460521:2050] [node 3] ICS01 socket: 162 reason# EndOfStream
2025-12-24T02:11:23.822578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241333967460426:2050] [node 3] ICP30 unregister session Session# [6:7587241333967460521:2050] VirtualId# [6:7587241333967460513:0]
2025-12-24T02:11:23.822585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241333967460521:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.822633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:23.822639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.822673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.822729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241334894984947:2050] [node 3] ICS01 socket: 127 reason# EndOfStream
2025-12-24T02:11:23.822742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241334894984861:2050] [node 3] ICP30 unregister session Session# [4:7587241334894984947:2050] VirtualId# [4:7587241334894984944:0]
2025-12-24T02:11:23.822748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241334894984947:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.822894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241336583027326:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18319) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:23.822909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241336583027416:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:23.822914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241336583027326:2050] [node 3] ICP30 unregister session Session# [5:7587241336583027416:2050] VirtualId# [5:7587241336583027409:0]
2025-12-24T02:11:23.822957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.822968Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241336583027326:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.822970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18319) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:23.822827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:23.822863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:23.832963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241336583027326:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:23.835252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:23.835330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241333967460519:2112] [node 4] ICS01 socket: 159 reason# EndOfStream
2025-12-24T02:11:23.835341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241333967460502:2112] [node 4] ICP30 unregister session Session# [6:7587241333967460519:2112] VirtualId# [6:7587241333967460512:0]
2025-12-24T02:11:23.835347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241333967460519:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.835086Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:23.835186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241336583027424:2110] [node 4] ICS01 socket: 122 reason# EndOfStream
2025-12-24T02:11:23.835196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241336583027393:2110] [node 4] ICP30 unregister session Session# [5:7587241336583027424:2110] VirtualId# [5:7587241336583027414:0]
2025-12-24T02:11:23.835204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241336583027424:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:23.835279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.835311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:23.835390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:23.835414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:23.857018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:23.857127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241333967460491:2108] [node 5] ICS01 socket: 102 reason# EndOfStream
2025-12-24T02:11:23.857144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241333967460486:2108] [node 5] ICP30 unregister session Session# [6:7587241333967460491:2108] VirtualId# [6:7587241333967460487:0]
2025-12-24T02:11:23.857153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241333967460491:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:23.895061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335554463200:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.895068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335554463201:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.896676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336022806663:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.896685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336022806664:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.896689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336022806665:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.900041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241332584601445:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.900054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241332584601446:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.900058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241332584601447:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.892039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332784560224:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:23.892051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332784560225:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:23.899793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333283312578:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.899812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333283312579:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:23.899817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333283312580:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:23.893499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335028732690:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:23.893511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335028732691:2049] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.930431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335028732750:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
2025-12-24T02:11:23.931591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336022806724:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:23.931621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332784560291:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:23.931024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333283312597:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:23.939065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241332584601464:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:23.939086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241332584601465:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:23.946026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332784560295:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:23.947814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241336022806731:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.947290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241332784560298:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.954167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241332584601509:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.959166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241332584601464:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20030087ns) {Configure:104(20029535ns) {ProcessConfigured:135(19175732ns) {StartInitialHandshake:157(19175306ns) {DropHandshakes:470(531ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:477(619ns) {SwitchToState:179(149ns)}}}}}
2025-12-24T02:11:23.958814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241333283312579:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24044242ns) {Configure:104(19529644ns) {ProcessConfigured:135(19224999ns) {StartInitialHandshake:157(19224458ns) {DropHandshakes:470(521ns) {DropIncomingHandshake:442(91ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(2136ns) {SwitchToState:179(1331ns)}}}}}
2025-12-24T02:11:23.961435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241335554463200:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20908096ns) {Configure:104(20907279ns) {ProcessConfigured:135(19973010ns) {StartInitialHandshake:157(19972531ns) {DropHandshakes:470(716ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(279ns) PrepareNewSessionHandshake:477(900ns) {SwitchToState:179(689ns)}}}}}
2025-12-24T02:11:23.961599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335554463268:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.961727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335554463269:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.961882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335554463260:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:23.962471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335554463270:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:23.962485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335554463271:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:23.974839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241332584601445:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34838078ns) {Configure:104(28593375ns) {ProcessConfigured:135(28201430ns) {StartInitialHandshake:157(28200529ns) {DropHandshakes:470(607ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(1178ns) {SwitchToState:179(531ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.975043Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241332784560295:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28771618ns) {Configure:104(28771154ns) {ProcessConfigured:135(27907349ns) {StartInitialHandshake:157(27906891ns) {DropHandshakes:470(443ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(144ns) PrepareNewSessionHandshake:477(438ns) {SwitchToState:179(379ns)}}}}}
2025-12-24T02:11:23.975270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241336022806665:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28233939ns) {Configure:104(28233359ns) {ProcessConfigured:135(27852469ns) {StartInitialHandshake:157(27851856ns) {DropHandshakes:470(870ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(132ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(967ns) {SwitchToState:179(600ns)}}}}}
2025-12-24T02:11:23.975904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336022806737:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:23.975959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241336022806738:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:23.979140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241335554463266:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:23.979215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463271:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.979258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463260:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.978998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335028732757:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:23.979221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335028732759:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:23.976212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241332784560298:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28804852ns) {Configure:104(28804385ns) {ProcessConfigured:135(28491279ns) {StartInitialHandshake:157(28491006ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(750ns) {SwitchToState:179(226ns)}}}}}
2025-12-24T02:11:23.978904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336022806737:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.979431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241335028732758:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:23.978951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336022806738:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.985361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241332584601524:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:23.985391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241332584601525:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:23.985411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241332584601526:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:23.985159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332784560303:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16494
2025-12-24T02:11:23.985185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332784560304:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16494
2025-12-24T02:11:23.985189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241332784560305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16494
2025-12-24T02:11:23.983237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241335028732691:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47557149ns) {Configure:104(47556142ns) {ProcessConfigured:135(47288070ns) {StartInitialHandshake:157(47287332ns) {DropHandshakes:470(491ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1483ns) {SwitchToState:179(1125ns)}}}}}
2025-12-24T02:11:23.984952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333283312654:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:23.984980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241333283312655:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:23.990794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335028732750:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:23.990896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241335028732690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:23.994874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241333283312653:2069] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:23.999601Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241335554463201:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58876759ns) {Configure:104(58876028ns) {ProcessConfigured:135(58017477ns) {StartInitialHandshake:157(58016356ns) {DropHandshakes:470(81ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(147ns) PrepareNewSessionHandshake:477(1723ns) {SwitchToState:179(1192ns)}}}}}
2025-12-24T02:11:24.001474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241335554463200:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.001620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335554463279:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:24.001633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241335554463280:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:24.001705Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241336022806663:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47341510ns) {Configure:104(47340908ns) {ProcessConfigured:135(47082215ns) {StartInitialHandshake:157(47080949ns) {DropHandshakes:470(235ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(1473ns) {SwitchToState:179(1144ns)}}}}}
2025-12-24T02:11:24.001802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336022806663:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.001980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241336022806664:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(54025553ns) {Configure:104(54024777ns) {ProcessConfigured:135(53727386ns) {StartInitialHandshake:157(53726831ns) {DropHandshakes:470(300ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(132ns) PrepareNewSessionHandshake:477(437ns) {SwitchToState:179(235ns)}}}}}
2025-12-24T02:11:23.999697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241335028732770:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
2025-12-24T02:11:24.002051Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241335028732757:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22399769ns) {Configure:104(22398285ns) {ProcessConfigured:135(22070274ns) {StartInitialHandshake:157(22069785ns) {DropHandshakes:470(841ns) {DropIncomingHandshake:442(177ns) DropOutgoingHandshake:457(37ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(447ns) {SwitchToState:179(209ns)}}}}}
2025-12-24T02:11:24.002818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560285:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.002830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332784560285:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.002898Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560287:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.002904Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332784560287:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.005284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241332584601509:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.005331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241332584601464:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.005434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463270:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.005141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333283312654:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.005178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333283312580:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.006825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335028732770:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.008298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339323700068:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
2025-12-24T02:11:24.010843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335028732750:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.014500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463273:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.014516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241335554463273:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.006758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560296:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.006783Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332784560296:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.006885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332784560285:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.007438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560303:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.007456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332784560224:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.007473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.007482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332784560224:2048] [node 2] ICP22 created new session: [1:7587241337079527603:2048]
2025-12-24T02:11:24.007504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337079527603:2048] [node 2] ICS09 handshake done sender: [1:7587241332784560285:2107] self: [1:7587241332784560284:0] peer: [2:7587241335028732769:0] socket: 108 qp: -1
2025-12-24T02:11:24.007513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337079527603:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:24.007537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.008570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332784560296:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.014628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332784560291:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.014644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.014655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332784560291:2109] [node 4] ICP22 created new session: [1:7587241337079527609:2109]
2025-12-24T02:11:24.014666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337079527609:2109] [node 4] ICS09 handshake done sender: [1:7587241332784560296:2112] self: [1:7587241332784560292:0] peer: [4:7587241340317774039:0] socket: 116 qp: -1
2025-12-24T02:11:24.014680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337079527609:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:24.014690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.014403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333283312625:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.014429Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241333283312625:2091] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.014756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241335028732690:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.014791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.014799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335028732690:2048] [node 1] ICP22 created new session: [2:7587241339323700070:2048]
2025-12-24T02:11:24.014806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339323700070:2048] [node 1] ICS09 handshake done sender: [2:7587241335028732750:2107] self: [2:7587241335028732769:0] peer: [1:7587241332784560284:0] socket: 110 qp: -1
2025-12-24T02:11:24.014813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339323700070:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.014847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.018864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774041:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.019572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336022806737:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.019715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336022806663:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.019735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.019742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336022806663:2048] [node 1] ICP22 created new session: [4:7587241340317774042:2048]
2025-12-24T02:11:24.019749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340317774042:2048] [node 1] ICS09 handshake done sender: [4:7587241336022806737:2113] self: [4:7587241340317774039:0] peer: [1:7587241332784560292:0] socket: 133 qp: -1
2025-12-24T02:11:24.019755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340317774042:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.019772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.020495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339323700073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.023400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568824:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:24.024235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335554463273:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.024551Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332784560287:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.024526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241339849430583:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.025018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339323700074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
2025-12-24T02:11:24.025193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568825:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:24.025390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241332584601509:2106] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.025927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335554463268:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.025947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.025952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335554463268:2109] [node 5] ICP22 created new session: [3:7587241339849430585:2109]
2025-12-24T02:11:24.025958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241339849430585:2109] [node 5] ICS09 handshake done sender: [3:7587241335554463273:2114] self: [3:7587241335554463272:0] peer: [5:7587241337578279954:0] socket: 126 qp: -1
2025-12-24T02:11:24.025963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241339849430585:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:24.025969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.025542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241332584601464:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.025561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.025568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241332584601464:2059] [node 5] ICP22 created new session: [6:7587241336879568826:2059]
2025-12-24T02:11:24.025574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336879568826:2059] [node 5] ICS09 handshake done sender: [6:7587241332584601509:2106] self: [6:7587241336879568823:0] peer: [5:7587241333283312598:0] socket: 117 qp: -1
2025-12-24T02:11:24.025580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336879568826:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:24.025588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.024369Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333283312625:2091] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.024504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241333283312597:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.024517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.024525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333283312597:2064] [node 6] ICP22 created new session: [5:7587241337578279956:2064]
2025-12-24T02:11:24.024531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241337578279956:2064] [node 6] ICS09 handshake done sender: [5:7587241333283312625:2091] self: [5:7587241333283312598:0] peer: [6:7587241336879568823:0] socket: 113 qp: -1
2025-12-24T02:11:24.024538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241337578279956:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:24.024566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.024793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279958:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.024808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279960:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.025027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279961:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.025237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333283312654:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.025295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333283312580:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.025305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.025311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333283312580:2050] [node 3] ICP22 created new session: [5:7587241337578279962:2050]
2025-12-24T02:11:24.025316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241337578279962:2050] [node 3] ICS09 handshake done sender: [5:7587241333283312654:2114] self: [5:7587241337578279954:0] peer: [3:7587241335554463272:0] socket: 129 qp: -1
2025-12-24T02:11:24.025322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241337578279962:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.025338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.026810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332784560225:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.026836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.026845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332784560225:2049] [node 3] ICP22 created new session: [1:7587241337079527612:2049]
2025-12-24T02:11:24.026852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337079527612:2049] [node 3] ICS09 handshake done sender: [1:7587241332784560287:2108] self: [1:7587241332784560286:0] peer: [3:7587241339849430581:0] socket: 109 qp: -1
2025-12-24T02:11:24.026859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337079527612:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.026869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.029031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339323700075:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20601
2025-12-24T02:11:24.029180Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339323700074:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.030738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339323700075:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.030779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335554463260:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.030926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241335554463200:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.030944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.030951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335554463200:2048] [node 1] ICP22 created new session: [3:7587241339849430588:2048]
2025-12-24T02:11:24.030958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241339849430588:2048] [node 1] ICS09 handshake done sender: [3:7587241335554463260:2107] self: [3:7587241339849430581:0] peer: [1:7587241332784560286:0] socket: 112 qp: -1
2025-12-24T02:11:24.030965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241339849430588:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.030975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.031020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560304:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.031106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337079527614:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16494
2025-12-24T02:11:24.031128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241332784560225:2049] [node 3] ICP18 handshake (actor [1:7587241332784560304:2116]) is too old
2025-12-24T02:11:24.032953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241336879568824:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.032983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241332584601445:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.033112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337079527614:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.034997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560302:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.035013Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332784560302:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.039123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568830:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:24.039288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332784560302:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.039517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241336879568824:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.039540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332784560298:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.039562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.039569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332784560298:2111] [node 6] ICP22 created new session: [1:7587241337079527617:2111]
2025-12-24T02:11:24.039575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337079527617:2111] [node 6] ICS09 handshake done sender: [1:7587241332784560302:2114] self: [1:7587241332784560300:0] peer: [6:7587241336879568829:0] socket: 157 qp: -1
2025-12-24T02:11:24.039581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337079527617:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:24.039588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.039657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241332584601445:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.039674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.039679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241332584601445:2048] [node 1] ICP22 created new session: [6:7587241336879568831:2048]
2025-12-24T02:11:24.039685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336879568831:2048] [node 1] ICS09 handshake done sender: [6:7587241336879568824:2119] self: [6:7587241336879568829:0] peer: [1:7587241332784560300:0] socket: 161 qp: -1
2025-12-24T02:11:24.039691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336879568831:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.039698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.042370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241332584601524:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.042405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241332584601447:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.040055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241337578279958:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.040096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333283312578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.042563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463275:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.042575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241335554463275:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.043449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241332784560301:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.043463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241332784560301:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.051107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241332584601525:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.051171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241332584601465:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.057642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241332584601447:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1035) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241332584601524:2116] held: no
2025-12-24T02:11:24.056119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335554463275:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.056149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.056424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335554463269:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.056433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241337578279958:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.056448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.056520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333283312578:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.056461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335554463269:2110] [node 6] ICP22 created new session: [3:7587241339849430592:2110]
2025-12-24T02:11:24.056540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.056470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241339849430592:2110] [node 6] ICS09 handshake done sender: [3:7587241335554463275:2115] self: [3:7587241335554463274:0] peer: [6:7587241336879568833:0] socket: 127 qp: -1
2025-12-24T02:11:24.056549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333283312578:2048] [node 1] ICP22 created new session: [5:7587241337578279967:2048]
2025-12-24T02:11:24.056267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241332784560301:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.056605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241332784560295:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.056618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.056479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241339849430592:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:24.056625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241332784560295:2110] [node 5] ICP22 created new session: [1:7587241337079527620:2110]
2025-12-24T02:11:24.056519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.056555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241337578279967:2048] [node 1] ICS09 handshake done sender: [5:7587241337578279958:2117] self: [5:7587241337578279965:0] peer: [1:7587241332784560297:0] socket: 156 qp: -1
2025-12-24T02:11:24.056631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337079527620:2110] [node 5] ICS09 handshake done sender: [1:7587241332784560301:2113] self: [1:7587241332784560297:0] peer: [5:7587241337578279965:0] socket: 153 qp: -1
2025-12-24T02:11:24.056561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241337578279967:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.056568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.056636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337079527620:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:24.056643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.059147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568835:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:24.059290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241332584601524:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.059421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241332584601447:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.059433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.059440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241332584601447:2050] [node 3] ICP22 created new session: [6:7587241336879568836:2050]
2025-12-24T02:11:24.059448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336879568836:2050] [node 3] ICS09 handshake done sender: [6:7587241332584601524:2116] self: [6:7587241336879568833:0] peer: [3:7587241335554463274:0] socket: 131 qp: -1
2025-12-24T02:11:24.059455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336879568836:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.059464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.060032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335028732763:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.060048Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335028732763:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.060949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335028732763:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.061118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335028732759:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.061143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.061152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335028732759:2110] [node 6] ICP22 created new session: [2:7587241339323700079:2110]
2025-12-24T02:11:24.061158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339323700079:2110] [node 6] ICS09 handshake done sender: [2:7587241335028732763:2113] self: [2:7587241335028732762:0] peer: [6:7587241336879568838:0] socket: 142 qp: -1
2025-12-24T02:11:24.061166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339323700079:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:24.061173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.059873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241332584601526:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.059916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241332584601446:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.066939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241332584601446:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241332584601526:2118] held: no
2025-12-24T02:11:24.066989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336022806728:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.067008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241336022806728:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.069442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463280:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.069494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241335554463201:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.069651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335028732761:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.069666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335028732761:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.070311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241339849430595:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2820
2025-12-24T02:11:24.070412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335028732761:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.070628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241335554463280:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.067467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774045:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.067555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774046:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.067701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774047:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.070718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241335554463201:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.070735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.070750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335554463201:2049] [node 2] ICP22 created new session: [3:7587241339849430596:2049]
2025-12-24T02:11:24.070757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241339849430596:2049] [node 2] ICS09 handshake done sender: [3:7587241335554463280:2117] self: [3:7587241339849430594:0] peer: [2:7587241335028732752:0] socket: 149 qp: -1
2025-12-24T02:11:24.070765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241339849430596:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.070774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.071258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335028732691:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.071278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.071284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335028732691:2049] [node 3] ICP22 created new session: [2:7587241339323700083:2049]
2025-12-24T02:11:24.071289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339323700083:2049] [node 3] ICS09 handshake done sender: [2:7587241335028732761:2112] self: [2:7587241335028732752:0] peer: [3:7587241339849430594:0] socket: 141 qp: -1
2025-12-24T02:11:24.071294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339323700083:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.071301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.074015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241333283312655:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.074091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333283312653:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.071166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336022806728:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.073280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340317774046:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.073361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340317774045:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.073369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241336022806724:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.073383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.073390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336022806724:2106] [node 6] ICP22 created new session: [4:7587241340317774049:2106]
2025-12-24T02:11:24.073395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340317774049:2106] [node 6] ICS09 handshake done sender: [4:7587241336022806728:2110] self: [4:7587241336022806725:0] peer: [6:7587241336879568834:0] socket: 132 qp: -1
2025-12-24T02:11:24.073399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340317774049:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:24.073406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.087474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241335554463279:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.087180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336022806665:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.087355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241336022806736:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.087364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241336022806736:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.091081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241335028732766:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.091097Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241335028732766:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.090891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241337578279960:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.090946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241333283312579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.095246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339323700073:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.098957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241339849430580:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.098979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241339849430580:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.099025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568839:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.102926Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241332584601465:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8843) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241332584601525:2117] held: no
2025-12-24T02:11:24.103160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241336879568840:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55061
2025-12-24T02:11:24.103345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241332584601526:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.104037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340317774047:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.104082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241336022806664:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.103594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241332584601446:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.103615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.103627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241332584601446:2049] [node 2] ICP22 created new session: [6:7587241336879568841:2049]
2025-12-24T02:11:24.103637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336879568841:2049] [node 2] ICS09 handshake done sender: [6:7587241332584601526:2118] self: [6:7587241336879568838:0] peer: [2:7587241335028732762:0] socket: 147 qp: -1
2025-12-24T02:11:24.103648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336879568841:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.103661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.104250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241332584601525:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.105563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774054:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.105830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340317774047:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.105898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336022806664:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.105914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.105919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336022806664:2049] [node 2] ICP22 created new session: [4:7587241340317774055:2049]
2025-12-24T02:11:24.105924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340317774055:2049] [node 2] ICS09 handshake done sender: [4:7587241340317774047:2120] self: [4:7587241340317774053:0] peer: [2:7587241335028732760:0] socket: 120 qp: -1
2025-12-24T02:11:24.105930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340317774055:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.105936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.104167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339323700067:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.104183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339323700067:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.105673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339323700067:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.108109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279971:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.108414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241333283312655:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.108509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333283312653:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.108522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.108527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333283312653:2069] [node 4] ICP22 created new session: [5:7587241337578279972:2069]
2025-12-24T02:11:24.108533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241337578279972:2069] [node 4] ICS09 handshake done sender: [5:7587241333283312655:2115] self: [5:7587241337578279969:0] peer: [4:7587241336022806735:0] socket: 143 qp: -1
2025-12-24T02:11:24.108539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241337578279972:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:24.108546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.106872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335028732757:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.106893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.106899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335028732757:2108] [node 4] ICP22 created new session: [2:7587241339323700087:2108]
2025-12-24T02:11:24.106907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339323700087:2108] [node 4] ICS09 handshake done sender: [2:7587241339323700067:2116] self: [2:7587241335028732760:0] peer: [4:7587241340317774053:0] socket: 122 qp: -1
2025-12-24T02:11:24.106913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339323700087:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:24.106920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.108252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241336022806736:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.109537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241336022806731:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.109549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.109555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336022806731:2109] [node 5] ICP22 created new session: [4:7587241340317774058:2109]
2025-12-24T02:11:24.109560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340317774058:2109] [node 5] ICS09 handshake done sender: [4:7587241336022806736:2112] self: [4:7587241336022806735:0] peer: [5:7587241337578279969:0] socket: 136 qp: -1
2025-12-24T02:11:24.109564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340317774058:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:24.109569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.105066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241332584601465:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.105080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.105086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241332584601465:2063] [node 4] ICP22 created new session: [6:7587241336879568843:2063]
2025-12-24T02:11:24.105092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241336879568843:2063] [node 4] ICS09 handshake done sender: [6:7587241332584601525:2117] self: [6:7587241336879568834:0] peer: [4:7587241336022806725:0] socket: 146 qp: -1
2025-12-24T02:11:24.105098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241336879568843:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:24.105104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.111333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241337578279974:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56904
2025-12-24T02:11:24.113624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241333283312579:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31056) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241337578279960:2118] held: no
2025-12-24T02:11:24.111444Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241335028732766:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.111544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241335028732758:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.111558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.111564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241335028732758:2109] [node 5] ICP22 created new session: [2:7587241339323700090:2109]
2025-12-24T02:11:24.111570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339323700090:2109] [node 5] ICS09 handshake done sender: [2:7587241335028732766:2114] self: [2:7587241335028732765:0] peer: [5:7587241337578279970:0] socket: 158 qp: -1
2025-12-24T02:11:24.111576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339323700090:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:24.111586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.114767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241337578279960:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.114851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241333283312579:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.114877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.114884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241333283312579:2049] [node 2] ICP22 created new session: [5:7587241337578279975:2049]
2025-12-24T02:11:24.114890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241337578279975:2049] [node 2] ICS09 handshake done sender: [5:7587241337578279960:2118] self: [5:7587241337578279970:0] peer: [2:7587241335028732765:0] socket: 165 qp: -1
2025-12-24T02:11:24.114897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241337578279975:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.114913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.115469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241339849430580:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.115612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241335554463266:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.115629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.115636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241335554463266:2108] [node 4] ICP22 created new session: [3:7587241339849430600:2108]
2025-12-24T02:11:24.115641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241339849430600:2108] [node 4] ICS09 handshake done sender: [3:7587241339849430580:2119] self: [3:7587241339849430579:0] peer: [4:7587241340317774051:0] socket: 125 qp: -1
2025-12-24T02:11:24.115647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241339849430600:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:24.115655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.115372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340317774060:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35618
2025-12-24T02:11:24.116171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340317774046:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.116256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241336022806665:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.116267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.116274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241336022806665:2050] [node 3] ICP22 created new session: [4:7587241340317774061:2050]
2025-12-24T02:11:24.116280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340317774061:2050] [node 3] ICS09 handshake done sender: [4:7587241340317774046:2119] self: [4:7587241340317774051:0] peer: [3:7587241339849430579:0] socket: 170 qp: -1
2025-12-24T02:11:24.116287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340317774061:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.116295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.131283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:24.131375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241339323700070:2048] [node 1] ICS01 socket: 110 reason# EndOfStream
2025-12-24T02:11:24.131385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241335028732690:2048] [node 1] ICP30 unregister session Session# [2:7587241339323700070:2048] VirtualId# [2:7587241335028732769:0]
2025-12-24T02:11:24.131394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241339323700070:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.131427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.131449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.150836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:24.150966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241339849430588:2048] [node 1] ICS07 socket disconnect 112 reason# EndOfStream
2025-12-24T02:11:24.150977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241339849430588:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.151042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.151067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241339849430588:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:24.151560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241335554463200:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:24.151578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241339849430588:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:24.151584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241335554463200:2048] [node 1] ICP30 unregister session Session# [3:7587241339849430588:2048] VirtualId# [3:7587241339849430581:0]
2025-12-24T02:11:24.151645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.151653Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241335554463200:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.151656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.158783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:24.158870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241337578279967:2048] [node 1] ICS01 socket: 156 reason# EndOfStream
2025-12-24T02:11:24.158878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333283312578:2048] [node 1] ICP30 unregister session Session# [5:7587241337578279967:2048] VirtualId# [5:7587241337578279965:0]
2025-12-24T02:11:24.158887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241337578279967:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.158934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.158957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.161347Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:24.161436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241340317774042:2048] [node 1] ICS01 socket: 133 reason# EndOfStream
2025-12-24T02:11:24.161443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241336022806663:2048] [node 1] ICP30 unregister session Session# [4:7587241340317774042:2048] VirtualId# [4:7587241340317774039:0]
2025-12-24T02:11:24.161449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241340317774042:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.161506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.161534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.161586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:24.161664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241336879568831:2048] [node 1] ICS07 socket disconnect 161 reason# EndOfStream
2025-12-24T02:11:24.161669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336879568831:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.161709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.161735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241336879568831:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:24.162069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241335554463200:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:24.162444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241332584601445:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:24.162459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336879568831:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:24.162465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241332584601445:2048] [node 1] ICP30 unregister session Session# [6:7587241336879568831:2048] VirtualId# [6:7587241336879568829:0]
2025-12-24T02:11:24.162515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.162522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241332584601445:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.162525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28224) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.178784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241332584601445:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:24.178940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:24.179049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241339849430596:2049] [node 2] ICS01 socket: 149 reason# EndOfStream
2025-12-24T02:11:24.179059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241335554463201:2049] [node 2] ICP30 unregister session Session# [3:7587241339849430596:2049] VirtualId# [3:7587241339849430594:0]
2025-12-24T02:11:24.179069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241339849430596:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.179130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.179158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.185418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:24.185526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241340317774055:2049] [node 2] ICS01 socket: 120 reason# EndOfStream
2025-12-24T02:11:24.185537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241336022806664:2049] [node 2] ICP30 unregister session Session# [4:7587241340317774055:2049] VirtualId# [4:7587241340317774053:0]
2025-12-24T02:11:24.185545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241340317774055:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.185594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.185620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.186877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:24.186960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241337578279975:2049] [node 2] ICS01 socket: 165 reason# EndOfStream
2025-12-24T02:11:24.186968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333283312579:2049] [node 2] ICP30 unregister session Session# [5:7587241337578279975:2049] VirtualId# [5:7587241337578279970:0]
2025-12-24T02:11:24.186973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241337578279975:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.187029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.187053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.194759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:24.194858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336879568841:2049] [node 2] ICS01 socket: 147 reason# EndOfStream
2025-12-24T02:11:24.194866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241332584601446:2049] [node 2] ICP30 unregister session Session# [6:7587241336879568841:2049] VirtualId# [6:7587241336879568838:0]
2025-12-24T02:11:24.194874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336879568841:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.194935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.194965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.206785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:24.206892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241337578279962:2050] [node 3] ICS01 socket: 129 reason# EndOfStream
2025-12-24T02:11:24.206902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241333283312580:2050] [node 3] ICP30 unregister session Session# [5:7587241337578279962:2050] VirtualId# [5:7587241337578279954:0]
2025-12-24T02:11:24.206910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241337578279962:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.206973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:24.206999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:24.210761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:24.210870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336879568836:2050] [node 3] ICS01 socket: 131 reason# EndOfStream
2025-12-24T02:11:24.210880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241332584601447:2050] [node 3] ICP30 unregister session Session# [6:7587241336879568836:2050] VirtualId# [6:7587241336879568833:0]
2025-12-24T02:11:24.210887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336879568836:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.210942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:24.210969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:24.221385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:24.221480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241336879568843:2063] [node 4] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:11:24.221490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241332584601465:2063] [node 4] ICP30 unregister session Session# [6:7587241336879568843:2063] VirtualId# [6:7587241336879568834:0]
2025-12-24T02:11:24.221498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241336879568843:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.221551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:24.221578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:24.294917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241338631894599:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:24.294926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241338631894600:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:24.302854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241340739234747:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.298799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339405453010:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.298812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339405453011:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:24.302863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241340739234748:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.310810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241339645876348:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.310829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241339645876349:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.310835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241339645876350:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.314892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340015861109:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.314906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340015861110:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.318756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340015861108:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.322779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340577635776:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.322884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340577635777:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.322889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340577635778:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.362497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339405453071:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:24.362933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241338631894660:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.364017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635787:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.367622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894663:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.367633Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241338631894663:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.368403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241338631894663:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.366947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340577635787:2058] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.367391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340577635776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.368262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635792:2062] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.368580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241338631894660:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.368599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.368606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241338631894660:2107] [node 6] ICP22 created new session: [1:7587241338631894666:2107]
2025-12-24T02:11:24.368614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241338631894666:2107] [node 6] ICS09 handshake done sender: [1:7587241338631894663:2109] self: [1:7587241338631894662:0] peer: [6:7587241340577635791:0] socket: 114 qp: -1
2025-12-24T02:11:24.368622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241338631894666:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:24.368632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.373462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861142:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.369069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340577635787:2058] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.369178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340577635776:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.369189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.369195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340577635776:2048] [node 1] ICP22 created new session: [6:7587241340577635793:2048]
2025-12-24T02:11:24.369202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340577635793:2048] [node 1] ICS09 handshake done sender: [6:7587241340577635787:2058] self: [6:7587241340577635791:0] peer: [1:7587241338631894662:0] socket: 115 qp: -1
2025-12-24T02:11:24.369209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340577635793:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.369229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.371100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241338631894669:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.376270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241338631894676:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46586
2025-12-24T02:11:24.376290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241338631894677:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46586
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.377977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340577635812:2060] [node 5] ICP01 ready to work
2025-12-24T02:11:24.377361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453081:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
2025-12-24T02:11:24.377389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241338631894678:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:24.382937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340577635820:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:24.384508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340015861155:2062] [node 6] ICP01 ready to work
2025-12-24T02:11:24.386919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340015861177:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:24.383545Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894677:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.387097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894676:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.390874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339405453083:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:24.391708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339405453087:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.395397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861142:2079] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.395437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861184:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.395462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340015861108:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.406993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241340577635820:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23968239ns) {Configure:104(23967553ns) {ProcessConfigured:135(23647466ns) {StartInitialHandshake:157(23646977ns) {DropHandshakes:470(407ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(962ns) {SwitchToState:179(239ns)}}}}}
2025-12-24T02:11:24.408131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241340577635777:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23083487ns) {Configure:104(23083173ns) {ProcessConfigured:135(22798277ns) {StartInitialHandshake:157(22798140ns) {DropHandshakes:470(319ns) {DropIncomingHandshake:442(93ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(719ns) {SwitchToState:179(185ns)}}}}}
2025-12-24T02:11:24.407972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241340015861155:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23369516ns) {Configure:104(23368366ns) {ProcessConfigured:135(23026943ns) {StartInitialHandshake:157(23026651ns) {DropHandshakes:470(389ns) {DropIncomingHandshake:442(90ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1435ns) {SwitchToState:179(982ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.410949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241340015861177:2075] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23890760ns) {Configure:104(23890347ns) {ProcessConfigured:135(23623479ns) {StartInitialHandshake:157(23623236ns) {DropHandshakes:470(371ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(719ns) {SwitchToState:179(494ns)}}}}}
2025-12-24T02:11:24.411356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876414:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.411431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876415:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.411463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876416:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.414778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241338631894599:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27636248ns) {Configure:104(27635502ns) {ProcessConfigured:135(27293389ns) {StartInitialHandshake:157(27292638ns) {DropHandshakes:470(443ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1527ns) {SwitchToState:179(947ns)}}}}}
2025-12-24T02:11:24.415406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876415:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.415441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876414:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.422840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241339645876410:2098] [node 6] ICP01 ready to work
2025-12-24T02:11:24.422732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894673:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.422755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241338631894673:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.427018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861184:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.426873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453081:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.430723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241339645876348:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.430799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876416:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.434874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241340739234815:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:24.435079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241340739234818:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:24.438718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241339645876413:2100] [node 5] ICP01 ready to work
2025-12-24T02:11:24.438832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894680:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.438857Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241338631894680:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.438970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241340739234814:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:24.442789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241339645876349:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63989913ns) {Configure:104(63988992ns) {ProcessConfigured:135(63354380ns) {StartInitialHandshake:157(63353929ns) {DropHandshakes:470(484ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1783ns) {SwitchToState:179(808ns)}}}}}
2025-12-24T02:11:24.442881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241339645876349:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.443636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241340739234748:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56743392ns) {Configure:104(56742437ns) {ProcessConfigured:135(56332315ns) {StartInitialHandshake:157(56332084ns) {DropHandshakes:470(387ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(669ns) {SwitchToState:179(214ns)}}}}}
2025-12-24T02:11:24.446792Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234817:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.450798Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241338631894673:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.451032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241338631894669:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.451053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.451061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241338631894669:2110] [node 5] ICP22 created new session: [1:7587241338631894688:2110]
2025-12-24T02:11:24.451068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241338631894688:2110] [node 5] ICS09 handshake done sender: [1:7587241338631894673:2112] self: [1:7587241338631894672:0] peer: [5:7587241340015861185:0] socket: 123 qp: -1
2025-12-24T02:11:24.451075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241338631894688:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:24.451097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.450978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234817:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.451034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234827:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-12-24T02:11:24.451086Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234828:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-12-24T02:11:24.451113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-12-24T02:11:24.451144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234830:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-12-24T02:11:24.451270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241340739234818:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16050071ns) {Configure:104(16049600ns) {ProcessConfigured:135(15692768ns) {StartInitialHandshake:157(15691755ns) {DropHandshakes:470(1243ns) {DropIncomingHandshake:442(252ns) DropOutgoingHandshake:457(296ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(554ns) {SwitchToState:179(40ns)}}}}}
2025-12-24T02:11:24.454777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234827:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.454945Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340015861108:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64181) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340015861142:2079] held: no
2025-12-24T02:11:24.455081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241338631894691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46586
2025-12-24T02:11:24.458718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453079:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.458741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339405453079:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.458825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234828:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.458859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234829:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.458890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241340739234748:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.459014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234830:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.459002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241338631894690:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46586
2025-12-24T02:11:24.459096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241338631894680:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.462750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894691:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.462773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894690:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.462905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241338631894678:2108] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.463093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453073:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.463105Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339405453073:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.463619Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453091:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
2025-12-24T02:11:24.463632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453092:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
2025-12-24T02:11:24.463645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453093:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
2025-12-24T02:11:24.462924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.462940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241338631894678:2108] [node 4] ICP22 created new session: [1:7587241338631894695:2108]
2025-12-24T02:11:24.462948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241338631894695:2108] [node 4] ICS09 handshake done sender: [1:7587241338631894680:2115] self: [1:7587241338631894679:0] peer: [4:7587241339645876423:0] socket: 135 qp: -1
2025-12-24T02:11:24.462956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241338631894695:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:24.462969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.463885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861191:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.464135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861192:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.464150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340015861142:2079] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.464191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340015861108:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.464201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.464207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340015861108:2048] [node 1] ICP22 created new session: [5:7587241340015861193:2048]
2025-12-24T02:11:24.464213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340015861193:2048] [node 1] ICS09 handshake done sender: [5:7587241340015861142:2079] self: [5:7587241340015861185:0] peer: [1:7587241338631894672:0] socket: 124 qp: -1
2025-12-24T02:11:24.464219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340015861193:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.464225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.466781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339405453073:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.466976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453093:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.466999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241339405453010:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.468107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
2025-12-24T02:11:24.468208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339405453093:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.468247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241339405453010:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.468256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.468261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339405453010:2048] [node 1] ICP22 created new session: [2:7587241339405453097:2048]
2025-12-24T02:11:24.468266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339405453097:2048] [node 1] ICS09 handshake done sender: [2:7587241339405453093:2118] self: [2:7587241339405453095:0] peer: [1:7587241338631894683:0] socket: 163 qp: -1
2025-12-24T02:11:24.468270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339405453097:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.468276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.470744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339405453011:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.470754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.470759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339405453011:2049] [node 3] ICP22 created new session: [2:7587241339405453100:2049]
2025-12-24T02:11:24.470765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339405453100:2049] [node 3] ICS09 handshake done sender: [2:7587241339405453073:2108] self: [2:7587241339405453072:0] peer: [3:7587241340739234834:0] socket: 126 qp: -1
2025-12-24T02:11:24.470771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339405453100:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.470790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.467129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894684:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.467152Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241338631894684:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.467588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241338631894684:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.467677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241338631894599:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.467691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.467697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241338631894599:2048] [node 2] ICP22 created new session: [1:7587241338631894698:2048]
2025-12-24T02:11:24.467703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241338631894698:2048] [node 2] ICS09 handshake done sender: [1:7587241338631894684:2116] self: [1:7587241338631894683:0] peer: [2:7587241339405453095:0] socket: 159 qp: -1
2025-12-24T02:11:24.467709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241338631894698:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:24.467715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.470974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453092:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.478717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241338631894671:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.478739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241338631894671:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.478717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241340739234747:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.478765Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241340739234747:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23928294ns) {IncomingHandshake:261(23927805ns) {DropIncomingHandshake:442(52ns) PrepareNewSessionHandshake:477(453ns) {SwitchToState:179(64ns)} GenerateSessionVirtualId:988(240ns) IssueIncomingHandshakeReply:193(23921874ns) {SwitchToState:179(1279ns)}}}
2025-12-24T02:11:24.478908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234837:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.483846Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241339645876410:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(60681189ns) {Configure:104(60680446ns) {ProcessConfigured:135(60311892ns) {StartInitialHandshake:157(60310929ns) {DropHandshakes:470(1371ns) {DropIncomingHandshake:442(562ns) DropOutgoingHandshake:457(245ns)} GenerateSessionVirtualId:988(282ns) PrepareNewSessionHandshake:477(1455ns) {SwitchToState:179(320ns)}}}}}
2025-12-24T02:11:24.483881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241339645876348:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64181) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241339645876415:2111] held: no
2025-12-24T02:11:24.484202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876429:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.482799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241340739234828:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.482865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241340739234748:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.482879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.482885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241340739234748:2049] [node 2] ICP22 created new session: [3:7587241340739234838:2049]
2025-12-24T02:11:24.482891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241340739234838:2049] [node 2] ICS09 handshake done sender: [3:7587241340739234828:2117] self: [3:7587241340739234834:0] peer: [2:7587241339405453072:0] socket: 156 qp: -1
2025-12-24T02:11:24.482898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241340739234838:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.482917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.486717Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339405453079:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.486855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339405453071:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.486869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.486877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339405453071:2107] [node 4] ICP22 created new session: [2:7587241339405453104:2107]
2025-12-24T02:11:24.486883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339405453104:2107] [node 4] ICS09 handshake done sender: [2:7587241339405453079:2110] self: [2:7587241339405453078:0] peer: [4:7587241339645876427:0] socket: 127 qp: -1
2025-12-24T02:11:24.486888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339405453104:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:24.486906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.490871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.490980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241339645876415:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.491183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861196:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.491410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635859:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.491443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.491465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635858:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.495984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861196:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.496124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340015861109:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.496321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861192:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.496340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340015861110:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.498203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234822:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.498218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241340739234822:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.494891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241339645876348:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.494911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.494917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241339645876348:2048] [node 1] ICP22 created new session: [4:7587241339645876434:2048]
2025-12-24T02:11:24.494923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241339645876434:2048] [node 1] ICS09 handshake done sender: [4:7587241339645876415:2111] self: [4:7587241339645876423:0] peer: [1:7587241338631894679:0] socket: 144 qp: -1
2025-12-24T02:11:24.494929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241339645876434:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.494950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.495218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241339645876414:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.495557Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241339645876413:2100] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56665499ns) {Configure:104(56664319ns) {ProcessConfigured:135(56299850ns) {StartInitialHandshake:157(56298850ns) {DropHandshakes:470(998ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(179ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(831ns) {SwitchToState:179(172ns)}}}}}
2025-12-24T02:11:24.496917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241339405453011:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(26075732ns) {HandleHandshakeStatus:329(26075352ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(18057507ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18032840ns max# 17954198ns}}}
2025-12-24T02:11:24.496963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241339405453011:2049] [node 3] ICP18 handshake (actor [2:7587241339405453092:2117]) is too old
2025-12-24T02:11:24.498806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340577635858:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.498849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340577635812:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.498923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453091:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.498958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453085:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.498961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339405453085:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.499048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241339645876349:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.499067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.499077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241339645876349:2049] [node 2] ICP22 created new session: [4:7587241339645876437:2049]
2025-12-24T02:11:24.499084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241339645876437:2049] [node 2] ICS09 handshake done sender: [4:7587241339645876414:2110] self: [4:7587241339645876427:0] peer: [2:7587241339405453078:0] socket: 130 qp: -1
2025-12-24T02:11:24.499090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241339645876437:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.499098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.498950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861188:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.498959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241340015861188:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.499592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340015861188:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.499666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241340015861155:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.499683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.499690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340015861155:2062] [node 6] ICP22 created new session: [5:7587241340015861200:2062]
2025-12-24T02:11:24.499696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340015861200:2062] [node 6] ICS09 handshake done sender: [5:7587241340015861188:2116] self: [5:7587241340015861156:0] peer: [6:7587241340577635860:0] socket: 128 qp: -1
2025-12-24T02:11:24.499702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340015861200:2062] [node 6] ICS10 traffic start
2025-12-24T02:11:24.499708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.503167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241339405453071:2107] [node 4] ICP18 handshake (actor [2:7587241339405453091:2116]) is too old
2025-12-24T02:11:24.506748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340577635857:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.506779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340577635777:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.506988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876439:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.507033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876440:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.510748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340577635859:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.510792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340577635820:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.510851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241338631894671:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.510965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241338631894600:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.510988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.510997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241338631894600:2049] [node 3] ICP22 created new session: [1:7587241338631894701:2049]
2025-12-24T02:11:24.511003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241338631894701:2049] [node 3] ICS09 handshake done sender: [1:7587241338631894671:2111] self: [1:7587241338631894670:0] peer: [3:7587241340739234833:0] socket: 122 qp: -1
2025-12-24T02:11:24.511010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241338631894701:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.511018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.510809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453089:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.510819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339405453089:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.514958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241340739234748:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(31983276ns) {HandleHandshakeStatus:329(31982746ns) {DropIncomingHandshake:442(580ns) DropOutgoingHandshake:457(965ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(16664316ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16641178ns max# 16555649ns}}}
2025-12-24T02:11:24.514815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876431:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.514826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241339645876431:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.515191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876439:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.515295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241339645876350:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.518774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241339645876431:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.524450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241340739234822:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.526800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241340739234815:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.526825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.526835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241340739234815:2107] [node 5] ICP22 created new session: [3:7587241340739234842:2107]
2025-12-24T02:11:24.526844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241340739234842:2107] [node 5] ICS09 handshake done sender: [3:7587241340739234822:2114] self: [3:7587241340739234821:0] peer: [5:7587241340015861198:0] socket: 153 qp: -1
2025-12-24T02:11:24.526851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241340739234842:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:24.526877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.532133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339405453085:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.531687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876440:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.531712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241339645876410:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.531730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.531739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241339645876410:2098] [node 6] ICP22 created new session: [4:7587241339645876443:2098]
2025-12-24T02:11:24.531746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241339645876443:2098] [node 6] ICS09 handshake done sender: [4:7587241339645876431:2118] self: [4:7587241339645876424:0] peer: [6:7587241340577635862:0] socket: 146 qp: -1
2025-12-24T02:11:24.531753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241339645876443:2098] [node 6] ICS10 traffic start
2025-12-24T02:11:24.531765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.534725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234824:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.534743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241340739234824:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.535676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241340739234844:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28189
2025-12-24T02:11:24.535870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241340739234827:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.536395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340015861110:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340015861192:2119] held: no
2025-12-24T02:11:24.537080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241340739234747:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.537101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.537110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241340739234747:2048] [node 1] ICP22 created new session: [3:7587241340739234846:2048]
2025-12-24T02:11:24.537116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241340739234846:2048] [node 1] ICS09 handshake done sender: [3:7587241340739234827:2116] self: [3:7587241340739234833:0] peer: [1:7587241338631894670:0] socket: 149 qp: -1
2025-12-24T02:11:24.537123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241340739234846:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.537131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.538772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241340577635812:2060] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1492) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241340577635858:2117] held: no
2025-12-24T02:11:24.539054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241338631894600:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27976413ns) {HandleHandshakeStatus:329(27975518ns) {DropIncomingHandshake:442(6623ns) DropOutgoingHandshake:457(221ns) SwitchToState:179(72ns) ProcessPendingSessionEvents:495(27881450ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27853405ns max# 27773841ns}}}
2025-12-24T02:11:24.543744Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241339645876349:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(44553299ns) {HandleHandshakeStatus:329(44552541ns) {DropIncomingHandshake:442(390ns) DropOutgoingHandshake:457(4790ns) SwitchToState:179(221ns) ProcessPendingSessionEvents:495(11727564ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11707981ns max# 11641609ns}}}
2025-12-24T02:11:24.545290Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241340577635820:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21217) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241340577635859:2118] held: no
2025-12-24T02:11:24.546765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339405453089:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.547550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339405453108:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32802
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.555217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241340739234824:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.555537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241340739234814:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.555555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.555559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241340739234814:2109] [node 4] ICP22 created new session: [3:7587241340739234849:2109]
2025-12-24T02:11:24.555565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241340739234849:2109] [node 4] ICS09 handshake done sender: [3:7587241340739234824:2115] self: [3:7587241340739234823:0] peer: [4:7587241339645876442:0] socket: 152 qp: -1
2025-12-24T02:11:24.555569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241340739234849:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:24.555576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.554920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241339645876447:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57682
2025-12-24T02:11:24.555332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241339645876439:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.555395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241339645876350:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.555411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.555421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241339645876350:2050] [node 3] ICP22 created new session: [4:7587241339645876448:2050]
2025-12-24T02:11:24.555429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241339645876448:2050] [node 3] ICS09 handshake done sender: [4:7587241339645876439:2120] self: [4:7587241339645876442:0] peer: [3:7587241340739234823:0] socket: 172 qp: -1
2025-12-24T02:11:24.554788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339405453108:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.555437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241339645876448:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.558740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339405453083:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.555446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.558764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.558771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339405453083:2112] [node 5] ICP22 created new session: [2:7587241339405453110:2112]
2025-12-24T02:11:24.558848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635864:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.558779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339405453110:2112] [node 5] ICS09 handshake done sender: [2:7587241339405453085:2113] self: [2:7587241339405453084:0] peer: [5:7587241340015861197:0] socket: 125 qp: -1
2025-12-24T02:11:24.558790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339405453110:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:24.558794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339405453087:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.558797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.558815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.558822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339405453087:2114] [node 6] ICP22 created new session: [2:7587241339405453112:2114]
2025-12-24T02:11:24.558828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339405453112:2114] [node 6] ICS09 handshake done sender: [2:7587241339405453089:2115] self: [2:7587241339405453088:0] peer: [6:7587241340577635861:0] socket: 132 qp: -1
2025-12-24T02:11:24.558833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339405453112:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:24.558842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.558899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861203:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.558938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340577635858:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.559018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340015861192:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.559096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340015861110:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.559107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.559114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340015861110:2050] [node 3] ICP22 created new session: [5:7587241340015861204:2050]
2025-12-24T02:11:24.559120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340015861204:2050] [node 3] ICS09 handshake done sender: [5:7587241340015861192:2119] self: [5:7587241340015861198:0] peer: [3:7587241340739234821:0] socket: 165 qp: -1
2025-12-24T02:11:24.559126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340015861204:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.559148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.562937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340577635812:2060] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.562952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.562957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340577635812:2060] [node 5] ICP22 created new session: [6:7587241340577635865:2060]
2025-12-24T02:11:24.562963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340577635865:2060] [node 5] ICS09 handshake done sender: [6:7587241340577635858:2117] self: [6:7587241340577635860:0] peer: [5:7587241340015861156:0] socket: 173 qp: -1
2025-12-24T02:11:24.562967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340577635865:2060] [node 5] ICS10 traffic start
2025-12-24T02:11:24.562974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.562861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861207:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.562944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861206:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.563760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340015861196:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.563846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340015861109:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.563861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.563868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340015861109:2049] [node 2] ICP22 created new session: [5:7587241340015861208:2049]
2025-12-24T02:11:24.563873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340015861208:2049] [node 2] ICS09 handshake done sender: [5:7587241340015861196:2120] self: [5:7587241340015861197:0] peer: [2:7587241339405453084:0] socket: 169 qp: -1
2025-12-24T02:11:24.563881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340015861208:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.563888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.571057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241339405453083:2112] [node 5] ICP35 event processing took too much time PendingConnection:279(36184944ns) {HandleHandshakeStatus:329(36183841ns) {DropIncomingHandshake:442(5949ns) DropOutgoingHandshake:457(104ns) SwitchToState:179(28ns) ProcessPendingSessionEvents:495(12105915ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12086757ns max# 12007023ns}}}
2025-12-24T02:11:24.574855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340015861207:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.575175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340015861177:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.577536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.577560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635868:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.577638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635869:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.575535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241339645876350:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(20015070ns) {HandleHandshakeStatus:329(20014635ns) {DropIncomingHandshake:442(274ns) DropOutgoingHandshake:457(2045ns) SwitchToState:179(98ns) ProcessPendingSessionEvents:495(15656562ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15629838ns max# 15540282ns}}}
2025-12-24T02:11:24.576614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241339645876436:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.576629Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241339645876436:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.578481Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241339645876436:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.578689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241339645876413:2100] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.578730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.578741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241339645876413:2100] [node 5] ICP22 created new session: [4:7587241339645876451:2100]
2025-12-24T02:11:24.578749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241339645876451:2100] [node 5] ICS09 handshake done sender: [4:7587241339645876436:2119] self: [4:7587241339645876425:0] peer: [5:7587241340015861210:0] socket: 138 qp: -1
2025-12-24T02:11:24.578758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241339645876451:2100] [node 5] ICS10 traffic start
2025-12-24T02:11:24.578771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.583821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340577635868:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.583894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340577635859:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.584334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340577635857:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.584449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340577635820:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.584471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.584479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340577635820:2064] [node 4] ICP22 created new session: [6:7587241340577635870:2064]
2025-12-24T02:11:24.584486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340577635870:2064] [node 4] ICS09 handshake done sender: [6:7587241340577635859:2118] self: [6:7587241340577635862:0] peer: [4:7587241339645876424:0] socket: 174 qp: -1
2025-12-24T02:11:24.584496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340577635870:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:24.584508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.586021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340577635777:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.586063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.586071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340577635777:2049] [node 2] ICP22 created new session: [6:7587241340577635872:2049]
2025-12-24T02:11:24.586078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340577635872:2049] [node 2] ICS09 handshake done sender: [6:7587241340577635857:2116] self: [6:7587241340577635861:0] peer: [2:7587241339405453088:0] socket: 170 qp: -1
2025-12-24T02:11:24.586087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340577635872:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.586095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.586335Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241340577635777:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:24.586596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340577635778:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.584428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340015861177:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21217) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340015861207:2123] held: no
2025-12-24T02:11:24.586962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241340739234832:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.586989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241340739234832:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.587906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241340739234832:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.588037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241340739234818:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.588067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.588077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241340739234818:2111] [node 6] ICP22 created new session: [3:7587241340739234852:2111]
2025-12-24T02:11:24.588085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241340739234852:2111] [node 6] ICS09 handshake done sender: [3:7587241340739234832:2120] self: [3:7587241340739234819:0] peer: [6:7587241340577635874:0] socket: 121 qp: -1
2025-12-24T02:11:24.588094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241340739234852:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:24.588103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.591573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241340577635778:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7534) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241340577635868:2121] held: no
2025-12-24T02:11:24.591163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340015861211:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54277
2025-12-24T02:11:24.591933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340015861207:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.591995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340015861177:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.592014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.592023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340015861177:2075] [node 4] ICP22 created new session: [5:7587241340015861212:2075]
2025-12-24T02:11:24.592030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340015861212:2075] [node 4] ICS09 handshake done sender: [5:7587241340015861207:2123] self: [5:7587241340015861210:0] peer: [4:7587241339645876425:0] socket: 158 qp: -1
2025-12-24T02:11:24.592039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340015861212:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:24.592049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.603117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340577635875:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41578
2025-12-24T02:11:24.603281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340577635868:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.603340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340577635778:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.603357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.603369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340577635778:2050] [node 3] ICP22 created new session: [6:7587241340577635876:2050]
2025-12-24T02:11:24.603377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340577635876:2050] [node 3] ICS09 handshake done sender: [6:7587241340577635868:2121] self: [6:7587241340577635874:0] peer: [3:7587241340739234819:0] socket: 137 qp: -1
2025-12-24T02:11:24.603386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340577635876:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:24.603410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.630844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:24.630747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:24.630844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241339405453097:2048] [node 1] ICS01 socket: 163 reason# EndOfStream
2025-12-24T02:11:24.630955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241340739234846:2048] [node 1] ICS01 socket: 149 reason# EndOfStream
2025-12-24T02:11:24.630853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241339405453010:2048] [node 1] ICP30 unregister session Session# [2:7587241339405453097:2048] VirtualId# [2:7587241339405453095:0]
2025-12-24T02:11:24.630969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241340739234747:2048] [node 1] ICP30 unregister session Session# [3:7587241340739234846:2048] VirtualId# [3:7587241340739234833:0]
2025-12-24T02:11:24.630862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241339405453097:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.630977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241340739234846:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.630905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.630930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.631038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.631074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.647397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:24.647513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241339645876434:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-12-24T02:11:24.647521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241339645876348:2048] [node 1] ICP30 unregister session Session# [4:7587241339645876434:2048] VirtualId# [4:7587241339645876423:0]
2025-12-24T02:11:24.647530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241339645876434:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.647590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.647619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.645468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:24.645566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340577635793:2048] [node 1] ICS01 socket: 115 reason# EndOfStream
2025-12-24T02:11:24.645575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340577635776:2048] [node 1] ICP30 unregister session Session# [6:7587241340577635793:2048] VirtualId# [6:7587241340577635791:0]
2025-12-24T02:11:24.645583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340577635793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.645631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.645657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.651294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:24.651380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241340015861193:2048] [node 1] ICS01 socket: 124 reason# EndOfStream
2025-12-24T02:11:24.651387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340015861108:2048] [node 1] ICP30 unregister session Session# [5:7587241340015861193:2048] VirtualId# [5:7587241340015861185:0]
2025-12-24T02:11:24.651394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241340015861193:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.651437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:24.651465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:24.658828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:24.658952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241340739234838:2049] [node 2] ICS07 socket disconnect 156 reason# EndOfStream
2025-12-24T02:11:24.658961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241340739234838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.659030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.659063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241340739234838:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:24.660584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241340739234748:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:24.660606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241340739234838:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:24.660612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241340739234748:2049] [node 2] ICP30 unregister session Session# [3:7587241340739234838:2049] VirtualId# [3:7587241340739234834:0]
2025-12-24T02:11:24.660671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.660678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241340739234748:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.660682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.661633Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:24.661732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241340015861208:2049] [node 2] ICS07 socket disconnect 169 reason# EndOfStream
2025-12-24T02:11:24.661739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241340015861208:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.661784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.661807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241340015861208:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:24.661642Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:24.661751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241339645876437:2049] [node 2] ICS07 socket disconnect 130 reason# EndOfStream
2025-12-24T02:11:24.661759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241339645876437:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.661821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.661850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241339645876437:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:24.662273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340015861109:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:24.662285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241340015861208:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:24.662290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340015861109:2049] [node 2] ICP30 unregister session Session# [5:7587241340015861208:2049] VirtualId# [5:7587241340015861197:0]
2025-12-24T02:11:24.662342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.662348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241340015861109:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.662351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.662607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241339645876349:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:24.662620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241339645876437:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:24.662626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241339645876349:2049] [node 2] ICP30 unregister session Session# [4:7587241339645876437:2049] VirtualId# [4:7587241339645876427:0]
2025-12-24T02:11:24.662678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.662684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241339645876349:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.662687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8832) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:24.662753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:24.662834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340577635872:2049] [node 2] ICS01 socket: 170 reason# EndOfStream
2025-12-24T02:11:24.662842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340577635777:2049] [node 2] ICP30 unregister session Session# [6:7587241340577635872:2049] VirtualId# [6:7587241340577635861:0]
2025-12-24T02:11:24.662848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340577635872:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.662893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:24.662917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:24.671006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241339645876349:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:24.674745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241340015861109:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:24.674786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:24.674879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241340015861204:2050] [node 3] ICS01 socket: 165 reason# EndOfStream
2025-12-24T02:11:24.674889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340015861110:2050] [node 3] ICP30 unregister session Session# [5:7587241340015861204:2050] VirtualId# [5:7587241340015861198:0]
2025-12-24T02:11:24.674897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241340015861204:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.674947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:24.674974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:24.674791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:24.674879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340577635876:2050] [node 3] ICS01 socket: 137 reason# EndOfStream
2025-12-24T02:11:24.674889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340577635778:2050] [node 3] ICP30 unregister session Session# [6:7587241340577635876:2050] VirtualId# [6:7587241340577635874:0]
2025-12-24T02:11:24.674898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340577635876:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.674949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:24.674977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:24.695670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:24.695774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340577635870:2064] [node 4] ICS01 socket: 174 reason# EndOfStream
2025-12-24T02:11:24.695787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340577635820:2064] [node 4] ICP30 unregister session Session# [6:7587241340577635870:2064] VirtualId# [6:7587241340577635862:0]
2025-12-24T02:11:24.695795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340577635870:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.695888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:24.695918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:24.696083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:24.696177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241340015861212:2075] [node 4] ICS01 socket: 158 reason# EndOfStream
2025-12-24T02:11:24.696185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340015861177:2075] [node 4] ICP30 unregister session Session# [5:7587241340015861212:2075] VirtualId# [5:7587241340015861210:0]
2025-12-24T02:11:24.696192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241340015861212:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:24.696247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:24.696270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:24.775539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339880987301:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.775554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339880987302:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:24.775603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241337868019264:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.775613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241337868019265:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.774942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241337395111383:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:24.774952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241337395111384:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:24.794867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340911896692:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.794880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340911896693:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.794891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340911896694:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.794744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340479036647:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.794800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340479036648:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.794817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340479036649:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.786885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241340634229687:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:24.786895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241340634229688:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:24.786901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241340634229689:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:24.820036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340911896709:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:24.821436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241340911896720:2066] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.822236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036659:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.822954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340479036664:2061] [node 6] ICP01 ready to work
2025-12-24T02:11:24.822939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241337395111444:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:24.823102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339880987363:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:24.823024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229748:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.827043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241340479036683:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:24.827316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241337395111450:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:24.834996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241337868019325:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:24.834723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896736:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.840321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340911896736:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.842713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339880987365:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:24.843730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241339880987369:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:24.843225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241340634229753:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.848989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987371:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.850287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229748:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.851292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036674:2070] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.851310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241340479036674:2070] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.852275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340479036674:2070] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.852403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241340479036664:2061] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.852421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.852429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340479036664:2061] [node 6] ICP22 created new session: [5:7587241340479036708:2061]
2025-12-24T02:11:24.852436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340479036708:2061] [node 6] ICS09 handshake done sender: [5:7587241340479036674:2070] self: [5:7587241340479036673:0] peer: [6:7587241340911896760:0] socket: 125 qp: -1
2025-12-24T02:11:24.852444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340479036708:2061] [node 6] ICS10 traffic start
2025-12-24T02:11:24.852481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.849094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987372:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
2025-12-24T02:11:24.849791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241339880987302:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18867607ns) {Configure:104(18866062ns) {ProcessConfigured:135(18513420ns) {StartInitialHandshake:157(18512974ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(685ns) PrepareNewSessionHandshake:477(740ns) {SwitchToState:179(429ns)}}}}}
2025-12-24T02:11:24.854773Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241340479036648:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27099676ns) {Configure:104(27098738ns) {ProcessConfigured:135(26833489ns) {StartInitialHandshake:157(26833052ns) {DropHandshakes:470(331ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(976ns) {SwitchToState:179(393ns)}}}}}
2025-12-24T02:11:24.854929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987371:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.858244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036659:2059] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.856963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241337868019325:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21894298ns) {Configure:104(21893775ns) {ProcessConfigured:135(21460431ns) {StartInitialHandshake:157(21459801ns) {DropHandshakes:470(366ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(112ns) PrepareNewSessionHandshake:477(857ns) {SwitchToState:179(485ns)}}}}}
2025-12-24T02:11:24.856306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229761:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:24.856368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229762:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:24.856607Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241339880987301:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21803420ns) {Configure:104(21802548ns) {ProcessConfigured:135(21500248ns) {StartInitialHandshake:157(21499762ns) {DropHandshakes:470(450ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(250ns) PrepareNewSessionHandshake:477(1138ns) {SwitchToState:179(751ns)}}}}}
2025-12-24T02:11:24.856768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241339880987301:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.858508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241337395111455:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:24.862774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036706:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:24.862858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229761:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.866920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019337:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.866985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019338:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.867019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019339:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.867537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019337:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.866900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896771:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.866991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896772:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.867170Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241340911896692:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44238735ns) {Configure:104(44237626ns) {ProcessConfigured:135(43906274ns) {StartInitialHandshake:157(43905645ns) {DropHandshakes:470(359ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1052ns) {SwitchToState:179(310ns)}}}}}
2025-12-24T02:11:24.874862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019338:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.874874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241340479036647:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35147949ns) {Configure:104(35146992ns) {ProcessConfigured:135(34839692ns) {StartInitialHandshake:157(34838625ns) {DropHandshakes:470(545ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(2603ns) {SwitchToState:179(872ns)}}}}}
2025-12-24T02:11:24.875713Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241339880987369:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31919616ns) {Configure:104(31919163ns) {ProcessConfigured:135(31603088ns) {StartInitialHandshake:157(31602921ns) {DropHandshakes:470(276ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(40ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(782ns) {SwitchToState:179(379ns)}}}}}
2025-12-24T02:11:24.875763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987372:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.875090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241340634229689:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35142348ns) {Configure:104(35141005ns) {ProcessConfigured:135(34677805ns) {StartInitialHandshake:157(34677540ns) {DropHandshakes:470(1148ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(160ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1305ns) {SwitchToState:179(253ns)}}}}}
2025-12-24T02:11:24.875552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241340634229753:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32247472ns) {Configure:104(32246940ns) {ProcessConfigured:135(31888899ns) {StartInitialHandshake:157(31888599ns) {DropHandshakes:470(470ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(727ns) {SwitchToState:179(138ns)}}}}}
2025-12-24T02:11:24.882990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111447:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.883021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241337395111447:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.883063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340911896720:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.883114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241340911896720:2066] [node 5] ICP35 event processing took too much time PendingConnection:279(42721657ns) {IncomingHandshake:261(42721023ns) {DropIncomingHandshake:442(124ns) PrepareNewSessionHandshake:477(1047ns) {SwitchToState:179(496ns)} GenerateSessionVirtualId:988(307ns) IssueIncomingHandshakeReply:193(42714817ns) {SwitchToState:179(1257ns)}}}
2025-12-24T02:11:24.883164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340911896736:2083] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.886718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241340634229752:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:24.886732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340911896720:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.886773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.886786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340911896720:2066] [node 5] ICP22 created new session: [6:7587241340911896774:2066]
2025-12-24T02:11:24.886794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340911896774:2066] [node 5] ICS09 handshake done sender: [6:7587241340911896736:2083] self: [6:7587241340911896760:0] peer: [5:7587241340479036673:0] socket: 126 qp: -1
2025-12-24T02:11:24.886805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340911896774:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:24.886845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.887807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241339880987363:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(64617240ns) {Configure:104(64616756ns) {ProcessConfigured:135(64192470ns) {StartInitialHandshake:157(64191905ns) {DropHandshakes:470(287ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(118ns) PrepareNewSessionHandshake:477(1346ns) {SwitchToState:179(918ns)}}}}}
2025-12-24T02:11:24.888101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229768:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:24.890793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340911896771:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.890954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036706:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.890995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340479036648:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.894852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229762:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.895099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987384:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.895429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241337868019265:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32544414ns) {Configure:104(32543905ns) {ProcessConfigured:135(32149002ns) {StartInitialHandshake:157(32148454ns) {DropHandshakes:470(489ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(932ns) {SwitchToState:179(345ns)}}}}}
2025-12-24T02:11:24.898434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111452:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.895910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019342:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.898450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241337395111452:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.898987Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987385:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
2025-12-24T02:11:24.899176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241337395111452:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.899173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987384:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.899306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241337395111450:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.899322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.899083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229770:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:24.899330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241337395111450:2109] [node 4] ICP22 created new session: [1:7587241337395111461:2109]
2025-12-24T02:11:24.899337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337395111461:2109] [node 4] ICS09 handshake done sender: [1:7587241337395111452:2110] self: [1:7587241337395111451:0] peer: [4:7587241340634229769:0] socket: 133 qp: -1
2025-12-24T02:11:24.899344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337395111461:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:24.899355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.905503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241337868019335:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:24.906739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241340479036664:2061] [node 6] ICP18 handshake (actor [5:7587241340479036659:2059]) is too old
2025-12-24T02:11:24.906795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [5:7587241340479036664:2061] [node 6] ICP35 event processing took too much time StateWork:294(48487227ns) {IncomingHandshake:261(48485301ns)}
2025-12-24T02:11:24.906724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241337868019334:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:24.906825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019339:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.906861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241337868019265:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.906743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987374:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.906769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339880987374:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.914825Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241337395111384:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(75872402ns) {Configure:104(75871430ns) {ProcessConfigured:135(75482318ns) {StartInitialHandshake:157(75481257ns) {DropHandshakes:470(704ns) {DropIncomingHandshake:442(228ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(477ns) PrepareNewSessionHandshake:477(2504ns) {SwitchToState:179(975ns)}}}}}
2025-12-24T02:11:24.914831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111456:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.914838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241337395111456:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.915033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241337395111447:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.915255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241337395111383:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.915278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.915288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241337395111383:2048] [node 2] ICP22 created new session: [1:7587241337395111466:2048]
2025-12-24T02:11:24.915295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337395111466:2048] [node 2] ICS09 handshake done sender: [1:7587241337395111447:2108] self: [1:7587241337395111446:0] peer: [2:7587241339880987380:0] socket: 130 qp: -1
2025-12-24T02:11:24.915303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337395111466:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:24.915345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.917192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019346:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.914871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987387:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
2025-12-24T02:11:24.918809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241340634229752:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31873287ns) {Configure:104(31872694ns) {ProcessConfigured:135(31479685ns) {StartInitialHandshake:157(31459178ns) {DropHandshakes:470(531ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(118ns)} GenerateSessionVirtualId:988(385ns) PrepareNewSessionHandshake:477(2261ns) {SwitchToState:179(192ns)}} IncomingHandshake:261(7689ns) {DropIncomingHandshake:442(41ns) PrepareNewSessionHandshake:477(525ns) {SwitchToState:179(495ns)} GenerateSessionVirtualId:988(112ns) IssueIncomingHandshakeReply:193(407ns)}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.918894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339880987371:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.918996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241339880987301:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.919014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.919021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339880987301:2048] [node 1] ICP22 created new session: [2:7587241339880987388:2048]
2025-12-24T02:11:24.919027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339880987388:2048] [node 1] ICS09 handshake done sender: [2:7587241339880987371:2110] self: [2:7587241339880987380:0] peer: [1:7587241337395111446:0] socket: 135 qp: -1
2025-12-24T02:11:24.919033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339880987388:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.919051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.922740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241340634229687:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.922785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241340634229687:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27864801ns) {IncomingHandshake:261(27864089ns) {DropIncomingHandshake:442(177ns) PrepareNewSessionHandshake:477(636ns) {SwitchToState:179(86ns)} GenerateSessionVirtualId:988(198ns) IssueIncomingHandshakeReply:193(27859549ns) {SwitchToState:179(1019ns)}}}
2025-12-24T02:11:24.922822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340634229762:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.921051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896777:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.922925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340911896692:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.922957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241340911896692:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32105965ns) {IncomingHandshake:261(32105429ns) {DropIncomingHandshake:442(47ns) PrepareNewSessionHandshake:477(992ns) {SwitchToState:179(440ns)} GenerateSessionVirtualId:988(270ns) IssueIncomingHandshakeReply:193(32098817ns) {SwitchToState:179(411ns)}}}
2025-12-24T02:11:24.923228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241340634229687:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.923251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.923257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241340634229687:2048] [node 1] ICP22 created new session: [4:7587241340634229773:2048]
2025-12-24T02:11:24.923263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340634229773:2048] [node 1] ICS09 handshake done sender: [4:7587241340634229762:2114] self: [4:7587241340634229769:0] peer: [1:7587241337395111451:0] socket: 150 qp: -1
2025-12-24T02:11:24.923268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340634229773:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.923284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.926736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987376:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.926756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339880987376:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.923109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019346:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.928224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339880987376:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.928366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339880987302:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.928379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.928387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339880987302:2049] [node 3] ICP22 created new session: [2:7587241339880987392:2049]
2025-12-24T02:11:24.928392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339880987392:2049] [node 3] ICS09 handshake done sender: [2:7587241339880987376:2113] self: [2:7587241339880987366:0] peer: [3:7587241337868019344:0] socket: 143 qp: -1
2025-12-24T02:11:24.928399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339880987392:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.928406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.931077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340911896777:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.931137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340911896693:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.931151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241337868019335:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25431688ns) {Configure:104(25431063ns) {ProcessConfigured:135(24718577ns) {StartInitialHandshake:157(24686273ns) {DropHandshakes:470(1155ns) {DropIncomingHandshake:442(227ns) DropOutgoingHandshake:457(280ns)} GenerateSessionVirtualId:988(543ns) PrepareNewSessionHandshake:477(998ns) {SwitchToState:179(779ns)}} IncomingHandshake:261(8287ns) {DropIncomingHandshake:442(225ns) PrepareNewSessionHandshake:477(210ns) {SwitchToState:179(155ns)} GenerateSessionVirtualId:988(255ns) IssueIncomingHandshakeReply:193(454ns)}}}}
2025-12-24T02:11:24.931348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019350:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.931524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337395111468:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8177
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.934780Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241337395111455:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(75768094ns) {Configure:104(75767361ns) {ProcessConfigured:135(75403305ns) {StartInitialHandshake:157(75402641ns) {DropHandshakes:470(464ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(180ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(1186ns) {SwitchToState:179(294ns)}}}}}
2025-12-24T02:11:24.934798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241337395111450:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:24.937264Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241337395111456:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.938241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111464:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.938250Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241337395111464:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.938743Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337395111470:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8177
2025-12-24T02:11:24.938769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241337395111444:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.938782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.938790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241337395111444:2107] [node 6] ICP22 created new session: [1:7587241337395111474:2107]
2025-12-24T02:11:24.938794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337395111469:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8177
2025-12-24T02:11:24.938797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337395111474:2107] [node 6] ICS09 handshake done sender: [1:7587241337395111456:2112] self: [1:7587241337395111445:0] peer: [6:7587241340911896776:0] socket: 137 qp: -1
2025-12-24T02:11:24.938804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337395111474:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:24.938813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.935925Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241337868019334:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28963889ns) {Configure:104(28963520ns) {ProcessConfigured:135(28581494ns) {StartInitialHandshake:157(28566229ns) {DropHandshakes:470(630ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(237ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:477(1226ns) {SwitchToState:179(835ns)}} IncomingHandshake:261(6717ns) {DropIncomingHandshake:442(164ns) PrepareNewSessionHandshake:477(185ns) {SwitchToState:179(132ns)} GenerateSessionVirtualId:988(665ns) IssueIncomingHandshakeReply:193(263ns)}}}}
2025-12-24T02:11:24.935485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241340911896720:2066] [node 5] ICP35 event processing took too much time PendingConnection:279(48633906ns) {HandleHandshakeStatus:329(48632966ns) {DropIncomingHandshake:442(1183ns) DropOutgoingHandshake:457(3313ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(28523406ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28503006ns max# 28418198ns}}}
2025-12-24T02:11:24.938909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987381:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.938921Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339880987381:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.939342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339880987374:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:24.939613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339880987381:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.939707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339880987369:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.939718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.939725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339880987369:2109] [node 6] ICP22 created new session: [2:7587241339880987397:2109]
2025-12-24T02:11:24.939731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339880987397:2109] [node 6] ICS09 handshake done sender: [2:7587241339880987381:2115] self: [2:7587241339880987370:0] peer: [6:7587241340911896779:0] socket: 170 qp: -1
2025-12-24T02:11:24.939735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339880987397:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:24.939742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.940115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339880987365:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.940133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:24.940138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339880987365:2107] [node 5] ICP22 created new session: [2:7587241339880987399:2107]
2025-12-24T02:11:24.940142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339880987399:2107] [node 5] ICS09 handshake done sender: [2:7587241339880987374:2112] self: [2:7587241339880987373:0] peer: [5:7587241340479036731:0] socket: 142 qp: -1
2025-12-24T02:11:24.940146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339880987399:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:24.940154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:24.938921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019342:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.938936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241337868019339:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.942741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241337868019265:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.942756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.942763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241337868019265:2049] [node 2] ICP22 created new session: [3:7587241337868019353:2049]
2025-12-24T02:11:24.942770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241337868019353:2049] [node 2] ICS09 handshake done sender: [3:7587241337868019339:2114] self: [3:7587241337868019344:0] peer: [2:7587241339880987366:0] socket: 159 qp: -1
2025-12-24T02:11:24.942777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241337868019353:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.942794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.943410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229776:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.947154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896782:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.947285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241339880987401:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61450
2025-12-24T02:11:24.942783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340479036648:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340479036706:2096] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.950751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896780:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.951089Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340911896777:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.951240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340911896693:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.951261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.951268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340911896693:2049] [node 2] ICP22 created new session: [6:7587241340911896783:2049]
2025-12-24T02:11:24.951273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340911896783:2049] [node 2] ICS09 handshake done sender: [6:7587241340911896777:2118] self: [6:7587241340911896779:0] peer: [2:7587241339880987370:0] socket: 171 qp: -1
2025-12-24T02:11:24.951281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340911896783:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.951291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.950814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229776:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.950874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241340634229688:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.957271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987401:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.957605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241339880987369:2109] [node 6] ICP18 handshake (actor [2:7587241339880987401:2120]) is too old
2025-12-24T02:11:24.957913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241337868019264:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.957952Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241337868019264:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(34778403ns) {IncomingHandshake:261(34777709ns) {DropIncomingHandshake:442(74ns) PrepareNewSessionHandshake:477(527ns) {SwitchToState:179(109ns)} GenerateSessionVirtualId:988(433ns) IssueIncomingHandshakeReply:193(34771166ns) {SwitchToState:179(432ns)}}}
2025-12-24T02:11:24.958871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241339880987382:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.958886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241339880987382:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.958836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896781:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.962953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340911896771:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.963105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340911896692:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.963127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.963136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340911896692:2048] [node 1] ICP22 created new session: [6:7587241340911896786:2048]
2025-12-24T02:11:24.963142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340911896786:2048] [node 1] ICS09 handshake done sender: [6:7587241340911896771:2115] self: [6:7587241340911896776:0] peer: [1:7587241337395111445:0] socket: 138 qp: -1
2025-12-24T02:11:24.963150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340911896786:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.963161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.963546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241337395111383:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:24.963938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337395111476:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8177
2025-12-24T02:11:24.963976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241337395111477:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8177
2025-12-24T02:11:24.964332Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241337395111444:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(25466030ns) {HandleHandshakeStatus:329(25465207ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(67ns) ProcessPendingSessionEvents:495(25374962ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25354659ns max# 25282996ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.964995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241337868019357:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18726
2025-12-24T02:11:24.962993Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241340634229687:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39631669ns) {HandleHandshakeStatus:329(39631147ns) {DropIncomingHandshake:442(585ns) DropOutgoingHandshake:457(5981ns) SwitchToState:179(82ns) ProcessPendingSessionEvents:495(24635108ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24606641ns max# 24519169ns}}}
2025-12-24T02:11:24.966781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111476:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.966845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241337395111464:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:24.966994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241337395111384:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.967011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:24.967017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241337395111384:2049] [node 3] ICP22 created new session: [1:7587241337395111480:2049]
2025-12-24T02:11:24.967022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241337395111480:2049] [node 3] ICS09 handshake done sender: [1:7587241337395111464:2113] self: [1:7587241337395111458:0] peer: [3:7587241337868019347:0] socket: 168 qp: -1
2025-12-24T02:11:24.967028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241337395111480:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:24.967035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:24.970942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241337868019346:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:24.974947Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340911896780:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.974992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340911896709:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.975206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036733:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.975863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896789:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.976189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340911896780:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.976300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340911896709:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.976320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.976327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340911896709:2063] [node 4] ICP22 created new session: [6:7587241340911896790:2063]
2025-12-24T02:11:24.976334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241340911896790:2063] [node 4] ICS09 handshake done sender: [6:7587241340911896780:2119] self: [6:7587241340911896788:0] peer: [4:7587241340634229754:0] socket: 146 qp: -1
2025-12-24T02:11:24.976341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241340911896790:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:24.976351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.975165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229765:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.975177Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241340634229765:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.976007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340634229765:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:24.977357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241340634229753:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.977372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:24.977378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241340634229753:2110] [node 6] ICP22 created new session: [4:7587241340634229779:2110]
2025-12-24T02:11:24.977383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340634229779:2110] [node 6] ICS09 handshake done sender: [4:7587241340634229765:2116] self: [4:7587241340634229754:0] peer: [6:7587241340911896788:0] socket: 151 qp: -1
2025-12-24T02:11:24.977389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340634229779:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:24.977396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:24.977357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241337868019264:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.977376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:24.977382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241337868019264:2048] [node 1] ICP22 created new session: [3:7587241337868019358:2048]
2025-12-24T02:11:24.977387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241337868019358:2048] [node 1] ICS09 handshake done sender: [3:7587241337868019346:2117] self: [3:7587241337868019347:0] peer: [1:7587241337395111458:0] socket: 169 qp: -1
2025-12-24T02:11:24.977392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241337868019358:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:24.977414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:24.982794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229768:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.982874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241340634229689:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.983032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019329:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.983040Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241337868019329:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:24.982829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036733:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.982881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340479036647:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.986814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241339880987382:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.986784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241337395111471:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.986797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241337395111471:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.990756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036735:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:24.991192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340479036706:2096] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.990882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241339880987363:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.990915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.991064Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241340911896692:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27754753ns) {HandleHandshakeStatus:329(27753950ns) {DropIncomingHandshake:442(415ns) DropOutgoingHandshake:457(3216ns) SwitchToState:179(235ns) ProcessPendingSessionEvents:495(27632019ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27593879ns max# 27507799ns}}}
2025-12-24T02:11:24.990924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241339880987363:2108] [node 4] ICP22 created new session: [2:7587241339880987403:2108]
2025-12-24T02:11:24.990932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241339880987403:2108] [node 4] ICS09 handshake done sender: [2:7587241339880987382:2116] self: [2:7587241339880987364:0] peer: [4:7587241340634229777:0] socket: 172 qp: -1
2025-12-24T02:11:24.990941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241339880987403:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:24.990956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.991469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340479036648:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.991482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.991487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340479036648:2049] [node 2] ICP22 created new session: [5:7587241340479036736:2049]
2025-12-24T02:11:24.991493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241340479036736:2049] [node 2] ICS09 handshake done sender: [5:7587241340479036706:2096] self: [5:7587241340479036731:0] peer: [2:7587241339880987373:0] socket: 145 qp: -1
2025-12-24T02:11:24.991507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241340479036736:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.991516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:24.990929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241340634229783:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:24.991378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340634229776:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:24.991611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241340634229688:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:24.991627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:24.991633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241340634229688:2049] [node 2] ICP22 created new session: [4:7587241340634229784:2049]
2025-12-24T02:11:24.991639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241340634229784:2049] [node 2] ICS09 handshake done sender: [4:7587241340634229776:2120] self: [4:7587241340634229777:0] peer: [2:7587241339880987364:0] socket: 173 qp: -1
2025-12-24T02:11:24.991644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241340634229784:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:24.991657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.997524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241337868019329:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:24.997653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241337868019325:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:24.997671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:24.997676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241337868019325:2107] [node 4] ICP22 created new session: [3:7587241337868019361:2107]
2025-12-24T02:11:24.997682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241337868019361:2107] [node 4] ICS09 handshake done sender: [3:7587241337868019329:2110] self: [3:7587241337868019326:0] peer: [4:7587241340634229782:0] socket: 152 qp: -1
2025-12-24T02:11:24.997687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241337868019361:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:24.997694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:24.994843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036738:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:24.994896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241340479036739:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:24.998900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241340634229689:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9801) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241340634229768:2117] held: no
2025-12-24T02:11:24.998859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036738:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.998891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340479036649:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.998956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241340479036739:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.998984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241340479036683:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:24.999137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241337395111444:2107] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:24.999421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241340911896792:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:24.999643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241340911896792:2123] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:24.999711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241340911896694:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.001000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241337395111471:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.001228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241337395111455:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.001244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.001251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241337395111455:2111] [node 5] ICP22 created new session: [1:7587241341690078779:2111]
2025-12-24T02:11:25.001256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241341690078779:2111] [node 5] ICS09 handshake done sender: [1:7587241337395111471:2117] self: [1:7587241337395111459:0] peer: [5:7587241340479036734:0] socket: 178 qp: -1
2025-12-24T02:11:25.001261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241341690078779:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.001268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.001860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340479036647:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340479036733:2117] held: no
2025-12-24T02:11:25.003343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019348:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.003356Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241337868019348:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.003437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241337868019351:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.003451Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241337868019351:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.006723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241340634229771:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.006743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241340634229771:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.007597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344774004038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:25.007931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340634229771:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.011064Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241337395111384:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:25.010848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241344929197083:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53264
2025-12-24T02:11:25.011335Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241340634229768:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.011401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241340634229689:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.011415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.011422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241340634229689:2050] [node 3] ICP22 created new session: [4:7587241344929197084:2050]
2025-12-24T02:11:25.011430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241344929197084:2050] [node 3] ICS09 handshake done sender: [4:7587241340634229768:2117] self: [4:7587241340634229782:0] peer: [3:7587241337868019326:0] socket: 154 qp: -1
2025-12-24T02:11:25.011438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241344929197084:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.011446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.015071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340479036733:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.015178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340479036647:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.015198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.015206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340479036647:2048] [node 1] ICP22 created new session: [5:7587241344774004040:2048]
2025-12-24T02:11:25.015216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344774004040:2048] [node 1] ICS09 handshake done sender: [5:7587241340479036733:2117] self: [5:7587241340479036734:0] peer: [1:7587241337395111459:0] socket: 179 qp: -1
2025-12-24T02:11:25.015223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344774004040:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.015235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.015691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241337868019348:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.015892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241337868019335:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.015911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.015917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241337868019335:2108] [node 5] ICP22 created new session: [3:7587241342162986661:2108]
2025-12-24T02:11:25.015923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342162986661:2108] [node 5] ICS09 handshake done sender: [3:7587241337868019348:2118] self: [3:7587241337868019343:0] peer: [5:7587241340479036740:0] socket: 121 qp: -1
2025-12-24T02:11:25.015929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342162986661:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:25.015936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.019131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344774004039:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.019347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340479036649:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9801) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340479036738:2119] held: no
2025-12-24T02:11:25.020248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344774004042:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19279
2025-12-24T02:11:25.019696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241340634229752:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.019729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.019737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241340634229752:2109] [node 5] ICP22 created new session: [4:7587241344929197086:2109]
2025-12-24T02:11:25.019745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241344929197086:2109] [node 5] ICS09 handshake done sender: [4:7587241340634229771:2119] self: [4:7587241340634229767:0] peer: [5:7587241340479036741:0] socket: 187 qp: -1
2025-12-24T02:11:25.019752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241344929197086:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:25.019764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.024199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241337868019351:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.024913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241337868019334:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.024934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.024942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241337868019334:2109] [node 6] ICP22 created new session: [3:7587241342162986664:2109]
2025-12-24T02:11:25.024949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342162986664:2109] [node 6] ICS09 handshake done sender: [3:7587241337868019351:2120] self: [3:7587241337868019345:0] peer: [6:7587241340911896793:0] socket: 183 qp: -1
2025-12-24T02:11:25.024956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342162986664:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:25.024964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.024027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241345206864090:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29194
2025-12-24T02:11:25.025646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241340911896792:2123] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.025748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241340911896694:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.025763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.025769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241340911896694:2050] [node 3] ICP22 created new session: [6:7587241345206864091:2050]
2025-12-24T02:11:25.025775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241345206864091:2050] [node 3] ICS09 handshake done sender: [6:7587241340911896792:2123] self: [6:7587241340911896793:0] peer: [3:7587241337868019345:0] socket: 189 qp: -1
2025-12-24T02:11:25.025781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241345206864091:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.025796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.026779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241340479036683:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4304) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241340479036739:2120] held: no
2025-12-24T02:11:25.027122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340479036739:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.027167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241340479036738:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.027234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340479036683:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.027249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.027256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340479036683:2075] [node 4] ICP22 created new session: [5:7587241344774004043:2075]
2025-12-24T02:11:25.027262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344774004043:2075] [node 4] ICS09 handshake done sender: [5:7587241340479036739:2120] self: [5:7587241340479036741:0] peer: [4:7587241340634229767:0] socket: 139 qp: -1
2025-12-24T02:11:25.027268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344774004043:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:25.027282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.027573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241340479036649:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.027587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.027593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241340479036649:2050] [node 3] ICP22 created new session: [5:7587241344774004045:2050]
2025-12-24T02:11:25.027599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344774004045:2050] [node 3] ICS09 handshake done sender: [5:7587241340479036738:2119] self: [5:7587241340479036740:0] peer: [3:7587241337868019343:0] socket: 188 qp: -1
2025-12-24T02:11:25.027605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344774004045:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.027616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.048401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:25.048507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241339880987388:2048] [node 1] ICS01 socket: 135 reason# EndOfStream
2025-12-24T02:11:25.048517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241339880987301:2048] [node 1] ICP30 unregister session Session# [2:7587241339880987388:2048] VirtualId# [2:7587241339880987380:0]
2025-12-24T02:11:25.048525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241339880987388:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.048560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.048581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.050753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:25.050837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241337868019358:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-12-24T02:11:25.050844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241337868019264:2048] [node 1] ICP30 unregister session Session# [3:7587241337868019358:2048] VirtualId# [3:7587241337868019347:0]
2025-12-24T02:11:25.050849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241337868019358:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.050904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.050930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.057506Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:25.057608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241340911896786:2048] [node 1] ICS07 socket disconnect 138 reason# EndOfStream
2025-12-24T02:11:25.057615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340911896786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.057670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.057698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241340911896786:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.058258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241340911896692:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.058272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340911896786:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.058278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340911896692:2048] [node 1] ICP30 unregister session Session# [6:7587241340911896786:2048] VirtualId# [6:7587241340911896776:0]
2025-12-24T02:11:25.058333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.058340Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241340911896692:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.058344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.058873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:25.058955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241344774004040:2048] [node 1] ICS01 socket: 179 reason# EndOfStream
2025-12-24T02:11:25.058962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340479036647:2048] [node 1] ICP30 unregister session Session# [5:7587241344774004040:2048] VirtualId# [5:7587241340479036734:0]
2025-12-24T02:11:25.058969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241344774004040:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.059023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.059056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.059156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:25.059234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241340634229773:2048] [node 1] ICS07 socket disconnect 150 reason# EndOfStream
2025-12-24T02:11:25.059239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241340634229773:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.059280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.059301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241340634229773:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.059725Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241340634229687:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.059738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241340634229773:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.059743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241340634229687:2048] [node 1] ICP30 unregister session Session# [4:7587241340634229773:2048] VirtualId# [4:7587241340634229769:0]
2025-12-24T02:11:25.059792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.059798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241340634229687:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.059800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61727) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.066750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:25.066837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241337868019353:2049] [node 2] ICS01 socket: 159 reason# EndOfStream
2025-12-24T02:11:25.066845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241337868019265:2049] [node 2] ICP30 unregister session Session# [3:7587241337868019353:2049] VirtualId# [3:7587241337868019344:0]
2025-12-24T02:11:25.066852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241337868019353:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.066911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.066945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.066986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241340911896692:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.071047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241340634229687:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.074864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:25.074962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340911896783:2049] [node 2] ICS01 socket: 171 reason# EndOfStream
2025-12-24T02:11:25.074971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340911896693:2049] [node 2] ICP30 unregister session Session# [6:7587241340911896783:2049] VirtualId# [6:7587241340911896779:0]
2025-12-24T02:11:25.074978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340911896783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.075029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.075052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.078461Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:25.078551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241340479036736:2049] [node 2] ICS01 socket: 145 reason# EndOfStream
2025-12-24T02:11:25.078558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340479036648:2049] [node 2] ICP30 unregister session Session# [5:7587241340479036736:2049] VirtualId# [5:7587241340479036731:0]
2025-12-24T02:11:25.078565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241340479036736:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.078612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.078635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.078742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:25.078836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241340634229784:2049] [node 2] ICS07 socket disconnect 173 reason# EndOfStream
2025-12-24T02:11:25.078842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241340634229784:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.078899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.078924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241340634229784:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.079373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241340634229688:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2800) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.079387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241340634229784:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.079392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241340634229688:2049] [node 2] ICP30 unregister session Session# [4:7587241340634229784:2049] VirtualId# [4:7587241340634229777:0]
2025-12-24T02:11:25.079443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.079450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241340634229688:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2800) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.079454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2800) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.090745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:25.090867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241344774004045:2050] [node 3] ICS01 socket: 188 reason# EndOfStream
2025-12-24T02:11:25.090876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241340479036649:2050] [node 3] ICP30 unregister session Session# [5:7587241344774004045:2050] VirtualId# [5:7587241340479036740:0]
2025-12-24T02:11:25.090883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241344774004045:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.090929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:25.090952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:25.090986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:25.091051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241345206864091:2050] [node 3] ICS01 socket: 189 reason# EndOfStream
2025-12-24T02:11:25.091056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340911896694:2050] [node 3] ICP30 unregister session Session# [6:7587241345206864091:2050] VirtualId# [6:7587241340911896793:0]
2025-12-24T02:11:25.091061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241345206864091:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.091085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:25.091103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:25.114844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:25.114948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340911896790:2063] [node 4] ICS01 socket: 146 reason# EndOfStream
2025-12-24T02:11:25.114957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340911896709:2063] [node 4] ICP30 unregister session Session# [6:7587241340911896790:2063] VirtualId# [6:7587241340911896788:0]
2025-12-24T02:11:25.114965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340911896790:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.115017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:25.115043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:25.130908Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:25.131011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241340911896774:2066] [node 5] ICS07 socket disconnect 126 reason# EndOfStream
2025-12-24T02:11:25.131018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241340911896774:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.131058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:25.131078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241340911896774:2066] [node 5] ICS15 start handshake
2025-12-24T02:11:25.131515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241340911896720:2066] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20299) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.131529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241340911896774:2066] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.131534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241340911896720:2066] [node 5] ICP30 unregister session Session# [6:7587241340911896774:2066] VirtualId# [6:7587241340911896760:0]
2025-12-24T02:11:25.131584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:25.131590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241340911896720:2066] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20299) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.131593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20299) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:25.170835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241341691740071:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.170855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241341691740072:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:25.170915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343500227870:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:25.170923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343500227871:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:25.175020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241343923520427:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.175037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241343923520428:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.175052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241343923520429:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.186787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241344150469843:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.186802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241344150469844:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.186807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241344150469845:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.182850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241342843686855:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.182861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241342843686856:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.214718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241343792895785:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.214764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241343792895786:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.214774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241343792895787:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.272151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241343792895847:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:25.266923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241344150469905:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:25.270926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241341691740133:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:25.271032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241341691740134:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.276823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469909:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.278442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520488:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.278783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343500227933:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:25.278946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343500227936:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:25.279535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343500227934:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:11:25.279909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343500227945:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:25.280774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520489:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.282829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241341691740145:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.288171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227934:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.287087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241343923520488:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.287125Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520490:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.287256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241343923520492:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:25.287695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241343923520492:2111] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.291899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241342843686915:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.295005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343500227951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61997
2025-12-24T02:11:25.295000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241343923520490:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.295346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241343923520427:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.295230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241344150469907:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.295247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241344150469907:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.295339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241342843686915:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.295392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241342843686855:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.298708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895851:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.302709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227938:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.302721Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343500227938:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.302832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241343923520489:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.303145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241343923520428:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.310141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241344150469907:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.314920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241344150469905:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.314942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.314949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241344150469905:2107] [node 4] ICP22 created new session: [5:7587241344150469911:2107]
2025-12-24T02:11:25.314956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344150469911:2107] [node 4] ICS09 handshake done sender: [5:7587241344150469907:2109] self: [5:7587241344150469906:0] peer: [4:7587241343923520493:0] socket: 126 qp: -1
2025-12-24T02:11:25.314964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344150469911:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:25.314976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.318712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520496:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.319684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241341691740072:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19770718ns) {Configure:104(19769772ns) {ProcessConfigured:135(19241321ns) {StartInitialHandshake:157(19240789ns) {DropHandshakes:470(650ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1389ns) {SwitchToState:179(433ns)}}}}}
2025-12-24T02:11:25.320389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241343923520488:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.320451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241343923520492:2111] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.320463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.320469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241343923520492:2111] [node 5] ICP22 created new session: [4:7587241343923520498:2111]
2025-12-24T02:11:25.320475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241343923520498:2111] [node 5] ICS09 handshake done sender: [4:7587241343923520488:2108] self: [4:7587241343923520493:0] peer: [5:7587241344150469906:0] socket: 132 qp: -1
2025-12-24T02:11:25.320482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241343923520498:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.320500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.320621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520497:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.320792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241341691740136:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.320800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241341691740136:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.321276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241341691740146:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43780
2025-12-24T02:11:25.321594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241344150469909:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.321776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241344150469914:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:25.322150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241344150469914:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.322373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520500:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.318987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227932:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.319001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343500227932:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.322772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343500227938:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.323431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343500227932:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.323568Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241342843686920:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35157
2025-12-24T02:11:25.323728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343500227871:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.323742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.323747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343500227871:2049] [node 3] ICP22 created new session: [1:7587241343500227954:2049]
2025-12-24T02:11:25.323752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343500227954:2049] [node 3] ICS09 handshake done sender: [1:7587241343500227932:2108] self: [1:7587241343500227930:0] peer: [3:7587241342843686919:0] socket: 138 qp: -1
2025-12-24T02:11:25.323757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343500227954:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:25.323763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.323962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227951:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.323969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343500227933:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.323976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.323993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343500227933:2107] [node 4] ICP22 created new session: [1:7587241343500227956:2107]
2025-12-24T02:11:25.324000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343500227956:2107] [node 4] ICS09 handshake done sender: [1:7587241343500227938:2111] self: [1:7587241343500227937:0] peer: [4:7587241343923520494:0] socket: 139 qp: -1
2025-12-24T02:11:25.324004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343500227956:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:25.324009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.324225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241343500227871:2049] [node 3] ICP18 handshake (actor [1:7587241343500227951:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.324649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241342843686921:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35157
2025-12-24T02:11:25.324669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241342843686915:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.324726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241342843686855:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.324739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.324744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241342843686855:2048] [node 1] ICP22 created new session: [3:7587241342843686922:2048]
2025-12-24T02:11:25.324749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342843686922:2048] [node 1] ICS09 handshake done sender: [3:7587241342843686915:2107] self: [3:7587241342843686919:0] peer: [1:7587241343500227930:0] socket: 143 qp: -1
2025-12-24T02:11:25.324754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342843686922:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.324759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.322831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241341691740136:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.322946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241341691740133:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.322956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.322962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241341691740133:2107] [node 4] ICP22 created new session: [2:7587241341691740152:2107]
2025-12-24T02:11:25.322968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241341691740152:2107] [node 4] ICS09 handshake done sender: [2:7587241341691740136:2109] self: [2:7587241341691740135:0] peer: [4:7587241343923520495:0] socket: 128 qp: -1
2025-12-24T02:11:25.322974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241341691740152:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:25.322983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.325580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241341691740146:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.326609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241341691740071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.326825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241343792895851:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.326861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241343792895849:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.326864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241343792895849:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.327340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895852:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.327714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241343792895786:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.328068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241343792895849:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.328417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241343792895847:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.328427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.328431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241343792895847:2107] [node 5] ICP22 created new session: [6:7587241343792895855:2107]
2025-12-24T02:11:25.328435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241343792895855:2107] [node 5] ICS09 handshake done sender: [6:7587241343792895849:2109] self: [6:7587241343792895848:0] peer: [5:7587241344150469915:0] socket: 130 qp: -1
2025-12-24T02:11:25.328440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241343792895855:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:25.328445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.327861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469916:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.327878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469917:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.328220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241344150469909:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.328256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241344150469914:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.328264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.328286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241344150469914:2111] [node 6] ICP22 created new session: [5:7587241344150469918:2111]
2025-12-24T02:11:25.328292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344150469918:2111] [node 6] ICS09 handshake done sender: [5:7587241344150469909:2110] self: [5:7587241344150469915:0] peer: [6:7587241343792895848:0] socket: 135 qp: -1
2025-12-24T02:11:25.328296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344150469918:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:25.328302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.330880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227941:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.330888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343500227941:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.336259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241342843686926:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:25.336381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241342843686927:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:25.338964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241341691740138:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.338978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241341691740138:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.338952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241342843686935:2115] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.342103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241341691740138:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.342228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241341691740134:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.342239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.342270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241341691740134:2108] [node 6] ICP22 created new session: [2:7587241341691740158:2108]
2025-12-24T02:11:25.342277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241341691740158:2108] [node 6] ICS09 handshake done sender: [2:7587241341691740138:2110] self: [2:7587241341691740137:0] peer: [6:7587241343792895854:0] socket: 129 qp: -1
2025-12-24T02:11:25.342283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241341691740158:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:25.342291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.342618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241341691740157:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43780
2025-12-24T02:11:25.344251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343500227941:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.344363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343500227870:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.344382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.344388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343500227870:2048] [node 2] ICP22 created new session: [1:7587241343500227961:2048]
2025-12-24T02:11:25.344395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343500227961:2048] [node 2] ICS09 handshake done sender: [1:7587241343500227941:2113] self: [1:7587241343500227940:0] peer: [2:7587241341691740155:0] socket: 140 qp: -1
2025-12-24T02:11:25.344400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343500227961:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:25.344407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.345330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895857:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.345346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895858:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.344900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241341691740146:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.344960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241341691740071:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.344969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.344974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241341691740071:2048] [node 1] ICP22 created new session: [2:7587241341691740161:2048]
2025-12-24T02:11:25.344980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241341691740161:2048] [node 1] ICS09 handshake done sender: [2:7587241341691740146:2113] self: [2:7587241341691740155:0] peer: [1:7587241343500227940:0] socket: 144 qp: -1
2025-12-24T02:11:25.344985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241341691740161:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.344990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.344962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469920:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.344995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469921:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.346302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241344150469916:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.346789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241344150469843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.345410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241343923520490:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.349173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241343792895852:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.349186Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241343792895851:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.350724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227944:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.350738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343500227944:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.351741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469923:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.351842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241344150469921:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.352230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241344150469844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.352913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343500227944:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.353087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343500227936:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.353101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.353107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343500227936:2110] [node 5] ICP22 created new session: [1:7587241343500227964:2110]
2025-12-24T02:11:25.353112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343500227964:2110] [node 5] ICS09 handshake done sender: [1:7587241343500227944:2114] self: [1:7587241343500227943:0] peer: [5:7587241344150469922:0] socket: 159 qp: -1
2025-12-24T02:11:25.353118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343500227964:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:25.353125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.353532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469925:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.353544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241344150469916:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.353898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241344150469843:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.353910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.353916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241344150469843:2048] [node 1] ICP22 created new session: [5:7587241344150469926:2048]
2025-12-24T02:11:25.353921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344150469926:2048] [node 1] ICS09 handshake done sender: [5:7587241344150469916:2112] self: [5:7587241344150469922:0] peer: [1:7587241343500227943:0] socket: 161 qp: -1
2025-12-24T02:11:25.353926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344150469926:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.353931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.354167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241344150469921:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.352358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241341691740150:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.352368Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241341691740150:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.353627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241341691740150:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.353733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241341691740145:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.353742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.353766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241341691740145:2112] [node 5] ICP22 created new session: [2:7587241341691740164:2112]
2025-12-24T02:11:25.353771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241341691740164:2112] [node 5] ICS09 handshake done sender: [2:7587241341691740150:2115] self: [2:7587241341691740149:0] peer: [5:7587241344150469924:0] socket: 170 qp: -1
2025-12-24T02:11:25.354227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241341691740164:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:25.354233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.354578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241344150469844:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.354590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.354595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241344150469844:2049] [node 2] ICP22 created new session: [5:7587241344150469928:2049]
2025-12-24T02:11:25.354599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344150469928:2049] [node 2] ICS09 handshake done sender: [5:7587241344150469921:2115] self: [5:7587241344150469924:0] peer: [2:7587241341691740149:0] socket: 171 qp: -1
2025-12-24T02:11:25.354603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344150469928:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.354609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:25.354751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241343923520489:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.355056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241343792895786:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.355072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.355080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241343792895786:2049] [node 2] ICP22 created new session: [6:7587241343792895859:2049]
2025-12-24T02:11:25.355086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241343792895859:2049] [node 2] ICS09 handshake done sender: [6:7587241343792895851:2110] self: [6:7587241343792895854:0] peer: [2:7587241341691740137:0] socket: 133 qp: -1
2025-12-24T02:11:25.355091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241343792895859:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.355099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:25.355851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241343792895785:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.356002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241343923520427:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.356016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.356022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241343923520427:2048] [node 1] ICP22 created new session: [4:7587241343923520501:2048]
2025-12-24T02:11:25.356028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241343923520501:2048] [node 1] ICS09 handshake done sender: [4:7587241343923520490:2110] self: [4:7587241343923520494:0] peer: [1:7587241343500227937:0] socket: 141 qp: -1
2025-12-24T02:11:25.356033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241343923520501:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.356040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.356270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241343923520428:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.356277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.356281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241343923520428:2049] [node 2] ICP22 created new session: [4:7587241343923520503:2049]
2025-12-24T02:11:25.356287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241343923520503:2049] [node 2] ICS09 handshake done sender: [4:7587241343923520489:2109] self: [4:7587241343923520495:0] peer: [2:7587241341691740135:0] socket: 136 qp: -1
2025-12-24T02:11:25.356290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241343923520503:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.356295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:25.356872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241342843686921:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.357423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241342843686856:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.357676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520505:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.358906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343500227947:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.358915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343500227947:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.360460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241344150469920:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.360842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241344150469845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.361397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343500227947:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.361476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343500227945:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.361488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.361493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343500227945:2112] [node 6] ICP22 created new session: [1:7587241343500227967:2112]
2025-12-24T02:11:25.361498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343500227967:2112] [node 6] ICS09 handshake done sender: [1:7587241343500227947:2115] self: [1:7587241343500227946:0] peer: [6:7587241343792895861:0] socket: 160 qp: -1
2025-12-24T02:11:25.361503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343500227967:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:25.361509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.359889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241341691740148:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.359899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241341691740148:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.360972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241341691740148:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.361036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241341691740072:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.361061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.361089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241341691740072:2049] [node 3] ICP22 created new session: [2:7587241341691740167:2049]
2025-12-24T02:11:25.361095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241341691740167:2049] [node 3] ICS09 handshake done sender: [2:7587241341691740148:2114] self: [2:7587241341691740147:0] peer: [3:7587241342843686941:0] socket: 157 qp: -1
2025-12-24T02:11:25.361101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241341691740167:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:25.361108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.362881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895862:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.362935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241342843686929:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.362949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241342843686929:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.366014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241343792895857:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.366040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241343792895852:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.364488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241342843686942:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35157
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.366891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241344150469931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26216
2025-12-24T02:11:25.366126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241343792895785:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.366139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.366146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241343792895785:2048] [node 1] ICP22 created new session: [6:7587241343792895863:2048]
2025-12-24T02:11:25.366152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241343792895863:2048] [node 1] ICS09 handshake done sender: [6:7587241343792895852:2111] self: [6:7587241343792895861:0] peer: [1:7587241343500227946:0] socket: 145 qp: -1
2025-12-24T02:11:25.366157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241343792895863:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.366168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.366798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241343792895787:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.370918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241342843686929:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.375669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241343923520505:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.376076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241343923520429:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.374876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241342843686926:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.374893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.374900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241342843686926:2111] [node 5] ICP22 created new session: [3:7587241342843686944:2111]
2025-12-24T02:11:25.374907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342843686944:2111] [node 5] ICS09 handshake done sender: [3:7587241342843686929:2113] self: [3:7587241342843686928:0] peer: [5:7587241344150469930:0] socket: 163 qp: -1
2025-12-24T02:11:25.374914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342843686944:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.374924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.375197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241342843686931:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.375201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241342843686931:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.375610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241342843686921:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.376347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241344150469920:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.376415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241344150469845:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.376427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.376465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241344150469845:2050] [node 3] ICP22 created new session: [5:7587241344150469932:2050]
2025-12-24T02:11:25.376470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241344150469932:2050] [node 3] ICS09 handshake done sender: [5:7587241344150469920:2114] self: [5:7587241344150469930:0] peer: [3:7587241342843686928:0] socket: 165 qp: -1
2025-12-24T02:11:25.376476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241344150469932:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.376483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.376956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241342843686937:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.376963Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241342843686937:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.377420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241342843686856:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.377430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.377452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241342843686856:2049] [node 2] ICP22 created new session: [3:7587241342843686947:2049]
2025-12-24T02:11:25.377457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342843686947:2049] [node 2] ICS09 handshake done sender: [3:7587241342843686921:2110] self: [3:7587241342843686941:0] peer: [2:7587241341691740147:0] socket: 158 qp: -1
2025-12-24T02:11:25.377463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342843686947:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.377468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.378133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520507:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.378260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241342843686931:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.378365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241342843686937:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.378519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241343923520505:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.378563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241343923520429:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.378573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.378600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241343923520429:2050] [node 3] ICP22 created new session: [4:7587241343923520508:2050]
2025-12-24T02:11:25.378605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241343923520508:2050] [node 3] ICS09 handshake done sender: [4:7587241343923520505:2115] self: [4:7587241343923520506:0] peer: [3:7587241342843686936:0] socket: 176 qp: -1
2025-12-24T02:11:25.378610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241343923520508:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.378616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.378959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241343792895866:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17774
2025-12-24T02:11:25.378807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241342843686927:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.378822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.378844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241342843686927:2112] [node 6] ICP22 created new session: [3:7587241342843686951:2112]
2025-12-24T02:11:25.378849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342843686951:2112] [node 6] ICS09 handshake done sender: [3:7587241342843686931:2114] self: [3:7587241342843686930:0] peer: [6:7587241343792895865:0] socket: 164 qp: -1
2025-12-24T02:11:25.378854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342843686951:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:25.378859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.379060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241342843686935:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.379066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.379077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241342843686935:2115] [node 4] ICP22 created new session: [3:7587241342843686953:2115]
2025-12-24T02:11:25.379081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241342843686953:2115] [node 4] ICS09 handshake done sender: [3:7587241342843686937:2116] self: [3:7587241342843686936:0] peer: [4:7587241343923520506:0] socket: 175 qp: -1
2025-12-24T02:11:25.379085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241342843686953:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:25.379089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.379352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241343792895857:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.379421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241343792895787:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.379434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.379440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241343792895787:2050] [node 3] ICP22 created new session: [6:7587241343792895867:2050]
2025-12-24T02:11:25.379445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241343792895867:2050] [node 3] ICS09 handshake done sender: [6:7587241343792895857:2112] self: [6:7587241343792895865:0] peer: [3:7587241342843686930:0] socket: 166 qp: -1
2025-12-24T02:11:25.379451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241343792895867:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.379458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.415107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241343792895870:2116] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.418931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520510:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.427139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241343923520510:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.427341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241343923520512:2118] [node 6] ICP01 ready to work
2025-12-24T02:11:25.427752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241343923520512:2118] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.430910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241343792895872:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.430922Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241343792895872:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.431824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241343923520514:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11783
2025-12-24T02:11:25.434953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241343792895872:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.435070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241343792895870:2116] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.435085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.435092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241343792895870:2116] [node 4] ICP22 created new session: [6:7587241343792895875:2116]
2025-12-24T02:11:25.435098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241343792895875:2116] [node 4] ICS09 handshake done sender: [6:7587241343792895872:2117] self: [6:7587241343792895871:0] peer: [4:7587241343923520513:0] socket: 187 qp: -1
2025-12-24T02:11:25.435105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241343792895875:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:25.435113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.439340Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241343923520510:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.439425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241343923520512:2118] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.439441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.439474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241343923520512:2118] [node 6] ICP22 created new session: [4:7587241343923520515:2118]
2025-12-24T02:11:25.439479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241343923520515:2118] [node 6] ICS09 handshake done sender: [4:7587241343923520510:2117] self: [4:7587241343923520513:0] peer: [6:7587241343792895871:0] socket: 188 qp: -1
2025-12-24T02:11:25.439487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241343923520515:2118] [node 6] ICS10 traffic start
2025-12-24T02:11:25.439497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.579189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:25.579296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241341691740161:2048] [node 1] ICS01 socket: 144 reason# EndOfStream
2025-12-24T02:11:25.579308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241341691740071:2048] [node 1] ICP30 unregister session Session# [2:7587241341691740161:2048] VirtualId# [2:7587241341691740155:0]
2025-12-24T02:11:25.579316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241341691740161:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.579350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.579372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.590775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:25.590880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241342843686922:2048] [node 1] ICS07 socket disconnect 143 reason# EndOfStream
2025-12-24T02:11:25.590890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241342843686922:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.590936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.590958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241342843686922:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.591275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241342843686855:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.591284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241342843686922:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.591289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241342843686855:2048] [node 1] ICP30 unregister session Session# [3:7587241342843686922:2048] VirtualId# [3:7587241342843686919:0]
2025-12-24T02:11:25.591313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.591321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241342843686855:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.591325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.598335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:25.598419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241343923520501:2048] [node 1] ICS07 socket disconnect 141 reason# EndOfStream
2025-12-24T02:11:25.598425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241343923520501:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.598473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.598495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241343923520501:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.596113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:25.596204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241343792895863:2048] [node 1] ICS07 socket disconnect 145 reason# EndOfStream
2025-12-24T02:11:25.596212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241343792895863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.596248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.596266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241343792895863:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.596536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241343792895785:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.596545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241343792895863:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.596550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241343792895785:2048] [node 1] ICP30 unregister session Session# [6:7587241343792895863:2048] VirtualId# [6:7587241343792895861:0]
2025-12-24T02:11:25.596597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.596602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241343792895785:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.596604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.598740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:25.598829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241344150469926:2048] [node 1] ICS07 socket disconnect 161 reason# EndOfStream
2025-12-24T02:11:25.598839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241344150469926:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.598895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:25.598867Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241343923520427:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.598882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241343923520501:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.598888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241343923520427:2048] [node 1] ICP30 unregister session Session# [4:7587241343923520501:2048] VirtualId# [4:7587241343923520494:0]
2025-12-24T02:11:25.598931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.598935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241343923520427:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.598938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.598919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241344150469926:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:25.599277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241344150469843:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.599283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241344150469926:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.599288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241344150469843:2048] [node 1] ICP30 unregister session Session# [5:7587241344150469926:2048] VirtualId# [5:7587241344150469922:0]
2025-12-24T02:11:25.599335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:25.599339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241344150469843:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.599341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11762) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.606744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241342843686855:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.607787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:25.607898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241342843686947:2049] [node 2] ICS07 socket disconnect 158 reason# EndOfStream
2025-12-24T02:11:25.607906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241342843686947:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.607945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.607961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241342843686947:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.610108Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241342843686856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:25.610115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241342843686947:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.610333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241342843686856:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.610341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241342843686947:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.610346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241342843686856:2049] [node 2] ICP30 unregister session Session# [3:7587241342843686947:2049] VirtualId# [3:7587241342843686941:0]
2025-12-24T02:11:25.610392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.610399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241342843686856:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.610401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.610743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241344150469843:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.610794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241343923520427:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.614722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241343792895785:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:25.616291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:25.616376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241343792895859:2049] [node 2] ICS07 socket disconnect 133 reason# EndOfStream
2025-12-24T02:11:25.616383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241343792895859:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.616419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.616439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241343792895859:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.616674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241343792895786:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.616681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241343792895859:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.616686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241343792895786:2049] [node 2] ICP30 unregister session Session# [6:7587241343792895859:2049] VirtualId# [6:7587241343792895854:0]
2025-12-24T02:11:25.616732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.616738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241343792895786:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.616740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.618741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:25.618838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241343923520503:2049] [node 2] ICS07 socket disconnect 136 reason# EndOfStream
2025-12-24T02:11:25.618844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241343923520503:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.618894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.618917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241343923520503:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.618976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:25.619060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241344150469928:2049] [node 2] ICS07 socket disconnect 171 reason# EndOfStream
2025-12-24T02:11:25.619079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241344150469928:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.619128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:25.619147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241344150469928:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:25.619304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241343923520428:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.619317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241343923520503:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.619322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241343923520428:2049] [node 2] ICP30 unregister session Session# [4:7587241343923520503:2049] VirtualId# [4:7587241343923520495:0]
2025-12-24T02:11:25.619376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.619382Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241343923520428:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.619384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.619552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241344150469844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.619563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241344150469928:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.619567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241344150469844:2049] [node 2] ICP30 unregister session Session# [5:7587241344150469928:2049] VirtualId# [5:7587241344150469924:0]
2025-12-24T02:11:25.619612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:25.619617Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241344150469844:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.619619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.626722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241343792895786:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:25.630750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241344150469844:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:25.630755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241343923520428:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:25.633310Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:25.633404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241344150469932:2050] [node 3] ICS07 socket disconnect 165 reason# EndOfStream
2025-12-24T02:11:25.633411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241344150469932:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.633459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:25.633478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241344150469932:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:25.633809Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241344150469845:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.633821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241344150469932:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.633825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241344150469845:2050] [node 3] ICP30 unregister session Session# [5:7587241344150469932:2050] VirtualId# [5:7587241344150469930:0]
2025-12-24T02:11:25.633868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:25.633873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241344150469845:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.633876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.634735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:25.634809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241343792895867:2050] [node 3] ICS07 socket disconnect 166 reason# EndOfStream
2025-12-24T02:11:25.634815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241343792895867:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.634852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:25.634873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241343792895867:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:25.635117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241343792895787:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:25.635123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241343792895867:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:25.635128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241343792895787:2050] [node 3] ICP30 unregister session Session# [6:7587241343792895867:2050] VirtualId# [6:7587241343792895865:0]
2025-12-24T02:11:25.635172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:25.635178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241343792895787:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.635181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21897) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:25.646796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:25.646882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241343792895875:2116] [node 4] ICS01 socket: 187 reason# EndOfStream
2025-12-24T02:11:25.646892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241343792895870:2116] [node 4] ICP30 unregister session Session# [6:7587241343792895875:2116] VirtualId# [6:7587241343792895871:0]
2025-12-24T02:11:25.646899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241343792895875:2116] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:25.646933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:25.646961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:25.647071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241343792895787:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:25.686827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241344103788503:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.686836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241344103788504:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.690952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343940085129:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:25.698776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241343290811894:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.690962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343940085130:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:25.698788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241343290811895:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:25.700019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345132647983:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.700028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345132647984:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.700033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345132647985:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.706906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241341277628549:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.706926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241341277628550:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.706932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241341277628551:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.722850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241342562914135:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:25.722863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241342562914136:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:25.722869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241342562914137:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:25.761436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241341277628597:2094] [node 6] ICP01 ready to work
2025-12-24T02:11:25.762113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241341277628602:2096] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.762735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914163:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.764107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241342562914177:2080] [node 4] ICP01 ready to work
2025-12-24T02:11:25.763570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343940085190:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.764764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788564:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
2025-12-24T02:11:25.764905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241344103788566:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.765117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343940085199:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:25.765211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241343940085200:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:25.765584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241343290811958:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:25.765667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241343290811959:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:25.765698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811957:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.762763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648043:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.766852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648044:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.766942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914188:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.770742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648045:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.771127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788563:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
2025-12-24T02:11:25.771284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241344103788572:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:25.771357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241344103788573:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:25.775357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241343290811968:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: 2025-12-24T02:11:25.782866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914189:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.783063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241342562914209:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:25.786770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345132648048:2111] [node 5] ICP01 ready to work
::ffff:127.1.0.0
2025-12-24T02:11:25.790766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648049:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.802978Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241344103788503:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16056863ns) {Configure:104(16055735ns) {ProcessConfigured:135(15515273ns) {StartInitialHandshake:157(15513678ns) {DropHandshakes:470(1165ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(302ns)} GenerateSessionVirtualId:988(346ns) PrepareNewSessionHandshake:477(2068ns) {SwitchToState:179(1234ns)}}}}}
2025-12-24T02:11:25.807005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241343290811968:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(31556473ns) {Configure:104(31555564ns) {ProcessConfigured:135(31165100ns) {StartInitialHandshake:157(31164205ns) {DropHandshakes:470(300ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(93ns) PrepareNewSessionHandshake:477(1556ns) {SwitchToState:179(593ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.814994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788584:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.823781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628620:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
2025-12-24T02:11:25.823805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628621:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.840104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343940085206:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22113
2025-12-24T02:11:25.840139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343940085209:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22113
2025-12-24T02:11:25.840150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343940085210:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22113
2025-12-24T02:11:25.840163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241343940085211:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22113
2025-12-24T02:11:25.840301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241345132647984:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(49110039ns) {Configure:104(49108683ns) {ProcessConfigured:135(48759583ns) {StartInitialHandshake:157(48759299ns) {DropHandshakes:470(1435ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(594ns)} GenerateSessionVirtualId:988(381ns) PrepareNewSessionHandshake:477(1250ns) {SwitchToState:179(23ns)}}}}}
2025-12-24T02:11:25.841189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648044:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.841221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345132647983:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.841403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345132648063:2117] [node 6] ICP01 ready to work
2025-12-24T02:11:25.846716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085192:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.846734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343940085192:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.846791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241341277628620:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.846943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241341277628551:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.850307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648043:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.865075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914214:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.866939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241342562914163:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.866981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241342562914209:2095] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.866833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343940085192:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.870932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085206:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.870949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343940085190:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.870964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.870973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343940085190:2107] [node 4] ICP22 created new session: [1:7587241343940085213:2107]
2025-12-24T02:11:25.870980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343940085213:2107] [node 4] ICS09 handshake done sender: [1:7587241343940085192:2108] self: [1:7587241343940085191:0] peer: [4:7587241345132648061:0] socket: 137 qp: -1
2025-12-24T02:11:25.870988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343940085213:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:25.871023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.874749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241342562914188:2092] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.874794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241342562914135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.874744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241341277628601:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.874765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241341277628601:2097] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.878918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085202:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.878936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343940085202:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.882841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648067:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.894791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241342562914189:2093] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.894851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241342562914136:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.895487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788575:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.895502Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241344103788575:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.898843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241344103788575:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.902769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343940085202:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.902793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241341277628601:2097] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.902944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788563:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.902989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788564:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.903039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241344103788503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.902885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343940085199:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.902912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.902921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343940085199:2110] [node 6] ICP22 created new session: [1:7587241343940085218:2110]
2025-12-24T02:11:25.902927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343940085218:2110] [node 6] ICS09 handshake done sender: [1:7587241343940085202:2112] self: [1:7587241343940085201:0] peer: [6:7587241342562914218:0] socket: 146 qp: -1
2025-12-24T02:11:25.902934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343940085218:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:25.902950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.903027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241341277628597:2094] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.903040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.903046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241341277628597:2094] [node 6] ICP22 created new session: [5:7587241341277628631:2094]
2025-12-24T02:11:25.903052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241341277628631:2094] [node 6] ICS09 handshake done sender: [5:7587241341277628601:2097] self: [5:7587241341277628600:0] peer: [6:7587241342562914217:0] socket: 132 qp: -1
2025-12-24T02:11:25.903058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241341277628631:2094] [node 6] ICS10 traffic start
2025-12-24T02:11:25.903066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.903230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241344103788572:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.903245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.903252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241344103788572:2111] [node 5] ICP22 created new session: [3:7587241344103788588:2111]
2025-12-24T02:11:25.903258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241344103788588:2111] [node 5] ICS09 handshake done sender: [3:7587241344103788575:2113] self: [3:7587241344103788574:0] peer: [5:7587241341277628629:0] socket: 159 qp: -1
2025-12-24T02:11:25.903265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241344103788588:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.903284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.906949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085211:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.906986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085194:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.906995Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343940085194:2109] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.907654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
2025-12-24T02:11:25.911643Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343940085194:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.911840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241344103788564:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.911906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241344103788503:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.911926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.911934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241344103788503:2048] [node 1] ICP22 created new session: [3:7587241344103788592:2048]
2025-12-24T02:11:25.911941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241344103788592:2048] [node 1] ICS09 handshake done sender: [3:7587241344103788564:2108] self: [3:7587241344103788587:0] peer: [1:7587241343940085193:0] socket: 143 qp: -1
2025-12-24T02:11:25.911947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241344103788592:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.911956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.914875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811974:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.918718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811963:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.918744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241343290811963:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.921671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343940085130:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.921697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.921706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343940085130:2049] [node 3] ICP22 created new session: [1:7587241343940085223:2049]
2025-12-24T02:11:25.921714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343940085223:2049] [node 3] ICS09 handshake done sender: [1:7587241343940085194:2109] self: [1:7587241343940085193:0] peer: [3:7587241344103788587:0] socket: 139 qp: -1
2025-12-24T02:11:25.921722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343940085223:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:25.921737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.921861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241342562914135:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241342562914188:2092] held: no
2025-12-24T02:11:25.922072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241342562914214:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.922163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914220:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.922372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914221:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.922631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241342562914137:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.922652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241342562914163:2075] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.922753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241342562914188:2092] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.921574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345132648044:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.921658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345132647983:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.921674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.921681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345132647983:2048] [node 1] ICP22 created new session: [4:7587241345132648068:2048]
2025-12-24T02:11:25.921687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345132648068:2048] [node 1] ICS09 handshake done sender: [4:7587241345132648044:2108] self: [4:7587241345132648061:0] peer: [1:7587241343940085191:0] socket: 149 qp: -1
2025-12-24T02:11:25.921696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345132648068:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.921719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.922936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241342562914209:2095] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.922948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.922954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241342562914209:2095] [node 5] ICP22 created new session: [6:7587241342562914224:2095]
2025-12-24T02:11:25.922959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241342562914224:2095] [node 5] ICS09 handshake done sender: [6:7587241342562914163:2075] self: [6:7587241342562914217:0] peer: [5:7587241341277628600:0] socket: 134 qp: -1
2025-12-24T02:11:25.922963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241342562914224:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:25.922970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.923056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241342562914135:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.923071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.923079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241342562914135:2048] [node 1] ICP22 created new session: [6:7587241342562914226:2048]
2025-12-24T02:11:25.923085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241342562914226:2048] [node 1] ICS09 handshake done sender: [6:7587241342562914188:2092] self: [6:7587241342562914218:0] peer: [1:7587241343940085201:0] socket: 152 qp: -1
2025-12-24T02:11:25.923090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241342562914226:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.923119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.923046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788577:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.923055Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241344103788577:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.927096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628635:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
2025-12-24T02:11:25.927034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811975:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.932018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648045:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.932251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345132647985:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.930779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628634:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
2025-12-24T02:11:25.935017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811974:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.932443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648070:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.938261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648049:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.938289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345132647984:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.938815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085210:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.939172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788584:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.939208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788568:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.939212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241344103788568:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.942761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241343290811963:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.942888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241344103788568:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.942940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241344103788577:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.942993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241341277628620:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.943030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241344103788573:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.943051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.943060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241344103788573:2112] [node 6] ICP22 created new session: [3:7587241344103788597:2112]
2025-12-24T02:11:25.943067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241344103788597:2112] [node 6] ICS09 handshake done sender: [3:7587241344103788577:2114] self: [3:7587241344103788576:0] peer: [6:7587241342562914223:0] socket: 160 qp: -1
2025-12-24T02:11:25.943074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241344103788597:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:25.943083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.943956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241341277628634:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.944006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241341277628551:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.944019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.944025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241341277628551:2050] [node 3] ICP22 created new session: [5:7587241341277628636:2050]
2025-12-24T02:11:25.944033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241341277628636:2050] [node 3] ICS09 handshake done sender: [5:7587241341277628620:2112] self: [5:7587241341277628629:0] peer: [3:7587241344103788574:0] socket: 169 qp: -1
2025-12-24T02:11:25.944039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241341277628636:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.944048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.944126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241343940085190:2107] [node 4] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.943239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811961:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.943248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241343290811961:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.943739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811978:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
2025-12-24T02:11:25.943761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811979:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
2025-12-24T02:11:25.946846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241343290811961:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.947356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085204:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.947366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343940085204:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.946930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241344103788566:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.946953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.946960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241344103788566:2109] [node 4] ICP22 created new session: [3:7587241344103788599:2109]
2025-12-24T02:11:25.946966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241344103788599:2109] [node 4] ICS09 handshake done sender: [3:7587241344103788568:2110] self: [3:7587241344103788567:0] peer: [4:7587241345132648072:0] socket: 144 qp: -1
2025-12-24T02:11:25.946977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241344103788599:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:25.946990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.947043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241341277628621:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.947084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241341277628602:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.947252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241341277628549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.946969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811979:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.946992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241343290811959:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.947003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.947010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241343290811959:2108] [node 6] ICP22 created new session: [2:7587241343290811981:2108]
2025-12-24T02:11:25.947016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241343290811981:2108] [node 6] ICS09 handshake done sender: [2:7587241343290811963:2111] self: [2:7587241343290811962:0] peer: [6:7587241342562914219:0] socket: 148 qp: -1
2025-12-24T02:11:25.947023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241343290811981:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:25.947031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.947556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241342562914209:2095] [node 5] ICP35 event processing took too much time PendingConnection:279(24518477ns) {HandleHandshakeStatus:329(24518052ns) {DropIncomingHandshake:442(123ns) DropOutgoingHandshake:457(1122ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(17471377ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17450020ns max# 17375145ns}}}
2025-12-24T02:11:25.950377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343940085204:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.948066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648074:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.949127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648053:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.949131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241345132648053:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.950462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811975:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.951354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345132648075:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23334
2025-12-24T02:11:25.951495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345132648049:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.951562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345132647984:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.951577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.951583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345132647984:2049] [node 2] ICP22 created new session: [4:7587241345132648076:2049]
2025-12-24T02:11:25.951590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345132648076:2049] [node 2] ICS09 handshake done sender: [4:7587241345132648049:2112] self: [4:7587241345132648073:0] peer: [2:7587241343290811960:0] socket: 151 qp: -1
2025-12-24T02:11:25.951598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345132648076:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.951605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:25.952254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241345132647985:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241345132648045:2109] held: no
2025-12-24T02:11:25.955237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345132648045:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.958130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345132648053:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.958257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345132647985:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.958268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.958276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345132647985:2050] [node 3] ICP22 created new session: [4:7587241345132648080:2050]
2025-12-24T02:11:25.958282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345132648080:2050] [node 3] ICS09 handshake done sender: [4:7587241345132648045:2109] self: [4:7587241345132648072:0] peer: [3:7587241344103788567:0] socket: 150 qp: -1
2025-12-24T02:11:25.958289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345132648080:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.958296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.959800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241344103788573:2112] [node 6] ICP35 event processing took too much time PendingConnection:279(16657286ns) {HandleHandshakeStatus:329(16656458ns) {DropIncomingHandshake:442(3657ns) DropOutgoingHandshake:457(366ns) SwitchToState:179(72ns) ProcessPendingSessionEvents:495(11668374ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11645870ns max# 11569156ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.962984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628640:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.962896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241342562914137:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26109) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241342562914214:2117] held: no
2025-12-24T02:11:25.963148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914228:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.963329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914229:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.963460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914230:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.963479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241342562914189:2093] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.963559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241342562914214:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.963615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241342562914136:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.963629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.963634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241342562914136:2049] [node 2] ICP22 created new session: [6:7587241342562914231:2049]
2025-12-24T02:11:25.963640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241342562914231:2049] [node 2] ICS09 handshake done sender: [6:7587241342562914189:2093] self: [6:7587241342562914219:0] peer: [2:7587241343290811962:0] socket: 153 qp: -1
2025-12-24T02:11:25.963645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241342562914231:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.963650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:25.963094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343940085200:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.963120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.963129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343940085200:2111] [node 5] ICP22 created new session: [1:7587241343940085227:2111]
2025-12-24T02:11:25.963139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343940085227:2111] [node 5] ICS09 handshake done sender: [1:7587241343940085204:2113] self: [1:7587241343940085203:0] peer: [5:7587241341277628639:0] socket: 176 qp: -1
2025-12-24T02:11:25.963147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343940085227:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.963160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.963614Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811957:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.963633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241343290811958:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.963645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.963651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241343290811958:2107] [node 4] ICP22 created new session: [2:7587241343290811986:2107]
2025-12-24T02:11:25.963657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241343290811986:2107] [node 4] ICS09 handshake done sender: [2:7587241343290811961:2110] self: [2:7587241343290811960:0] peer: [4:7587241345132648073:0] socket: 147 qp: -1
2025-12-24T02:11:25.963661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241343290811986:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:25.963681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.967419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241343290811959:2108] [node 6] ICP18 handshake (actor [2:7587241343290811957:2109]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.970302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241345132648048:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.970328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.970335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345132648048:2111] [node 5] ICP22 created new session: [4:7587241345132648082:2111]
2025-12-24T02:11:25.970341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345132648082:2111] [node 5] ICS09 handshake done sender: [4:7587241345132648053:2114] self: [4:7587241345132648052:0] peer: [5:7587241341277628638:0] socket: 163 qp: -1
2025-12-24T02:11:25.970347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345132648082:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:25.970354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.971208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788601:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
2025-12-24T02:11:25.971535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241342562914137:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.971553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.971560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241342562914137:2050] [node 3] ICP22 created new session: [6:7587241342562914233:2050]
2025-12-24T02:11:25.971567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241342562914233:2050] [node 3] ICS09 handshake done sender: [6:7587241342562914214:2117] self: [6:7587241342562914223:0] peer: [3:7587241344103788576:0] socket: 161 qp: -1
2025-12-24T02:11:25.971572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241342562914233:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:25.971578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.975238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811978:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.975495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241343290811894:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.975919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648070:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.976024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241342562914230:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.976053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241342562914177:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.975634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241343940085208:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.975645Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241343940085208:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.975685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241341277628549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241341277628634:2117] held: no
2025-12-24T02:11:25.976295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241343940085208:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.976382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241343940085129:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.976396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.976402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241343940085129:2048] [node 2] ICP22 created new session: [1:7587241343940085230:2048]
2025-12-24T02:11:25.976408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241343940085230:2048] [node 2] ICS09 handshake done sender: [1:7587241343940085208:2115] self: [1:7587241343940085207:0] peer: [2:7587241343290811989:0] socket: 185 qp: -1
2025-12-24T02:11:25.976412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241343940085230:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:25.976419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.976631Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241341277628602:2096] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241341277628621:2113] held: no
2025-12-24T02:11:25.976740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241341277628640:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.976158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345132648065:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.976167Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241345132648065:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.976959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345132648065:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:25.977024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241345132648063:2117] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.977035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:25.977040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345132648063:2117] [node 6] ICP22 created new session: [4:7587241345132648086:2117]
2025-12-24T02:11:25.977045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345132648086:2117] [node 6] ICS09 handshake done sender: [4:7587241345132648065:2118] self: [4:7587241345132648064:0] peer: [6:7587241342562914235:0] socket: 192 qp: -1
2025-12-24T02:11:25.977050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345132648086:2117] [node 6] ICS10 traffic start
2025-12-24T02:11:25.977056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:25.978127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628641:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.983351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241342562914177:2080] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9819) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241342562914230:2123] held: no
2025-12-24T02:11:25.983612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241342562914236:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39689
2025-12-24T02:11:25.983818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628642:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
2025-12-24T02:11:25.984291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241341277628550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.984305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241341277628634:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.984366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241341277628621:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.984460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241341277628549:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.984473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.984478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241341277628549:2048] [node 1] ICP22 created new session: [5:7587241341277628644:2048]
2025-12-24T02:11:25.984484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241341277628644:2048] [node 1] ICS09 handshake done sender: [5:7587241341277628634:2117] self: [5:7587241341277628639:0] peer: [1:7587241343940085203:0] socket: 177 qp: -1
2025-12-24T02:11:25.984490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241341277628644:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.984496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:25.984700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241341277628602:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.984705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.984709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241341277628602:2096] [node 4] ICP22 created new session: [5:7587241341277628646:2096]
2025-12-24T02:11:25.984714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241341277628646:2096] [node 4] ICS09 handshake done sender: [5:7587241341277628621:2113] self: [5:7587241341277628638:0] peer: [4:7587241345132648052:0] socket: 170 qp: -1
2025-12-24T02:11:25.984717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241341277628646:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:25.984722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.985399Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241343290811894:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241343290811978:2118] held: no
2025-12-24T02:11:25.985661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241343290811990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61974
2025-12-24T02:11:25.986606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811970:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.986617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241343290811970:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.986958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241343290811978:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:25.987259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241343290811973:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.987266Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241343290811973:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:25.987723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241343290811894:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.987733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:25.987738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241343290811894:2048] [node 1] ICP22 created new session: [2:7587241343290811991:2048]
2025-12-24T02:11:25.987744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241343290811991:2048] [node 1] ICS09 handshake done sender: [2:7587241343290811978:2118] self: [2:7587241343290811989:0] peer: [1:7587241343940085207:0] socket: 190 qp: -1
2025-12-24T02:11:25.987749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241343290811991:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:25.987757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.988570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241343290811973:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:25.989635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241343290811895:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.989652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:25.989657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241343290811895:2049] [node 3] ICP22 created new session: [2:7587241343290811995:2049]
2025-12-24T02:11:25.989663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241343290811995:2049] [node 3] ICS09 handshake done sender: [2:7587241343290811973:2115] self: [2:7587241343290811972:0] peer: [3:7587241344103788602:0] socket: 145 qp: -1
2025-12-24T02:11:25.989667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241343290811995:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:25.989672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:25.990632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241343290811970:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:25.986872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241342562914230:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:25.987090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241342562914177:2080] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.987099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:25.987105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241342562914177:2080] [node 4] ICP22 created new session: [6:7587241342562914237:2080]
2025-12-24T02:11:25.987110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241342562914237:2080] [node 4] ICS09 handshake done sender: [6:7587241342562914230:2123] self: [6:7587241342562914235:0] peer: [4:7587241345132648064:0] socket: 201 qp: -1
2025-12-24T02:11:25.987115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241342562914237:2080] [node 4] ICS10 traffic start
2025-12-24T02:11:25.987123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:25.987012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241344103788601:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:25.987053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241344103788504:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:25.988299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241344103788603:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64869
2025-12-24T02:11:25.988735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241344103788601:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.988795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241344103788504:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.988805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.988810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241344103788504:2049] [node 2] ICP22 created new session: [3:7587241344103788604:2049]
2025-12-24T02:11:25.988815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241344103788604:2049] [node 2] ICS09 handshake done sender: [3:7587241344103788601:2119] self: [3:7587241344103788602:0] peer: [2:7587241343290811972:0] socket: 173 qp: -1
2025-12-24T02:11:25.988820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241344103788604:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.988826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:25.990986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241341277628648:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21802
2025-12-24T02:11:25.990758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241343290811968:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:25.990772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:25.990777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241343290811968:2113] [node 5] ICP22 created new session: [2:7587241343290811997:2113]
2025-12-24T02:11:25.990781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241343290811997:2113] [node 5] ICS09 handshake done sender: [2:7587241343290811970:2114] self: [2:7587241343290811969:0] peer: [5:7587241341277628643:0] socket: 142 qp: -1
2025-12-24T02:11:25.990786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241343290811997:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:25.990793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:25.991212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241341277628640:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:25.991873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241341277628550:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5874) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241341277628640:2119] held: no
2025-12-24T02:11:25.991957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241341277628550:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:25.991966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:25.991970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241341277628550:2049] [node 2] ICP22 created new session: [5:7587241341277628649:2049]
2025-12-24T02:11:25.991975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241341277628649:2049] [node 2] ICS09 handshake done sender: [5:7587241341277628640:2119] self: [5:7587241341277628643:0] peer: [2:7587241343290811969:0] socket: 198 qp: -1
2025-12-24T02:11:25.991980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241341277628649:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:25.991986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.002768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:26.002876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241343290811991:2048] [node 1] ICS01 socket: 190 reason# EndOfStream
2025-12-24T02:11:26.002888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241343290811894:2048] [node 1] ICP30 unregister session Session# [2:7587241343290811991:2048] VirtualId# [2:7587241343290811989:0]
2025-12-24T02:11:26.002897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241343290811991:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.002939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.002959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.007096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:26.007199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241344103788592:2048] [node 1] ICS01 socket: 143 reason# EndOfStream
2025-12-24T02:11:26.007208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241344103788503:2048] [node 1] ICP30 unregister session Session# [3:7587241344103788592:2048] VirtualId# [3:7587241344103788587:0]
2025-12-24T02:11:26.007216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241344103788592:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.007246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.007268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.014944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:26.015030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241341277628644:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:11:26.015041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241341277628549:2048] [node 1] ICP30 unregister session Session# [5:7587241341277628644:2048] VirtualId# [5:7587241341277628639:0]
2025-12-24T02:11:26.015048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241341277628644:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.015086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.015110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.015248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:26.015319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241342562914226:2048] [node 1] ICS01 socket: 152 reason# EndOfStream
2025-12-24T02:11:26.015329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241342562914135:2048] [node 1] ICP30 unregister session Session# [6:7587241342562914226:2048] VirtualId# [6:7587241342562914218:0]
2025-12-24T02:11:26.015336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241342562914226:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.015363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.015387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.014803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:26.014917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241345132648068:2048] [node 1] ICS07 socket disconnect 149 reason# EndOfStream
2025-12-24T02:11:26.014926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241345132648068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.014981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.015002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241345132648068:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:26.015645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241345132647983:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.015653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241345132648068:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.015658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241345132647983:2048] [node 1] ICP30 unregister session Session# [4:7587241345132648068:2048] VirtualId# [4:7587241345132648061:0]
2025-12-24T02:11:26.015711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.015718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241345132647983:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.015721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.018806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:26.018899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241344103788604:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:11:26.018907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241344103788504:2049] [node 2] ICP30 unregister session Session# [3:7587241344103788604:2049] VirtualId# [3:7587241344103788602:0]
2025-12-24T02:11:26.018913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241344103788604:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.018939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.018959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.029685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:26.029782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241341277628649:2049] [node 2] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:11:26.029791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241341277628550:2049] [node 2] ICP30 unregister session Session# [5:7587241341277628649:2049] VirtualId# [5:7587241341277628643:0]
2025-12-24T02:11:26.029798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241341277628649:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.029843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.029869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.030764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241345132647983:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:26.030804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:26.030836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:26.030925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241342562914231:2049] [node 2] ICS07 socket disconnect 153 reason# EndOfStream
2025-12-24T02:11:26.030931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241342562914231:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.030968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.030999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241342562914231:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:26.031383Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241342562914136:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5874) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.031394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241342562914231:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.031399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241342562914136:2049] [node 2] ICP30 unregister session Session# [6:7587241342562914231:2049] VirtualId# [6:7587241342562914219:0]
2025-12-24T02:11:26.031451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.031458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241342562914136:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.031461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5874) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.030877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241345132648076:2049] [node 2] ICS01 socket: 151 reason# EndOfStream
2025-12-24T02:11:26.030887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241345132647984:2049] [node 2] ICP30 unregister session Session# [4:7587241345132648076:2049] VirtualId# [4:7587241345132648073:0]
2025-12-24T02:11:26.030893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241345132648076:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.030938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.030961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.046745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241342562914136:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:26.050524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:26.050622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241341277628636:2050] [node 3] ICS07 socket disconnect 169 reason# EndOfStream
2025-12-24T02:11:26.050631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241341277628636:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.050684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.050775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:26.050924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241342562914233:2050] [node 3] ICS01 socket: 161 reason# EndOfStream
2025-12-24T02:11:26.050933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241342562914137:2050] [node 3] ICP30 unregister session Session# [6:7587241342562914233:2050] VirtualId# [6:7587241342562914223:0]
2025-12-24T02:11:26.050941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241342562914233:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.050993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.050830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241341277628636:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:26.051020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.051204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241341277628551:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26109) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.051215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241341277628636:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.051220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241341277628551:2050] [node 3] ICP30 unregister session Session# [5:7587241341277628636:2050] VirtualId# [5:7587241341277628629:0]
2025-12-24T02:11:26.051271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.051277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241341277628551:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.051281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26109) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.062842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241341277628551:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:26.078841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:26.078951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241342562914237:2080] [node 4] ICS01 socket: 201 reason# EndOfStream
2025-12-24T02:11:26.078962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241342562914177:2080] [node 4] ICP30 unregister session Session# [6:7587241342562914237:2080] VirtualId# [6:7587241342562914235:0]
2025-12-24T02:11:26.078972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241342562914237:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.079028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:26.079054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:26.159182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346480633871:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.159193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346480633872:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:26.159044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241347156001661:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:26.159051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241347156001662:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:26.171135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346423441864:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.171144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346423441865:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.191102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241349123740048:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.191111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241349123740049:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.191116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241349123740050:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:26.170950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345750324951:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.170958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345750324952:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.170962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345750324953:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:26.190841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348857132759:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.190853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348857132760:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.190859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348857132761:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.238709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441924:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.239079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241347156001725:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.241955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325012:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.243407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346480633933:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:26.243426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441924:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.244064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325012:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.244512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345750324951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.247740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001727:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.247758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241347156001727:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.250721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348857132818:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:26.251374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241349123740098:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:26.251436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241349123740099:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.254939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346423441927:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:26.254936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633938:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
2025-12-24T02:11:26.255781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241346423441864:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.258718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001722:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.258742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241347156001722:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.259025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346423441930:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:26.263232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345750325015:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.267581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241347156001732:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:26.267700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241347156001733:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:26.268661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241347156001734:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15450
2025-12-24T02:11:26.270839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346480633936:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:26.274888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241347156001727:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.282862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241347156001661:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39973553ns) {Configure:104(39972639ns) {ProcessConfigured:135(39641137ns) {StartInitialHandshake:157(39640396ns) {DropHandshakes:470(465ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(2633ns) {SwitchToState:179(1319ns)}}}}}
2025-12-24T02:11:26.282944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241347156001725:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.282959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.282969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241347156001725:2108] [node 4] ICP22 created new session: [1:7587241347156001742:2108]
2025-12-24T02:11:26.282976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241347156001742:2108] [node 4] ICS09 handshake done sender: [1:7587241347156001727:2109] self: [1:7587241347156001726:0] peer: [4:7587241345750325013:0] socket: 141 qp: -1
2025-12-24T02:11:26.282985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241347156001742:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:26.283010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.287944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346480633945:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:26.287323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740126:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.290889Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241346480633871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35888241ns) {Configure:104(35887497ns) {ProcessConfigured:135(35568426ns) {StartInitialHandshake:157(35567921ns) {DropHandshakes:470(360ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(1597ns) {SwitchToState:179(899ns)}}}}}
2025-12-24T02:11:26.295118Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241346423441927:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40104794ns) {Configure:104(40104106ns) {ProcessConfigured:135(39801611ns) {StartInitialHandshake:157(39800848ns) {DropHandshakes:470(385ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(98ns) PrepareNewSessionHandshake:477(2013ns) {SwitchToState:179(1246ns)}}}}}
2025-12-24T02:11:26.295382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346423441934:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.295450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348857132829:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:26.296061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132825:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.299420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740127:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.299576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
2025-12-24T02:11:26.305875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241345750325023:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.312068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441942:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.312118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441943:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.312148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441944:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.311388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001734:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.311783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241347156001747:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15450
2025-12-24T02:11:26.312040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
2025-12-24T02:11:26.314924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241349123740126:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.314976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241349123740048:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.315102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241347156001722:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.318809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241346423441930:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(55992063ns) {Configure:104(55989759ns) {ProcessConfigured:135(55655549ns) {StartInitialHandshake:157(55654453ns) {DropHandshakes:470(706ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(178ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:477(1279ns) {SwitchToState:179(471ns)}}}}}
2025-12-24T02:11:26.318868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441942:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.318903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241346423441865:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.319013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346423441924:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.320734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132826:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.320884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001737:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.320902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241347156001737:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.339922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241348857132761:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(68795758ns) {Configure:104(68794858ns) {ProcessConfigured:135(68446127ns) {StartInitialHandshake:157(68445809ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(399ns) PrepareNewSessionHandshake:477(1055ns) {SwitchToState:179(679ns)}}}}}
2025-12-24T02:11:26.338824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241346480633936:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(67848669ns) {Configure:104(67848004ns) {ProcessConfigured:135(67468848ns) {StartInitialHandshake:157(67468181ns) {DropHandshakes:470(1084ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(2000ns) {SwitchToState:179(572ns)}}}}}
2025-12-24T02:11:26.338917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633938:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.338953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633940:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.338957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346480633940:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.346829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241346423441864:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.346861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.346874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346423441864:2048] [node 1] ICP22 created new session: [3:7587241346423441947:2048]
2025-12-24T02:11:26.346885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346423441947:2048] [node 1] ICS09 handshake done sender: [3:7587241346423441924:2107] self: [3:7587241346423441925:0] peer: [1:7587241347156001721:0] socket: 139 qp: -1
2025-12-24T02:11:26.346894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346423441947:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.346922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.347365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441949:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.346864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132825:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.350874Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241348857132818:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(100053551ns) {Configure:104(100052905ns) {ProcessConfigured:135(99662166ns) {StartInitialHandshake:157(99661209ns) {DropHandshakes:470(653ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(194ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1219ns) {SwitchToState:179(776ns)}}}}}
2025-12-24T02:11:26.350846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346480633940:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.350925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132826:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.351051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348857132760:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.354888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633937:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.354902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346480633937:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.354979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346480633872:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.354996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.355003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346480633872:2049] [node 3] ICP22 created new session: [2:7587241346480633957:2049]
2025-12-24T02:11:26.355009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346480633957:2049] [node 3] ICS09 handshake done sender: [2:7587241346480633940:2111] self: [2:7587241346480633931:0] peer: [3:7587241346423441946:0] socket: 149 qp: -1
2025-12-24T02:11:26.355018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346480633957:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:26.355033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.355825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633955:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
2025-12-24T02:11:26.355841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.356193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740129:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.356363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346480633937:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.356469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346480633933:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.356485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.356491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346480633933:2107] [node 5] ICP22 created new session: [2:7587241346480633961:2107]
2025-12-24T02:11:26.356498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346480633961:2107] [node 5] ICS09 handshake done sender: [2:7587241346480633937:2109] self: [2:7587241346480633934:0] peer: [5:7587241348857132840:0] socket: 148 qp: -1
2025-12-24T02:11:26.356503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346480633961:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:26.356509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.358476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241347156001662:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.358502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.358510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241347156001662:2049] [node 3] ICP22 created new session: [1:7587241347156001750:2049]
2025-12-24T02:11:26.358519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241347156001750:2049] [node 3] ICS09 handshake done sender: [1:7587241347156001722:2107] self: [1:7587241347156001721:0] peer: [3:7587241346423441925:0] socket: 138 qp: -1
2025-12-24T02:11:26.358524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241347156001750:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:26.358533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.359097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241347156001749:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15450
2025-12-24T02:11:26.358814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346423441942:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.360342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241347156001737:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.359394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132844:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.360424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241347156001732:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.360440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.358966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241346423441865:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.360446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241347156001732:2110] [node 6] ICP22 created new session: [1:7587241347156001753:2110]
2025-12-24T02:11:26.360453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241347156001753:2110] [node 6] ICS09 handshake done sender: [1:7587241347156001737:2113] self: [1:7587241347156001736:0] peer: [6:7587241349123740128:0] socket: 155 qp: -1
2025-12-24T02:11:26.358978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.360459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241347156001753:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:26.358986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346423441865:2049] [node 2] ICP22 created new session: [3:7587241346423441950:2049]
2025-12-24T02:11:26.360465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.358991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346423441950:2049] [node 2] ICS09 handshake done sender: [3:7587241346423441942:2114] self: [3:7587241346423441946:0] peer: [2:7587241346480633931:0] socket: 152 qp: -1
2025-12-24T02:11:26.358997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346423441950:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.359004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.362853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241349123740129:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.362952Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241349123740048:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23100) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241349123740126:2115] held: no
2025-12-24T02:11:26.363165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241349123740049:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.363790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325028:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.363970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325029:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.363986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345750325012:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.364038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345750324951:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.364048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.364055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345750324951:2048] [node 1] ICP22 created new session: [4:7587241345750325031:2048]
2025-12-24T02:11:26.364061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345750325031:2048] [node 1] ICS09 handshake done sender: [4:7587241345750325012:2108] self: [4:7587241345750325013:0] peer: [1:7587241347156001726:0] socket: 142 qp: -1
2025-12-24T02:11:26.364068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345750325031:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.364087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.364682Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325030:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.362879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132841:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.366623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441943:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.367366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132843:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.367389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348857132826:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.368331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132842:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.368391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348857132760:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.368404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.368413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348857132760:2049] [node 2] ICP22 created new session: [5:7587241348857132845:2049]
2025-12-24T02:11:26.368420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348857132845:2049] [node 2] ICS09 handshake done sender: [5:7587241348857132826:2111] self: [5:7587241348857132840:0] peer: [2:7587241346480633934:0] socket: 160 qp: -1
2025-12-24T02:11:26.368427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348857132845:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.368445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.373974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633948:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.373990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346480633948:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.375038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241345750325015:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(111719980ns) {Configure:104(111719386ns) {ProcessConfigured:135(111400315ns) {StartInitialHandshake:157(111399115ns) {DropHandshakes:470(212ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(39ns) PrepareNewSessionHandshake:477(1172ns) {SwitchToState:179(616ns)}}}}}
2025-12-24T02:11:26.375616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740131:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.379065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633951:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.379449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346480633948:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.378912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132842:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.382786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325030:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.382825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345750324952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.382871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325034:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.383087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346480633945:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.383111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.383120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346480633945:2112] [node 6] ICP22 created new session: [2:7587241346480633966:2112]
2025-12-24T02:11:26.383127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346480633966:2112] [node 6] ICS09 handshake done sender: [2:7587241346480633948:2114] self: [2:7587241346480633947:0] peer: [6:7587241349123740130:0] socket: 188 qp: -1
2025-12-24T02:11:26.383134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346480633966:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:26.383144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.383424Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633952:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.383427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346480633952:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.383902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346480633955:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.385452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348857132759:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.386928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241349123740126:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.387022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241349123740048:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.387038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.387047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241349123740048:2048] [node 1] ICP22 created new session: [6:7587241349123740133:2048]
2025-12-24T02:11:26.387054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241349123740133:2048] [node 1] ICS09 handshake done sender: [6:7587241349123740126:2115] self: [6:7587241349123740128:0] peer: [1:7587241347156001736:0] socket: 159 qp: -1
2025-12-24T02:11:26.387061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241349123740133:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.387069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.387872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241348857132760:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19387607ns) {HandleHandshakeStatus:329(19386895ns) {DropIncomingHandshake:442(144ns) DropOutgoingHandshake:457(2203ns) SwitchToState:179(230ns) ProcessPendingSessionEvents:495(8457238ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8428376ns max# 8336652ns}}}
2025-12-24T02:11:26.386974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241346480633872:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(31875033ns) {HandleHandshakeStatus:329(31873943ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(243ns) SwitchToState:179(248ns) ProcessPendingSessionEvents:495(18541961ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18514235ns max# 18423387ns}}}
2025-12-24T02:11:26.387547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241346480633871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.388690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346480633952:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.388961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346480633936:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.388980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.389005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346480633936:2108] [node 4] ICP22 created new session: [2:7587241346480633970:2108]
2025-12-24T02:11:26.389010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346480633970:2108] [node 4] ICS09 handshake done sender: [2:7587241346480633952:2117] self: [2:7587241346480633943:0] peer: [4:7587241345750325035:0] socket: 190 qp: -1
2025-12-24T02:11:26.389015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346480633970:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:26.389021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.388998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.388400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001747:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.390815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740135:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.394781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001741:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.395031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132841:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.395068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348857132761:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.394797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241347156001741:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.398777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325034:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.394835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001739:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.394847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241347156001739:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.398849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325029:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.398864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241345750324952:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241345750325030:2116] held: no
2025-12-24T02:11:26.402795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132843:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.402827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348857132829:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.402920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241345750324953:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.406814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441938:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.406834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346423441938:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.407021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241349123740049:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25693) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241349123740129:2117] held: no
2025-12-24T02:11:26.407173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241349123740129:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.407276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241349123740049:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.407293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.407307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241349123740049:2049] [node 2] ICP22 created new session: [6:7587241349123740136:2049]
2025-12-24T02:11:26.407315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241349123740136:2049] [node 2] ICS09 handshake done sender: [6:7587241349123740129:2117] self: [6:7587241349123740130:0] peer: [2:7587241346480633947:0] socket: 189 qp: -1
2025-12-24T02:11:26.407325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241349123740136:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.407351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.407150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241347156001749:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.410812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325025:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.410827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241345750325025:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.410619Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241347156001741:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.410857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241347156001739:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.411437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241347156001733:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.411453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.411460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241347156001733:2111] [node 5] ICP22 created new session: [1:7587241347156001759:2111]
2025-12-24T02:11:26.411467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241347156001759:2111] [node 5] ICS09 handshake done sender: [1:7587241347156001739:2114] self: [1:7587241347156001738:0] peer: [5:7587241348857132847:0] socket: 164 qp: -1
2025-12-24T02:11:26.411473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241347156001759:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:26.411482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.411733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241347156001661:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.411743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.411748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241347156001661:2048] [node 2] ICP22 created new session: [1:7587241347156001761:2048]
2025-12-24T02:11:26.411753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241347156001761:2048] [node 2] ICS09 handshake done sender: [1:7587241347156001741:2115] self: [1:7587241347156001729:0] peer: [2:7587241346480633969:0] socket: 176 qp: -1
2025-12-24T02:11:26.411757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241347156001761:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:26.411761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.414895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345750325025:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.418528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325039:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.419077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346480633972:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23908
2025-12-24T02:11:26.418586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325040:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.422787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346423441938:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.422834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348857132759:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23100) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241348857132842:2118] held: no
2025-12-24T02:11:26.423012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346423441956:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53086
2025-12-24T02:11:26.423052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346423441934:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.423072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.423079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346423441934:2110] [node 5] ICP22 created new session: [3:7587241346423441957:2110]
2025-12-24T02:11:26.423086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346423441957:2110] [node 5] ICS09 handshake done sender: [3:7587241346423441938:2113] self: [3:7587241346423441937:0] peer: [5:7587241348857132849:0] socket: 163 qp: -1
2025-12-24T02:11:26.423094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346423441957:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:26.423123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.423559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346480633955:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.424719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241346480633871:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.424735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.424740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346480633871:2048] [node 1] ICP22 created new session: [2:7587241346480633973:2048]
2025-12-24T02:11:26.424747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346480633973:2048] [node 1] ICS09 handshake done sender: [2:7587241346480633955:2118] self: [2:7587241346480633969:0] peer: [1:7587241347156001729:0] socket: 177 qp: -1
2025-12-24T02:11:26.424752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346480633973:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.424761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.425925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441956:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.426022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241349123740127:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.426055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241349123740050:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.426966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132852:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.427033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132853:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.427117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740140:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.429232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441933:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.429242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346423441933:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.428731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348857132851:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9494
2025-12-24T02:11:26.428894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348857132841:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.429029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348857132842:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.430640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348857132761:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.430654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.430662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348857132761:2050] [node 3] ICP22 created new session: [5:7587241348857132855:2050]
2025-12-24T02:11:26.430669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348857132855:2050] [node 3] ICS09 handshake done sender: [5:7587241348857132841:2117] self: [5:7587241348857132849:0] peer: [3:7587241346423441937:0] socket: 178 qp: -1
2025-12-24T02:11:26.430676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348857132855:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.430684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.430863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348857132843:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.430803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740141:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.431008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346423441945:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.431021Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346423441945:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.431028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346423441933:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.432411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346423441927:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.432427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.432433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346423441927:2108] [node 6] ICP22 created new session: [3:7587241346423441961:2108]
2025-12-24T02:11:26.432438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346423441961:2108] [node 6] ICS09 handshake done sender: [3:7587241346423441933:2111] self: [3:7587241346423441928:0] peer: [6:7587241349123740139:0] socket: 161 qp: -1
2025-12-24T02:11:26.432442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346423441961:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:26.432448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.431269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241345750325023:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.431290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.431295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345750325023:2111] [node 5] ICP22 created new session: [4:7587241345750325041:2111]
2025-12-24T02:11:26.431301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345750325041:2111] [node 5] ICS09 handshake done sender: [4:7587241345750325025:2113] self: [4:7587241345750325024:0] peer: [5:7587241348857132850:0] socket: 170 qp: -1
2025-12-24T02:11:26.431307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345750325041:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:26.431314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.431556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325039:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.431564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345750325030:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.431919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345750324952:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.431926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.431931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345750324952:2049] [node 2] ICP22 created new session: [4:7587241345750325044:2049]
2025-12-24T02:11:26.431936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345750325044:2049] [node 2] ICS09 handshake done sender: [4:7587241345750325030:2116] self: [4:7587241345750325035:0] peer: [2:7587241346480633943:0] socket: 196 qp: -1
2025-12-24T02:11:26.431940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345750325044:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.431946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.433349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348857132759:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.433362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.433366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348857132759:2048] [node 1] ICP22 created new session: [5:7587241348857132857:2048]
2025-12-24T02:11:26.433371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348857132857:2048] [node 1] ICS09 handshake done sender: [5:7587241348857132842:2118] self: [5:7587241348857132847:0] peer: [1:7587241347156001738:0] socket: 192 qp: -1
2025-12-24T02:11:26.433375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348857132857:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.433381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.433596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348857132829:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.433603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.433606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348857132829:2109] [node 4] ICP22 created new session: [5:7587241348857132859:2109]
2025-12-24T02:11:26.433611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348857132859:2109] [node 4] ICS09 handshake done sender: [5:7587241348857132843:2119] self: [5:7587241348857132850:0] peer: [4:7587241345750325024:0] socket: 193 qp: -1
2025-12-24T02:11:26.433615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348857132859:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:26.433620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.434789Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241349123740050:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24271) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241349123740127:2116] held: no
2025-12-24T02:11:26.443089Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241348857132829:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:26.444113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241346423441934:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:26.445900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241349123740140:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.445999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241349123740127:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.446810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348857132838:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.446821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241348857132838:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.446550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740142:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.446609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241349123740098:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.447487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348857132838:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.447581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241348857132818:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.447603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.447611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348857132818:2105] [node 6] ICP22 created new session: [5:7587241348857132862:2105]
2025-12-24T02:11:26.447617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348857132862:2105] [node 6] ICS09 handshake done sender: [5:7587241348857132838:2116] self: [5:7587241348857132821:0] peer: [6:7587241349123740143:0] socket: 180 qp: -1
2025-12-24T02:11:26.447624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348857132862:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:26.447645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.450838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241345750325047:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:124
2025-12-24T02:11:26.450871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346423441945:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.451001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346423441930:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.451023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.451031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346423441930:2109] [node 4] ICP22 created new session: [3:7587241346423441964:2109]
2025-12-24T02:11:26.451038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346423441964:2109] [node 4] ICS09 handshake done sender: [3:7587241346423441945:2117] self: [3:7587241346423441931:0] peer: [4:7587241345750325037:0] socket: 197 qp: -1
2025-12-24T02:11:26.451045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346423441964:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:26.451054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.452998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241349123740142:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.453628Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241349123740098:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5669) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241349123740140:2120] held: no
2025-12-24T02:11:26.453811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241349123740099:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.453878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740144:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.453931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241349123740050:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.453943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.453949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241349123740050:2050] [node 3] ICP22 created new session: [6:7587241349123740146:2050]
2025-12-24T02:11:26.453954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241349123740146:2050] [node 3] ICS09 handshake done sender: [6:7587241349123740127:2116] self: [6:7587241349123740139:0] peer: [3:7587241346423441928:0] socket: 167 qp: -1
2025-12-24T02:11:26.453962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241349123740146:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.453969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.454673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241349123740140:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.454997Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241345750324953:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24271) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241345750325034:2118] held: no
2025-12-24T02:11:26.455150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345750325034:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.458763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241349123740098:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.458781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.458788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241349123740098:2066] [node 5] ICP22 created new session: [6:7587241349123740148:2066]
2025-12-24T02:11:26.458796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241349123740148:2066] [node 5] ICS09 handshake done sender: [6:7587241349123740140:2120] self: [6:7587241349123740143:0] peer: [5:7587241348857132821:0] socket: 207 qp: -1
2025-12-24T02:11:26.458804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241349123740148:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:26.458813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.462727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241345750325033:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.462755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241345750325033:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.462960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241345750324953:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.462973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.462981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345750324953:2050] [node 3] ICP22 created new session: [4:7587241345750325048:2050]
2025-12-24T02:11:26.462987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345750325048:2050] [node 3] ICS09 handshake done sender: [4:7587241345750325034:2118] self: [4:7587241345750325037:0] peer: [3:7587241346423441931:0] socket: 198 qp: -1
2025-12-24T02:11:26.462993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345750325048:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.462999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.463374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241345750325033:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.463477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241345750325015:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.463499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.463508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241345750325015:2109] [node 6] ICP22 created new session: [4:7587241345750325051:2109]
2025-12-24T02:11:26.463515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241345750325051:2109] [node 6] ICS09 handshake done sender: [4:7587241345750325033:2117] self: [4:7587241345750325016:0] peer: [6:7587241349123740145:0] socket: 172 qp: -1
2025-12-24T02:11:26.463529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241345750325051:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:26.463537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.467462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241349123740099:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31744) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241349123740142:2122] held: no
2025-12-24T02:11:26.467608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241349123740150:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39921
2025-12-24T02:11:26.467850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241349123740142:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.467927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241349123740099:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.467938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.467944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241349123740099:2068] [node 4] ICP22 created new session: [6:7587241349123740151:2068]
2025-12-24T02:11:26.467950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241349123740151:2068] [node 4] ICS09 handshake done sender: [6:7587241349123740142:2122] self: [6:7587241349123740145:0] peer: [4:7587241345750325016:0] socket: 166 qp: -1
2025-12-24T02:11:26.467955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241349123740151:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:26.467974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.482797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:26.482911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241346480633973:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:11:26.482920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241346480633871:2048] [node 1] ICP30 unregister session Session# [2:7587241346480633973:2048] VirtualId# [2:7587241346480633969:0]
2025-12-24T02:11:26.482929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241346480633973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.482972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.482998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.486765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:26.486857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241346423441947:2048] [node 1] ICS01 socket: 139 reason# EndOfStream
2025-12-24T02:11:26.486864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241346423441864:2048] [node 1] ICP30 unregister session Session# [3:7587241346423441947:2048] VirtualId# [3:7587241346423441925:0]
2025-12-24T02:11:26.486870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241346423441947:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.486912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.486937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.490827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:26.490908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348857132857:2048] [node 1] ICS01 socket: 192 reason# EndOfStream
2025-12-24T02:11:26.490914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348857132759:2048] [node 1] ICP30 unregister session Session# [5:7587241348857132857:2048] VirtualId# [5:7587241348857132847:0]
2025-12-24T02:11:26.490919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348857132857:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.490954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.490978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.491016Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:26.491089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241349123740133:2048] [node 1] ICS01 socket: 159 reason# EndOfStream
2025-12-24T02:11:26.491100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241349123740048:2048] [node 1] ICP30 unregister session Session# [6:7587241349123740133:2048] VirtualId# [6:7587241349123740128:0]
2025-12-24T02:11:26.491105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241349123740133:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.491133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.490786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:26.490883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241345750325031:2048] [node 1] ICS01 socket: 142 reason# EndOfStream
2025-12-24T02:11:26.490894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241345750324951:2048] [node 1] ICP30 unregister session Session# [4:7587241345750325031:2048] VirtualId# [4:7587241345750325013:0]
2025-12-24T02:11:26.490901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241345750325031:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.490951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.490978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.491154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.502761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:26.502877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241346423441950:2049] [node 2] ICS01 socket: 152 reason# EndOfStream
2025-12-24T02:11:26.502885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241346423441865:2049] [node 2] ICP30 unregister session Session# [3:7587241346423441950:2049] VirtualId# [3:7587241346423441946:0]
2025-12-24T02:11:26.502891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241346423441950:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.502924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.502952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.518796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:26.518903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241345750325044:2049] [node 2] ICS01 socket: 196 reason# EndOfStream
2025-12-24T02:11:26.518914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241345750324952:2049] [node 2] ICP30 unregister session Session# [4:7587241345750325044:2049] VirtualId# [4:7587241345750325035:0]
2025-12-24T02:11:26.518921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241345750325044:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.518977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.519012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.521975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:26.519060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:26.519142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241349123740136:2049] [node 2] ICS01 socket: 189 reason# EndOfStream
2025-12-24T02:11:26.519149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241349123740049:2049] [node 2] ICP30 unregister session Session# [6:7587241349123740136:2049] VirtualId# [6:7587241349123740130:0]
2025-12-24T02:11:26.519154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241349123740136:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.519184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.519207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.522059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241348857132845:2049] [node 2] ICS07 socket disconnect 160 reason# EndOfStream
2025-12-24T02:11:26.522069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348857132845:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.522116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.522140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241348857132845:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:26.522456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348857132760:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25693) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.522464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348857132845:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.522469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348857132760:2049] [node 2] ICP30 unregister session Session# [5:7587241348857132845:2049] VirtualId# [5:7587241348857132840:0]
2025-12-24T02:11:26.522516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.522522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241348857132760:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.522526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25693) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.530748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241348857132760:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:26.538041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:26.538116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241349123740146:2050] [node 3] ICS01 socket: 167 reason# EndOfStream
2025-12-24T02:11:26.538123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241349123740050:2050] [node 3] ICP30 unregister session Session# [6:7587241349123740146:2050] VirtualId# [6:7587241349123740139:0]
2025-12-24T02:11:26.538129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241349123740146:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.538170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.538192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.537964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:26.538062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348857132855:2050] [node 3] ICS01 socket: 178 reason# EndOfStream
2025-12-24T02:11:26.538072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348857132761:2050] [node 3] ICP30 unregister session Session# [5:7587241348857132855:2050] VirtualId# [5:7587241348857132849:0]
2025-12-24T02:11:26.538082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348857132855:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.538138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.538163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.570756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:26.570868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241349123740151:2068] [node 4] ICS01 socket: 166 reason# EndOfStream
2025-12-24T02:11:26.570878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241349123740099:2068] [node 4] ICP30 unregister session Session# [6:7587241349123740151:2068] VirtualId# [6:7587241349123740145:0]
2025-12-24T02:11:26.570887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241349123740151:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.570935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:26.570958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:26.620317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241348118468832:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:26.620328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241348118468833:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:26.642860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346284704916:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.642884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346284704917:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:26.650859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346947996840:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.650872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346947996841:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.655030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241348118468881:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:26.655220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241348118468882:2094] [node 5] ICP01 ready to work
2025-12-24T02:11:26.655468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346284704976:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.664853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996902:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.665641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346284704986:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:26.666369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346947996904:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.668355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346284704985:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.668366Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346284704985:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.669062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346284704985:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.669609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346284704917:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.669624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.669630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346284704917:2049] [node 3] ICP22 created new session: [2:7587241346284704994:2049]
2025-12-24T02:11:26.669635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346284704994:2049] [node 3] ICS09 handshake done sender: [2:7587241346284704985:2111] self: [2:7587241346284704984:0] peer: [3:7587241346947996908:0] socket: 159 qp: -1
2025-12-24T02:11:26.669641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346284704994:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:26.669647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.667629Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996906:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.667649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996907:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.667905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996907:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.668211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241346947996841:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.668982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996909:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.669265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346947996907:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.669324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241346947996841:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.669337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.669344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346947996841:2049] [node 2] ICP22 created new session: [3:7587241346947996910:2049]
2025-12-24T02:11:26.669352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346947996910:2049] [node 2] ICS09 handshake done sender: [3:7587241346947996907:2110] self: [3:7587241346947996908:0] peer: [2:7587241346284704984:0] socket: 162 qp: -1
2025-12-24T02:11:26.669360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346947996910:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.669379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.674894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346947996914:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.682806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241348118468832:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27878066ns) {Configure:104(27876743ns) {ProcessConfigured:135(22333560ns) {StartInitialHandshake:157(22332455ns) {DropHandshakes:470(737ns) {DropIncomingHandshake:442(181ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1927ns) {SwitchToState:179(1280ns)}}}}}
2025-12-24T02:11:26.682963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241348118468897:2098] [node 6] ICP01 ready to work
2025-12-24T02:11:26.683571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241348118468900:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16990
2025-12-24T02:11:26.683593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241348118468901:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16990
2025-12-24T02:11:26.683626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241348118468898:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16990
2025-12-24T02:11:26.683635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241348118468899:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16990
2025-12-24T02:11:26.690857Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241348118468882:2094] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(35584410ns) {Configure:104(35583904ns) {ProcessConfigured:135(30130448ns) {StartInitialHandshake:157(30129393ns) {DropHandshakes:470(407ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(1524ns) {SwitchToState:179(1025ns)}}}}}
2025-12-24T02:11:26.650826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348238424437:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.650837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348238424438:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.650844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348238424439:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:26.662174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348238424493:2100] [node 6] ICP01 ready to work
2025-12-24T02:11:26.679055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241348238424507:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:26.694718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241346284704982:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:26.655745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241347673870545:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.655769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241347673870546:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.655789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241347673870547:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:26.655938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241347673870593:2085] [node 5] ICP01 ready to work
2025-12-24T02:11:26.656124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241347673870594:2088] [node 4] ICP01 ready to work
2025-12-24T02:11:26.639895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241346679179354:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:26.639905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241346679179355:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:26.639911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241346679179356:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:26.656357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241346679179415:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:26.663971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241346679179420:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.698779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241346947996904:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32338356ns) {Configure:104(32337757ns) {ProcessConfigured:135(31998556ns) {StartInitialHandshake:157(31998064ns) {DropHandshakes:470(511ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(134ns) PrepareNewSessionHandshake:477(1513ns) {SwitchToState:179(825ns)}}}}}
2025-12-24T02:11:26.702259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179431:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.702302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179432:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.702317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179433:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.702655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241348238424438:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27833391ns) {Configure:104(27832438ns) {ProcessConfigured:135(27427500ns) {StartInitialHandshake:157(27426988ns) {DropHandshakes:470(389ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(288ns) PrepareNewSessionHandshake:477(1361ns) {SwitchToState:179(942ns)}}}}}
2025-12-24T02:11:26.703314Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241346947996914:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28350365ns) {Configure:104(28349536ns) {ProcessConfigured:135(28041928ns) {StartInitialHandshake:157(28041774ns) {DropHandshakes:470(391ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(518ns) {SwitchToState:179(203ns)}}}}}
2025-12-24T02:11:26.711194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241346947996917:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.718808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346284704999:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53758
2025-12-24T02:11:26.718852Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241348118468833:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56406111ns) {Configure:104(56405303ns) {ProcessConfigured:135(54688906ns) {StartInitialHandshake:157(54688308ns) {DropHandshakes:470(316ns) {DropIncomingHandshake:442(104ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(532ns) PrepareNewSessionHandshake:477(1981ns) {SwitchToState:179(1357ns)}}}}}
2025-12-24T02:11:26.719860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468899:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.719931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468901:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.723080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241348118468881:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(67961971ns) {Configure:104(67961203ns) {ProcessConfigured:135(62255763ns) {StartInitialHandshake:157(62254613ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1258ns) {SwitchToState:179(752ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.723677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870623:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.723736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870624:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.723747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870625:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.723782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870626:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.726912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241347673870623:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.727208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179431:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.727317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241347673870625:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.730970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241347673870626:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.731001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241347673870547:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.730718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346284705000:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53758
2025-12-24T02:11:26.731092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346284704990:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.731098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346284704990:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.731782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346284704990:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.731864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346284704986:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.731878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.731888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346284704986:2110] [node 4] ICP22 created new session: [2:7587241346284705002:2110]
2025-12-24T02:11:26.731896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346284705002:2110] [node 4] ICS09 handshake done sender: [2:7587241346284704990:2113] self: [2:7587241346284704989:0] peer: [4:7587241346679179434:0] socket: 165 qp: -1
2025-12-24T02:11:26.731903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346284705002:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:26.731913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.733750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346284704998:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.733754Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346284704998:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.734323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346284704998:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.732415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870630:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.733650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241347673870630:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.733673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241347673870546:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.734251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870632:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.734493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241347673870630:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.734532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241347673870546:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.734543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.734550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241347673870546:2049] [node 2] ICP22 created new session: [6:7587241347673870633:2049]
2025-12-24T02:11:26.734556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241347673870633:2049] [node 2] ICS09 handshake done sender: [6:7587241347673870630:2119] self: [6:7587241347673870631:0] peer: [2:7587241346284704997:0] socket: 190 qp: -1
2025-12-24T02:11:26.734562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241347673870633:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.734568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.730772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241346679179355:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.734792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996920:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.734806Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346947996920:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.739142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241346679179354:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44361778ns) {Configure:104(44359809ns) {ProcessConfigured:135(43894334ns) {StartInitialHandshake:157(43893228ns) {DropHandshakes:470(1085ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(754ns) PrepareNewSessionHandshake:477(1315ns) {SwitchToState:179(395ns)}}}}}
2025-12-24T02:11:26.739400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179433:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.739453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241346679179356:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.739002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468900:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.739551Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996918:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.739560Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346947996918:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.740999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346947996918:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.741080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346947996904:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.741089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.741095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346947996904:2108] [node 4] ICP22 created new session: [3:7587241346947996924:2108]
2025-12-24T02:11:26.741101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346947996924:2108] [node 4] ICS09 handshake done sender: [3:7587241346947996918:2115] self: [3:7587241346947996905:0] peer: [4:7587241346679179436:0] socket: 178 qp: -1
2025-12-24T02:11:26.741107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346947996924:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:26.741114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.750859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179417:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.750873Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241346679179417:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.750794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468903:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.750810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241348118468903:2110] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.759047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996906:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.759098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241346947996904:2108] [node 4] ICP18 handshake (actor [3:7587241346947996906:2109]) is too old
2025-12-24T02:11:26.762649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346947996920:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.763136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346947996914:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.763156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.763163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346947996914:2113] [node 6] ICP22 created new session: [3:7587241346947996927:2113]
2025-12-24T02:11:26.763169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346947996927:2113] [node 6] ICS09 handshake done sender: [3:7587241346947996920:2116] self: [3:7587241346947996915:0] peer: [6:7587241347673870629:0] socket: 181 qp: -1
2025-12-24T02:11:26.763177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346947996927:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:26.763185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.762861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241346679179356:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10991) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241346679179433:2116] held: no
2025-12-24T02:11:26.766858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241347673870545:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.766899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241347673870545:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(39529485ns) {IncomingHandshake:261(39529087ns) {DropIncomingHandshake:442(206ns) PrepareNewSessionHandshake:477(424ns) {SwitchToState:179(80ns)} GenerateSessionVirtualId:988(215ns) IssueIncomingHandshakeReply:193(39524825ns) {SwitchToState:179(806ns)}}}
2025-12-24T02:11:26.766872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424515:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.766896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996902:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.771090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241346947996917:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(59498151ns) {Configure:104(59496767ns) {ProcessConfigured:135(59132945ns) {StartInitialHandshake:157(59132103ns) {DropHandshakes:470(1388ns) {DropIncomingHandshake:442(194ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(341ns) PrepareNewSessionHandshake:477(652ns) {SwitchToState:179(307ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.772857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424515:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.772889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348238424438:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.770718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346284704982:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.770743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.770753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346284704982:2109] [node 6] ICP22 created new session: [2:7587241346284705006:2109]
2025-12-24T02:11:26.770765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346284705006:2109] [node 6] ICS09 handshake done sender: [2:7587241346284704998:2114] self: [2:7587241346284704997:0] peer: [6:7587241347673870631:0] socket: 189 qp: -1
2025-12-24T02:11:26.770773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346284705006:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:26.770792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.770938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346284705000:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.770971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241346284704916:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.772990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241346284704978:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.772994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241346284704978:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.773504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346284704978:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.773600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241346284704976:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.773610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.773617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346284704976:2106] [node 5] ICP22 created new session: [2:7587241346284705010:2106]
2025-12-24T02:11:26.773623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346284705010:2106] [node 5] ICS09 handshake done sender: [2:7587241346284704978:2107] self: [2:7587241346284704977:0] peer: [5:7587241348238424516:0] socket: 145 qp: -1
2025-12-24T02:11:26.773629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346284705010:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:26.773637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.770820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241347673870594:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.770862Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241347673870594:2088] [node 4] ICP35 event processing took too much time PendingConnection:279(43885343ns) {IncomingHandshake:261(43884727ns) {DropIncomingHandshake:442(260ns) PrepareNewSessionHandshake:477(706ns) {SwitchToState:179(359ns)} GenerateSessionVirtualId:988(416ns) IssueIncomingHandshakeReply:193(43876279ns) {SwitchToState:179(683ns)}}}
2025-12-24T02:11:26.772513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.773962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.774956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468896:2105] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.774966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241348118468896:2105] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.775090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241348118468903:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.775078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241346679179417:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.775141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241346679179431:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.775241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241348118468897:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.775261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.775269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241348118468897:2098] [node 6] ICP22 created new session: [1:7587241348118468919:2098]
2025-12-24T02:11:26.775279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241348118468919:2098] [node 6] ICS09 handshake done sender: [1:7587241348118468903:2110] self: [1:7587241348118468902:0] peer: [6:7587241347673870628:0] socket: 173 qp: -1
2025-12-24T02:11:26.775285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241348118468919:2098] [node 6] ICS10 traffic start
2025-12-24T02:11:26.775293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.775311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179432:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.775325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241346679179355:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.775336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.775341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241346679179355:2049] [node 2] ICP22 created new session: [4:7587241346679179440:2049]
2025-12-24T02:11:26.775347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241346679179440:2049] [node 2] ICS09 handshake done sender: [4:7587241346679179431:2114] self: [4:7587241346679179434:0] peer: [2:7587241346284704989:0] socket: 166 qp: -1
2025-12-24T02:11:26.775351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241346679179440:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.775358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.775569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241346679179415:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.775577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.775582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241346679179415:2106] [node 6] ICP22 created new session: [4:7587241346679179442:2106]
2025-12-24T02:11:26.775589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241346679179442:2106] [node 6] ICS09 handshake done sender: [4:7587241346679179417:2108] self: [4:7587241346679179416:0] peer: [6:7587241347673870627:0] socket: 147 qp: -1
2025-12-24T02:11:26.775594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241346679179442:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:26.775607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.776304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241347673870545:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241347673870625:2117] held: no
2025-12-24T02:11:26.776372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870636:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.780978Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241348118468896:2105] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.781106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241348118468832:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.781124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.781129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241348118468832:2048] [node 2] ICP22 created new session: [1:7587241348118468924:2048]
2025-12-24T02:11:26.781137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241348118468924:2048] [node 2] ICS09 handshake done sender: [1:7587241348118468896:2105] self: [1:7587241348118468884:0] peer: [2:7587241346284705008:0] socket: 172 qp: -1
2025-12-24T02:11:26.781142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241348118468924:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:26.781174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.782374Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241346284704916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241346284705000:2116] held: no
2025-12-24T02:11:26.782539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241346284705012:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53758
2025-12-24T02:11:26.782811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468898:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.782903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241348118468922:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16990
2025-12-24T02:11:26.782998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241346284705000:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.783172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241346947996914:2113] [node 6] ICP35 event processing took too much time PendingConnection:279(19927917ns) {HandleHandshakeStatus:329(19927397ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(100ns) SwitchToState:179(371ns) ProcessPendingSessionEvents:495(19838659ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19817822ns max# 19738707ns}}}
2025-12-24T02:11:26.783221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241346947996914:2113] [node 6] ICP18 handshake (actor [3:7587241346947996902:2107]) is too old
2025-12-24T02:11:26.786712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870638:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.786852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870637:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.787535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241346284704982:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(52521079ns) {HandleHandshakeStatus:329(52520558ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(66ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(10594192ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 10573695ns max# 10493778ns}}}
2025-12-24T02:11:26.787790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241346284704916:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.787801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.787807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241346284704916:2048] [node 1] ICP22 created new session: [2:7587241346284705013:2048]
2025-12-24T02:11:26.787813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241346284705013:2048] [node 1] ICS09 handshake done sender: [2:7587241346284705000:2116] self: [2:7587241346284705008:0] peer: [1:7587241348118468884:0] socket: 184 qp: -1
2025-12-24T02:11:26.787819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241346284705013:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.787826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.787253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241346679179433:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.790399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241346679179356:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.790411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.790420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241346679179356:2050] [node 3] ICP22 created new session: [4:7587241346679179447:2050]
2025-12-24T02:11:26.790427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241346679179447:2050] [node 3] ICS09 handshake done sender: [4:7587241346679179433:2116] self: [4:7587241346679179436:0] peer: [3:7587241346947996905:0] socket: 180 qp: -1
2025-12-24T02:11:26.790433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241346679179447:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.790441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.791556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.791342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241347673870626:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.791451Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241347673870623:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.791669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241347673870594:2088] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.791679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.791685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241347673870594:2088] [node 4] ICP22 created new session: [6:7587241347673870639:2088]
2025-12-24T02:11:26.791691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241347673870639:2088] [node 4] ICS09 handshake done sender: [6:7587241347673870623:2115] self: [6:7587241347673870627:0] peer: [4:7587241346679179416:0] socket: 182 qp: -1
2025-12-24T02:11:26.791696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241347673870639:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:26.791702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.791980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241347673870625:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.792146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241347673870547:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.792178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.792184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241347673870547:2050] [node 3] ICP22 created new session: [6:7587241347673870641:2050]
2025-12-24T02:11:26.792189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241347673870641:2050] [node 3] ICS09 handshake done sender: [6:7587241347673870626:2118] self: [6:7587241347673870629:0] peer: [3:7587241346947996915:0] socket: 187 qp: -1
2025-12-24T02:11:26.792194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241347673870641:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.792199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.793212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241347673870545:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.793221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.793228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241347673870545:2048] [node 1] ICP22 created new session: [6:7587241347673870643:2048]
2025-12-24T02:11:26.793234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241347673870643:2048] [node 1] ICS09 handshake done sender: [6:7587241347673870625:2117] self: [6:7587241347673870628:0] peer: [1:7587241348118468902:0] socket: 186 qp: -1
2025-12-24T02:11:26.793239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241347673870643:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.793245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.794989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424517:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.795015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424518:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.795025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.795042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.795056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.795199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241347673870624:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.795276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241347673870593:2085] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.795412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179446:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.795577Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241346679179355:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20134712ns) {HandleHandshakeStatus:329(20134330ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(1233ns) SwitchToState:179(100ns) ProcessPendingSessionEvents:495(18603645ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18579706ns max# 18513086ns}}}
2025-12-24T02:11:26.798748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468922:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424517:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424518:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424519:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424520:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241348238424495:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.798880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241348238424495:2101] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.799161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348238424437:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.799179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348238424439:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.799515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348238424515:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:26.799661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348238424495:2101] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:26.799730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241348238424493:2100] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.799745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:26.799750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348238424493:2100] [node 6] ICP22 created new session: [5:7587241348238424527:2100]
2025-12-24T02:11:26.799757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348238424527:2100] [node 6] ICS09 handshake done sender: [5:7587241348238424495:2101] self: [5:7587241348238424494:0] peer: [6:7587241347673870645:0] socket: 154 qp: -1
2025-12-24T02:11:26.799762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348238424527:2100] [node 6] ICS10 traffic start
2025-12-24T02:11:26.799778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:26.800138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348238424438:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.800149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:26.800152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348238424438:2049] [node 2] ICP22 created new session: [5:7587241348238424529:2049]
2025-12-24T02:11:26.800157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348238424529:2049] [node 2] ICS09 handshake done sender: [5:7587241348238424515:2115] self: [5:7587241348238424516:0] peer: [2:7587241346284704977:0] socket: 176 qp: -1
2025-12-24T02:11:26.800161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348238424529:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:26.800166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:26.801281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468906:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.801291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241348118468906:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.802345Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241348118468906:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.802429Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241347673870593:2085] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2276) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241347673870624:2116] held: no
2025-12-24T02:11:26.806792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241348238424507:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.806739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241348118468882:2094] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.806764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.806772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241348118468882:2094] [node 5] ICP22 created new session: [1:7587241348118468930:2094]
2025-12-24T02:11:26.806778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241348118468930:2094] [node 5] ICS09 handshake done sender: [1:7587241348118468906:2111] self: [1:7587241348118468886:0] peer: [5:7587241348238424523:0] socket: 175 qp: -1
2025-12-24T02:11:26.806783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241348118468930:2094] [node 5] ICS10 traffic start
2025-12-24T02:11:26.806800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.806860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996929:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.806872Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241346947996929:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.802849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179422:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.802859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241346679179422:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.807045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241346679179446:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.807081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241346679179354:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:26.807201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241346947996931:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.807225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241346947996840:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.815877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424533:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.814761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468916:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.814772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241348118468916:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:26.815303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241348118468913:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:26.815306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241348118468913:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.818980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241346679179450:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38987
2025-12-24T02:11:26.821266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348238424518:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.821357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348238424437:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.821375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.821382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348238424437:2048] [node 1] ICP22 created new session: [5:7587241348238424534:2048]
2025-12-24T02:11:26.821389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348238424534:2048] [node 1] ICS09 handshake done sender: [5:7587241348238424518:2117] self: [5:7587241348238424523:0] peer: [1:7587241348118468886:0] socket: 148 qp: -1
2025-12-24T02:11:26.821395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348238424534:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.821402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.822886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241346679179446:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.822950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241346679179354:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.822962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.822968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241346679179354:2048] [node 1] ICP22 created new session: [4:7587241346679179451:2048]
2025-12-24T02:11:26.822974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241346679179451:2048] [node 1] ICS09 handshake done sender: [4:7587241346679179446:2120] self: [4:7587241346679179449:0] peer: [1:7587241348118468887:0] socket: 158 qp: -1
2025-12-24T02:11:26.822979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241346679179451:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.822985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.818982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241348118468916:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.821105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241348118468913:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.833338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424536:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.832222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241348118468882:2094] [node 5] ICP35 event processing took too much time PendingConnection:279(25382967ns) {HandleHandshakeStatus:329(25382148ns) {DropIncomingHandshake:442(4472ns) DropOutgoingHandshake:457(111ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(25281766ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 25260620ns max# 25191425ns}}}
2025-12-24T02:11:26.832242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241348118468881:2089] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.832260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.832268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241348118468881:2089] [node 4] ICP22 created new session: [1:7587241348118468934:2089]
2025-12-24T02:11:26.832273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241348118468934:2089] [node 4] ICS09 handshake done sender: [1:7587241348118468916:2118] self: [1:7587241348118468887:0] peer: [4:7587241346679179449:0] socket: 171 qp: -1
2025-12-24T02:11:26.832280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241348118468934:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:26.832287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.834895Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346947996929:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.834886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241347673870646:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10337
2025-12-24T02:11:26.835131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241347673870624:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.835333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241348118468833:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.835360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.835368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241348118468833:2049] [node 3] ICP22 created new session: [1:7587241348118468936:2049]
2025-12-24T02:11:26.835201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241347673870593:2085] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.835374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241348118468936:2049] [node 3] ICS09 handshake done sender: [1:7587241348118468913:2117] self: [1:7587241348118468885:0] peer: [3:7587241346947996932:0] socket: 169 qp: -1
2025-12-24T02:11:26.835216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.835380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241348118468936:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:26.835387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:26.835222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241347673870593:2085] [node 5] ICP22 created new session: [6:7587241347673870647:2085]
2025-12-24T02:11:26.835228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241347673870647:2085] [node 5] ICS09 handshake done sender: [6:7587241347673870624:2116] self: [6:7587241347673870645:0] peer: [5:7587241348238424494:0] socket: 185 qp: -1
2025-12-24T02:11:26.835235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241347673870647:2085] [node 5] ICS10 traffic start
2025-12-24T02:11:26.835243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.835754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348238424520:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:26.835863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241346947996917:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.835876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.835882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346947996917:2114] [node 5] ICP22 created new session: [3:7587241346947996934:2114]
2025-12-24T02:11:26.835887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346947996934:2114] [node 5] ICS09 handshake done sender: [3:7587241346947996929:2117] self: [3:7587241346947996922:0] peer: [5:7587241348238424525:0] socket: 196 qp: -1
2025-12-24T02:11:26.835892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346947996934:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:26.835897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.836060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241346679179422:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:26.838814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241346679179420:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:26.838835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:26.838843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241346679179420:2109] [node 5] ICP22 created new session: [4:7587241346679179454:2109]
2025-12-24T02:11:26.838850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241346679179454:2109] [node 5] ICS09 handshake done sender: [4:7587241346679179422:2110] self: [4:7587241346679179421:0] peer: [5:7587241348238424522:0] socket: 156 qp: -1
2025-12-24T02:11:26.838856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241346679179454:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:26.838864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:26.839401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241346679179354:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16348200ns) {HandleHandshakeStatus:329(16347780ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(1248ns) SwitchToState:179(118ns) ProcessPendingSessionEvents:495(8437303ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 8420983ns max# 8350131ns}}}
2025-12-24T02:11:26.840172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348238424439:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.840190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:26.840196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348238424439:2050] [node 3] ICP22 created new session: [5:7587241348238424537:2050]
2025-12-24T02:11:26.840202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348238424537:2050] [node 3] ICS09 handshake done sender: [5:7587241348238424520:2119] self: [5:7587241348238424525:0] peer: [3:7587241346947996922:0] socket: 160 qp: -1
2025-12-24T02:11:26.840208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348238424537:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:26.840214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.844453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241346947996840:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241346947996931:2118] held: no
2025-12-24T02:11:26.844665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996936:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.844709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241346947996937:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61226
2025-12-24T02:11:26.844858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241346947996931:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:26.844909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241346947996840:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.844923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:26.844931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241346947996840:2048] [node 1] ICP22 created new session: [3:7587241346947996938:2048]
2025-12-24T02:11:26.844937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241346947996938:2048] [node 1] ICS09 handshake done sender: [3:7587241346947996931:2118] self: [3:7587241346947996932:0] peer: [1:7587241348118468885:0] socket: 202 qp: -1
2025-12-24T02:11:26.844943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241346947996938:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:26.844949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:26.854984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348238424507:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19352) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241348238424517:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:26.862867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241348238424539:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58376
2025-12-24T02:11:26.863125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241348238424517:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:26.863164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241348238424507:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:26.863178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:26.863188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241348238424507:2110] [node 4] ICP22 created new session: [5:7587241348238424540:2110]
2025-12-24T02:11:26.863197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241348238424540:2110] [node 4] ICS09 handshake done sender: [5:7587241348238424517:2116] self: [5:7587241348238424522:0] peer: [4:7587241346679179421:0] socket: 195 qp: -1
2025-12-24T02:11:26.863205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241348238424540:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:26.863216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:26.867542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241346947996917:2114] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:26.891063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:26.891170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241346284705013:2048] [node 1] ICS07 socket disconnect 184 reason# EndOfStream
2025-12-24T02:11:26.891178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241346284705013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.891219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.891239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241346284705013:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:26.891929Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:26.891999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241346947996938:2048] [node 1] ICS01 socket: 202 reason# EndOfStream
2025-12-24T02:11:26.892006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241346947996840:2048] [node 1] ICP30 unregister session Session# [3:7587241346947996938:2048] VirtualId# [3:7587241346947996932:0]
2025-12-24T02:11:26.892014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241346947996938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.892047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.892069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.892596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241346284704916:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.892613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241346284705013:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.892618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241346284704916:2048] [node 1] ICP30 unregister session Session# [2:7587241346284705013:2048] VirtualId# [2:7587241346284705008:0]
2025-12-24T02:11:26.892637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.892643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241346284704916:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.892647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.896785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:26.896856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241346679179451:2048] [node 1] ICS07 socket disconnect 158 reason# EndOfStream
2025-12-24T02:11:26.896861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241346679179451:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.896904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.896922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241346679179451:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:26.898189Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:26.898275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241347673870643:2048] [node 1] ICS07 socket disconnect 186 reason# EndOfStream
2025-12-24T02:11:26.898282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241347673870643:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.898330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.898353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241347673870643:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:26.897119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:26.897185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241348238424534:2048] [node 1] ICS07 socket disconnect 148 reason# EndOfStream
2025-12-24T02:11:26.897190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348238424534:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.897252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:26.897269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241348238424534:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:26.898846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241347673870545:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.898859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241347673870643:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.898864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241347673870545:2048] [node 1] ICP30 unregister session Session# [6:7587241347673870643:2048] VirtualId# [6:7587241347673870628:0]
2025-12-24T02:11:26.898886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.898892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241347673870545:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.898895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.899033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241346679179354:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.899046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241346679179451:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.899050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241346679179354:2048] [node 1] ICP30 unregister session Session# [4:7587241346679179451:2048] VirtualId# [4:7587241346679179449:0]
2025-12-24T02:11:26.899064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.899072Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241346679179354:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.899076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.903025Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348238424437:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.903042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348238424534:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.903047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348238424437:2048] [node 1] ICP30 unregister session Session# [5:7587241348238424534:2048] VirtualId# [5:7587241348238424523:0]
2025-12-24T02:11:26.903063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:26.903073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241348238424437:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.903076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24130) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.910787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241346679179354:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:26.914741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241347673870545:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:26.914843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241348238424437:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:26.918862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:26.918968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241346947996910:2049] [node 2] ICS07 socket disconnect 162 reason# EndOfStream
2025-12-24T02:11:26.918975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241346947996910:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.919021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.919039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241346947996910:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:26.919780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241346947996841:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.919793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241346947996910:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.919798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241346947996841:2049] [node 2] ICP30 unregister session Session# [3:7587241346947996910:2049] VirtualId# [3:7587241346947996908:0]
2025-12-24T02:11:26.919814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.919821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241346947996841:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.919824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.922728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:26.922803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241347673870633:2049] [node 2] ICS01 socket: 190 reason# EndOfStream
2025-12-24T02:11:26.922809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241347673870546:2049] [node 2] ICP30 unregister session Session# [6:7587241347673870633:2049] VirtualId# [6:7587241347673870631:0]
2025-12-24T02:11:26.922815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241347673870633:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.922864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.922888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.922709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:26.922785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241348238424529:2049] [node 2] ICS07 socket disconnect 176 reason# EndOfStream
2025-12-24T02:11:26.922791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348238424529:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.922837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.922859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241348238424529:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:26.923069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:26.923141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241346679179440:2049] [node 2] ICS01 socket: 166 reason# EndOfStream
2025-12-24T02:11:26.923146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241346679179355:2049] [node 2] ICP30 unregister session Session# [4:7587241346679179440:2049] VirtualId# [4:7587241346679179434:0]
2025-12-24T02:11:26.923152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241346679179440:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.923184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:26.923206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.930996Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348238424438:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.931011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348238424529:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.931017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348238424438:2049] [node 2] ICP30 unregister session Session# [5:7587241348238424529:2049] VirtualId# [5:7587241348238424516:0]
2025-12-24T02:11:26.931038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:26.931044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241348238424438:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.931048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65233) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.942800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241348238424438:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:26.946766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:26.946872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241347673870641:2050] [node 3] ICS01 socket: 187 reason# EndOfStream
2025-12-24T02:11:26.946883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241347673870547:2050] [node 3] ICP30 unregister session Session# [6:7587241347673870641:2050] VirtualId# [6:7587241347673870629:0]
2025-12-24T02:11:26.946892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241347673870641:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.946952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.946981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.954756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:26.954866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241348238424537:2050] [node 3] ICS07 socket disconnect 160 reason# EndOfStream
2025-12-24T02:11:26.954872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241348238424537:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.954926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:26.954950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241348238424537:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:26.955522Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241348238424439:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10991) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.955548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241348238424537:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.955554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241348238424439:2050] [node 3] ICP30 unregister session Session# [5:7587241348238424537:2050] VirtualId# [5:7587241348238424525:0]
2025-12-24T02:11:26.955615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:26.955623Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241348238424439:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10991) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.955627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10991) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.974941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:26.975049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241347673870639:2088] [node 4] ICS07 socket disconnect 182 reason# EndOfStream
2025-12-24T02:11:26.975057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241347673870639:2088] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:26.975108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:26.975134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241347673870639:2088] [node 4] ICS15 start handshake
2025-12-24T02:11:26.978986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241347673870594:2088] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19352) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:26.979006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241347673870639:2088] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:26.979012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241347673870594:2088] [node 4] ICP30 unregister session Session# [6:7587241347673870639:2088] VirtualId# [6:7587241347673870627:0]
2025-12-24T02:11:26.979037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:26.979045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241347673870594:2088] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.979049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19352) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:26.990780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241347673870594:2088] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:27.029345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241350783777966:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:27.029373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241350783777967:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.046804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241353342572923:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.046818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241353342572924:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.046803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241351744843871:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.046811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241351744843872:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.046816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241351744843873:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.044424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241351456817181:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.044441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241351456817182:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.050794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241351382148305:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.050805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241351382148306:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.050810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241351382148307:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.075082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353303133775:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.075104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353303133776:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.086849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353303133774:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.144159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353303133796:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:27.143859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241351456817242:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:27.145386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241353342572985:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:27.145479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241353342572986:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:27.155065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353303133814:2073] [node 4] ICP01 ready to work
2025-12-24T02:11:27.154900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241351456817251:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.158901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241350783778033:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-12-24T02:11:27.158873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342572999:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.162738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241350783778032:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:27.162776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241350783778029:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.170960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843932:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.175427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133835:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.175461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133836:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.178853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241353342572998:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:27.179122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241351456817256:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:27.179254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148369:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.179332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148370:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.179353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148371:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.176065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241350783778031:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:27.182840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148369:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.183223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241351382148306:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.183464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241351456817246:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.183476Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241351456817246:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.184055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241351456817246:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.184155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241351456817242:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.184168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.184176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241351456817242:2107] [node 4] ICP22 created new session: [2:7587241351456817262:2107]
2025-12-24T02:11:27.184182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241351456817262:2107] [node 4] ICS09 handshake done sender: [2:7587241351456817246:2109] self: [2:7587241351456817245:0] peer: [4:7587241351382148372:0] socket: 152 qp: -1
2025-12-24T02:11:27.184188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241351456817262:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:27.184212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.187018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148373:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.187276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241351382148369:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.187399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241351382148306:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.187411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.187417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241351382148306:2049] [node 2] ICP22 created new session: [4:7587241351382148374:2049]
2025-12-24T02:11:27.187422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241351382148374:2049] [node 2] ICS09 handshake done sender: [4:7587241351382148369:2109] self: [4:7587241351382148372:0] peer: [2:7587241351456817245:0] socket: 161 qp: -1
2025-12-24T02:11:27.187430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241351382148374:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.187447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.187076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241350783777967:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27132464ns) {Configure:104(27131873ns) {ProcessConfigured:135(26844874ns) {StartInitialHandshake:157(26844632ns) {DropHandshakes:470(337ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(41ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(1037ns) {SwitchToState:179(365ns)}}}}}
2025-12-24T02:11:27.186833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241353303133774:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27651369ns) {Configure:104(27650386ns) {ProcessConfigured:135(27274196ns) {StartInitialHandshake:157(27273441ns) {DropHandshakes:470(725ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(400ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(794ns)}}}}}
2025-12-24T02:11:27.194894Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241350783778029:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32024162ns) {Configure:104(32022887ns) {ProcessConfigured:135(31725151ns) {StartInitialHandshake:157(31724919ns) {DropHandshakes:470(491ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:477(1468ns) {SwitchToState:179(1046ns)}}}}}
2025-12-24T02:11:27.194983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778033:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.195293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241350783778032:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32457009ns) {Configure:104(32456185ns) {ProcessConfigured:135(32126806ns) {StartInitialHandshake:157(32126610ns) {DropHandshakes:470(280ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(757ns) {SwitchToState:179(94ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.207325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241353303133775:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28019950ns) {Configure:104(28017773ns) {ProcessConfigured:135(27640267ns) {StartInitialHandshake:157(27640101ns) {DropHandshakes:470(520ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(384ns) PrepareNewSessionHandshake:477(1068ns) {SwitchToState:179(385ns)}}}}}
2025-12-24T02:11:27.206832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241350783777966:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(59948500ns) {Configure:104(59946804ns) {ProcessConfigured:135(59609679ns) {StartInitialHandshake:157(59608474ns) {DropHandshakes:470(370ns) {DropIncomingHandshake:442(39ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1255ns) {SwitchToState:179(532ns)}}}}}
2025-12-24T02:11:27.206967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241350783778031:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30824724ns) {Configure:104(30824368ns) {ProcessConfigured:135(30481243ns) {StartInitialHandshake:157(30481118ns) {DropHandshakes:470(498ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(834ns) {SwitchToState:179(478ns)}}}}}
2025-12-24T02:11:27.210734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241351456817249:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60439
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.211247Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241353342572923:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60141342ns) {Configure:104(60139843ns) {ProcessConfigured:135(59776699ns) {StartInitialHandshake:157(59775991ns) {DropHandshakes:470(758ns) {DropIncomingHandshake:442(97ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1931ns) {SwitchToState:179(1285ns)}}}}}
2025-12-24T02:11:27.211484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573003:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.218331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241351382148377:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:27.223370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342572999:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.223423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241353342572924:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.228013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133853:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.231000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573005:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.231021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573006:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.231766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353303133836:2102] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.231807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353303133775:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.231943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241351456817253:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.231956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241351456817253:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.232883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133855:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.232997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241351456817253:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.233217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353303133836:2102] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.233292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353303133775:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.233310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.233316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353303133775:2049] [node 2] ICP22 created new session: [6:7587241353303133856:2049]
2025-12-24T02:11:27.233323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353303133856:2049] [node 2] ICS09 handshake done sender: [6:7587241353303133836:2102] self: [6:7587241353303133854:0] peer: [2:7587241351456817252:0] socket: 168 qp: -1
2025-12-24T02:11:27.233331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353303133856:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.233350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.234097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241351456817251:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.234113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.234120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241351456817251:2111] [node 6] ICP22 created new session: [2:7587241351456817266:2111]
2025-12-24T02:11:27.229886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241351744843940:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:27.230002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241351744843941:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:27.234128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241351456817266:2111] [node 6] ICS09 handshake done sender: [2:7587241351456817253:2112] self: [2:7587241351456817252:0] peer: [6:7587241353303133854:0] socket: 165 qp: -1
2025-12-24T02:11:27.234133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241351456817266:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:27.234141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.234782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241353303133776:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71583663ns) {Configure:104(71582763ns) {ProcessConfigured:135(71328676ns) {StartInitialHandshake:157(71327975ns) {DropHandshakes:470(383ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:477(921ns) {SwitchToState:179(352ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.235583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573007:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.235000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843946:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.234922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148384:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.239209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241351456817244:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.239228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241351456817244:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.243253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241350783778047:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-12-24T02:11:27.243275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241350783778048:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-12-24T02:11:27.246983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778047:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.246839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241351744843872:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28033158ns) {Configure:104(28032111ns) {ProcessConfigured:135(27624805ns) {StartInitialHandshake:157(27623788ns) {DropHandshakes:470(575ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(1761ns) {SwitchToState:179(1060ns)}}}}}
2025-12-24T02:11:27.247100Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241351744843871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27869354ns) {Configure:104(27868643ns) {ProcessConfigured:135(27596361ns) {StartInitialHandshake:157(27596197ns) {DropHandshakes:470(237ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(611ns) {SwitchToState:179(337ns)}}}}}
2025-12-24T02:11:27.252719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778048:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.259749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241351382148380:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.274494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148386:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.274616Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241351382148377:2113] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56182101ns) {Configure:104(56180648ns) {ProcessConfigured:135(55794457ns) {StartInitialHandshake:157(55793472ns) {DropHandshakes:470(1455ns) {DropIncomingHandshake:442(266ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:477(1426ns) {SwitchToState:179(403ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.276046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241351456817244:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.276072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148371:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.276075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573008:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.276199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241351456817182:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.276221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.276229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241351456817182:2049] [node 3] ICP22 created new session: [2:7587241351456817270:2049]
2025-12-24T02:11:27.276237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241351456817270:2049] [node 3] ICS09 handshake done sender: [2:7587241351456817244:2108] self: [2:7587241351456817243:0] peer: [3:7587241353342573004:0] socket: 150 qp: -1
2025-12-24T02:11:27.276245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241351456817270:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:27.276261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.276348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241353342572999:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.276441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241353342572924:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.276459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.276465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241353342572924:2049] [node 2] ICP22 created new session: [3:7587241353342573009:2049]
2025-12-24T02:11:27.276471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241353342573009:2049] [node 2] ICS09 handshake done sender: [3:7587241353342572999:2113] self: [3:7587241353342573004:0] peer: [2:7587241351456817243:0] socket: 160 qp: -1
2025-12-24T02:11:27.276478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241353342573009:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.276500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.278443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843951:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.283101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353303133835:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.283159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133860:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.283187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353303133776:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.283714Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241351382148380:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23769795ns) {Configure:104(23768784ns) {ProcessConfigured:135(23282479ns) {StartInitialHandshake:157(23281116ns) {DropHandshakes:470(1210ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(189ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:477(962ns) {SwitchToState:179(161ns)}}}}}
2025-12-24T02:11:27.284328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843946:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.284388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241351744843872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.285759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.286001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241351744843946:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.286128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241351744843872:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.286143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.286149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241351744843872:2049] [node 2] ICP22 created new session: [5:7587241351744843954:2049]
2025-12-24T02:11:27.286155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241351744843954:2049] [node 2] ICS09 handshake done sender: [5:7587241351744843946:2114] self: [5:7587241351744843952:0] peer: [2:7587241351456817259:0] socket: 192 qp: -1
2025-12-24T02:11:27.286162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241351744843954:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.286179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.286567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342572992:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.286583Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241353342572992:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.287472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843932:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.287693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241353342572992:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.287799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241353342572986:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.287811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.287818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241353342572986:2108] [node 6] ICP22 created new session: [3:7587241353342573012:2108]
2025-12-24T02:11:27.287824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241353342573012:2108] [node 6] ICS09 handshake done sender: [3:7587241353342572992:2111] self: [3:7587241353342572991:0] peer: [6:7587241353303133861:0] socket: 156 qp: -1
2025-12-24T02:11:27.287830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241353342573012:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:27.287858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.284527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241351456817260:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.284540Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241351456817260:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.285169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241351456817260:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.285279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241351456817256:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.285292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.285302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241351456817256:2113] [node 5] ICP22 created new session: [2:7587241351456817273:2113]
2025-12-24T02:11:27.285309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241351456817273:2113] [node 5] ICS09 handshake done sender: [2:7587241351456817260:2115] self: [2:7587241351456817259:0] peer: [5:7587241351744843952:0] socket: 191 qp: -1
2025-12-24T02:11:27.285317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241351456817273:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:27.285328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.294771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148370:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.294904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241351382148307:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.295242Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241351456817182:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(18944647ns) {HandleHandshakeStatus:329(18943861ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(771ns) SwitchToState:179(356ns) ProcessPendingSessionEvents:495(18831428ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18809304ns max# 18723572ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.299895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133862:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.303128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342572990:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.303141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241353342572990:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.303102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148387:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.303115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241351382148387:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.303948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241351382148387:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.304381Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353303133835:2101] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.304069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241351382148377:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.304085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.304094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241351382148377:2113] [node 5] ICP22 created new session: [4:7587241351382148393:2113]
2025-12-24T02:11:27.304101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241351382148393:2113] [node 5] ICS09 handshake done sender: [4:7587241351382148387:2118] self: [4:7587241351382148378:0] peer: [5:7587241351744843959:0] socket: 200 qp: -1
2025-12-24T02:11:27.304108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241351382148393:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:27.304129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.304526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133863:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.304653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353303133776:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.304670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.304676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353303133776:2050] [node 3] ICP22 created new session: [6:7587241353303133864:2050]
2025-12-24T02:11:27.304681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353303133864:2050] [node 3] ICS09 handshake done sender: [6:7587241353303133835:2101] self: [6:7587241353303133861:0] peer: [3:7587241353342572991:0] socket: 162 qp: -1
2025-12-24T02:11:27.304687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353303133864:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.304704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.303845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342573007:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.306167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342573006:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.302787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843951:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.302936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241351744843940:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.303053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843958:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.304315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843960:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.305629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241351744843951:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.305708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241351744843940:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.305717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.305722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241351744843940:2110] [node 4] ICP22 created new session: [5:7587241351744843961:2110]
2025-12-24T02:11:27.305727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241351744843961:2110] [node 4] ICS09 handshake done sender: [5:7587241351744843951:2117] self: [5:7587241351744843959:0] peer: [4:7587241351382148378:0] socket: 203 qp: -1
2025-12-24T02:11:27.305733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241351744843961:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:27.305739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.315108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843958:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.315153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241351744843873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.317109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843964:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.317356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241351744843958:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.311564Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241353342572986:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(23629493ns) {HandleHandshakeStatus:329(23628789ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(165ns) SwitchToState:179(247ns) ProcessPendingSessionEvents:495(12364133ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12342236ns max# 12260868ns}}}
2025-12-24T02:11:27.311741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241353342572986:2108] [node 6] ICP18 handshake (actor [3:7587241353342573007:2119]) is too old
2025-12-24T02:11:27.315265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342573001:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.315272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241353342573001:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.315966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241353342573001:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.316068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241353342572998:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.316081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.316090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241353342572998:2112] [node 5] ICP22 created new session: [3:7587241353342573017:2112]
2025-12-24T02:11:27.316097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241353342573017:2112] [node 5] ICS09 handshake done sender: [3:7587241353342573001:2114] self: [3:7587241353342573000:0] peer: [5:7587241351744843963:0] socket: 209 qp: -1
2025-12-24T02:11:27.316105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241353342573017:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:27.316113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.318807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353303133860:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.318870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353303133796:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.322982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241351456817182:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:27.323017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353303133863:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.323078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353303133814:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.322874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843945:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.322884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241351744843945:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.323509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241351744843945:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.323277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148389:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.323289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241351382148389:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.323918Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241351382148389:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.323647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241351744843941:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.323666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.323673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241351744843941:2111] [node 6] ICP22 created new session: [5:7587241351744843966:2111]
2025-12-24T02:11:27.323681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241351744843966:2111] [node 6] ICS09 handshake done sender: [5:7587241351744843945:2113] self: [5:7587241351744843944:0] peer: [6:7587241353303133866:0] socket: 194 qp: -1
2025-12-24T02:11:27.323689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241351744843966:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:27.323699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.324006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241351382148380:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.324026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.324034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241351382148380:2114] [node 6] ICP22 created new session: [4:7587241351382148397:2114]
2025-12-24T02:11:27.324040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241351382148397:2114] [node 6] ICS09 handshake done sender: [4:7587241351382148389:2119] self: [4:7587241351382148385:0] peer: [6:7587241353303133867:0] socket: 212 qp: -1
2025-12-24T02:11:27.324045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241351382148397:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:27.324052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.324292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241351744843873:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.324305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.324311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241351744843873:2050] [node 3] ICP22 created new session: [5:7587241351744843968:2050]
2025-12-24T02:11:27.324316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241351744843968:2050] [node 3] ICS09 handshake done sender: [5:7587241351744843958:2119] self: [5:7587241351744843963:0] peer: [3:7587241353342573000:0] socket: 210 qp: -1
2025-12-24T02:11:27.324320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241351744843968:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.324342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.326740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241353303133796:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9359) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241353303133860:2119] held: no
2025-12-24T02:11:27.331355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241350783778052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-12-24T02:11:27.334712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241350783778051:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:499
2025-12-24T02:11:27.335443Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241351382148384:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.335498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241351382148305:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.338674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133869:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.338838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133868:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.338855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778051:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.338964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778039:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.338968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241350783778039:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.342992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148400:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
2025-12-24T02:11:27.343033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241351382148401:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13897
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.342804Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241350783778039:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.342924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241350783778029:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.342944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.342952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241350783778029:2107] [node 4] ICP22 created new session: [1:7587241350783778055:2107]
2025-12-24T02:11:27.342960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241350783778055:2107] [node 4] ICS09 handshake done sender: [1:7587241350783778039:2112] self: [1:7587241350783778035:0] peer: [4:7587241351382148399:0] socket: 178 qp: -1
2025-12-24T02:11:27.342968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241350783778055:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:27.342979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.345329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778045:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.345335Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241350783778045:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.345956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241350783778045:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.343523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241351456817275:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60439
2025-12-24T02:11:27.345155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241351456817275:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.345193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241351456817181:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.345806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241351456817277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60439
2025-12-24T02:11:27.346134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241351456817275:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.346861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241350783777966:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.346886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.346892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241350783777966:2048] [node 2] ICP22 created new session: [1:7587241350783778059:2048]
2025-12-24T02:11:27.346898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241350783778059:2048] [node 2] ICS09 handshake done sender: [1:7587241350783778045:2114] self: [1:7587241350783778030:0] peer: [2:7587241351456817276:0] socket: 167 qp: -1
2025-12-24T02:11:27.346903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241350783778059:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:27.346924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.348097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241351382148370:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.346502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241351456817181:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.346529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.346537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241351456817181:2048] [node 1] ICP22 created new session: [2:7587241351456817278:2048]
2025-12-24T02:11:27.346543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241351456817278:2048] [node 1] ICS09 handshake done sender: [2:7587241351456817275:2116] self: [2:7587241351456817276:0] peer: [1:7587241350783778030:0] socket: 220 qp: -1
2025-12-24T02:11:27.346551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241351456817278:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.346565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.346894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241353342572990:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.347904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241353342573003:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.347968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241353342572985:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.347982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.347988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241353342572985:2107] [node 4] ICP22 created new session: [3:7587241353342573021:2107]
2025-12-24T02:11:27.349284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778052:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.347994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241353342573021:2107] [node 4] ICS09 handshake done sender: [3:7587241353342572990:2110] self: [3:7587241353342572989:0] peer: [4:7587241351382148391:0] socket: 155 qp: -1
2025-12-24T02:11:27.348000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241353342573021:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:27.348007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.350345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241353342572923:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.350715Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241353342572998:2112] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:27.350583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353303133860:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.350919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353303133796:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.350934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.350941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353303133796:2068] [node 5] ICP22 created new session: [6:7587241353303133870:2068]
2025-12-24T02:11:27.350947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353303133870:2068] [node 5] ICS09 handshake done sender: [6:7587241353303133860:2119] self: [6:7587241353303133866:0] peer: [5:7587241351744843944:0] socket: 197 qp: -1
2025-12-24T02:11:27.350953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353303133870:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:27.350962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.350939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843970:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.355059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241351744843970:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.355208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241351744843871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.356918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241351382148305:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241351382148384:2116] held: no
2025-12-24T02:11:27.357142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241353303133814:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18742) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241353303133863:2121] held: no
2025-12-24T02:11:27.358955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353303133863:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.357075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778038:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.357091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241350783778038:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.358810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778046:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.358820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241350783778046:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.359617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241350783778046:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.359638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241351744843972:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36644
2025-12-24T02:11:27.359332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353303133853:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.359347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353303133814:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.359359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.359367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353303133814:2073] [node 4] ICP22 created new session: [6:7587241353303133872:2073]
2025-12-24T02:11:27.359373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353303133872:2073] [node 4] ICS09 handshake done sender: [6:7587241353303133863:2121] self: [6:7587241353303133867:0] peer: [4:7587241351382148385:0] socket: 214 qp: -1
2025-12-24T02:11:27.359378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353303133872:2073] [node 4] ICS10 traffic start
2025-12-24T02:11:27.359387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.359687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353303133774:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.359741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241350783778031:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.359760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.359768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241350783778031:2108] [node 5] ICP22 created new session: [1:7587241350783778064:2108]
2025-12-24T02:11:27.359775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241350783778064:2108] [node 5] ICS09 handshake done sender: [1:7587241350783778046:2115] self: [1:7587241350783778037:0] peer: [5:7587241351744843971:0] socket: 223 qp: -1
2025-12-24T02:11:27.359781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241350783778064:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:27.359802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.359894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241351744843970:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.360000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241351744843871:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.360017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.360023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241351744843871:2048] [node 1] ICP22 created new session: [5:7587241351744843973:2048]
2025-12-24T02:11:27.360029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241351744843973:2048] [node 1] ICS09 handshake done sender: [5:7587241351744843970:2122] self: [5:7587241351744843971:0] peer: [1:7587241350783778037:0] socket: 173 qp: -1
2025-12-24T02:11:27.360035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241351744843973:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.360043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.359972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241350783778043:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.359979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241350783778043:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.360944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241350783778043:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.361032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241350783778032:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.361050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.361056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241350783778032:2109] [node 6] ICP22 created new session: [1:7587241350783778067:2109]
2025-12-24T02:11:27.361062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241350783778067:2109] [node 6] ICS09 handshake done sender: [1:7587241350783778043:2113] self: [1:7587241350783778036:0] peer: [6:7587241353303133874:0] socket: 179 qp: -1
2025-12-24T02:11:27.361068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241350783778067:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:27.361074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.365415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353303133875:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42277
2025-12-24T02:11:27.365569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353303133853:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.365649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353303133774:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.365665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.365673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353303133774:2048] [node 1] ICP22 created new session: [6:7587241353303133876:2048]
2025-12-24T02:11:27.365680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353303133876:2048] [node 1] ICS09 handshake done sender: [6:7587241353303133853:2116] self: [6:7587241353303133874:0] peer: [1:7587241350783778036:0] socket: 183 qp: -1
2025-12-24T02:11:27.365686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353303133876:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.365693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.371438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241351382148384:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.371709Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241351382148377:2113] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:27.371724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241351382148305:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.371739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.371749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241351382148305:2048] [node 1] ICP22 created new session: [4:7587241351382148402:2048]
2025-12-24T02:11:27.371754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241351382148402:2048] [node 1] ICS09 handshake done sender: [4:7587241351382148384:2116] self: [4:7587241351382148399:0] peer: [1:7587241350783778035:0] socket: 181 qp: -1
2025-12-24T02:11:27.371763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241351382148402:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.371772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.379454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241350783778038:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.379491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241351382148307:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.379521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.379549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241351382148307:2050] [node 3] ICP22 created new session: [4:7587241351382148404:2050]
2025-12-24T02:11:27.379558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241351382148404:2050] [node 3] ICS09 handshake done sender: [4:7587241351382148370:2110] self: [4:7587241351382148391:0] peer: [3:7587241353342572989:0] socket: 170 qp: -1
2025-12-24T02:11:27.379566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241351382148404:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.379582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.379706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241353342572985:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(31606316ns) {HandleHandshakeStatus:329(31605483ns) {DropIncomingHandshake:442(3759ns) DropOutgoingHandshake:457(140ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(27793665ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27767759ns max# 27667742ns}}}
2025-12-24T02:11:27.380008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241350783777967:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.380028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.380034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241350783777967:2049] [node 3] ICP22 created new session: [1:7587241350783778070:2049]
2025-12-24T02:11:27.380040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241350783778070:2049] [node 3] ICS09 handshake done sender: [1:7587241350783778038:2111] self: [1:7587241350783778034:0] peer: [3:7587241353342573023:0] socket: 177 qp: -1
2025-12-24T02:11:27.380045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241350783778070:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:27.380052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.384050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241351744843871:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(23941843ns) {HandleHandshakeStatus:329(23941261ns) {DropIncomingHandshake:442(157ns) DropOutgoingHandshake:457(3131ns) SwitchToState:179(79ns) ProcessPendingSessionEvents:495(23859368ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23833974ns max# 23746706ns}}}
2025-12-24T02:11:27.383660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241353342573024:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9300
2025-12-24T02:11:27.384678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241353342573003:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.384743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241353342572923:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.384758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.384765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241353342572923:2048] [node 1] ICP22 created new session: [3:7587241353342573025:2048]
2025-12-24T02:11:27.384772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241353342573025:2048] [node 1] ICS09 handshake done sender: [3:7587241353342573003:2116] self: [3:7587241353342573023:0] peer: [1:7587241350783778034:0] socket: 180 qp: -1
2025-12-24T02:11:27.384779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241353342573025:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.384786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.387442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241351382148307:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(30450269ns) {HandleHandshakeStatus:329(30449425ns) {DropIncomingHandshake:442(814ns) DropOutgoingHandshake:457(2712ns) SwitchToState:179(311ns) ProcessPendingSessionEvents:495(7706839ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7685535ns max# 7610164ns}}}
2025-12-24T02:11:27.387473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241351382148307:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:27.414849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:27.414958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241351456817278:2048] [node 1] ICS01 socket: 220 reason# EndOfStream
2025-12-24T02:11:27.414969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241351456817181:2048] [node 1] ICP30 unregister session Session# [2:7587241351456817278:2048] VirtualId# [2:7587241351456817276:0]
2025-12-24T02:11:27.414979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241351456817278:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.415024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.415049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.419039Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:27.419143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241353342573025:2048] [node 1] ICS07 socket disconnect 180 reason# EndOfStream
2025-12-24T02:11:27.419150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241353342573025:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.419183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.419199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241353342573025:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:27.419929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241353342572923:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:27.419942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241353342573025:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:27.419948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241353342572923:2048] [node 1] ICP30 unregister session Session# [3:7587241353342573025:2048] VirtualId# [3:7587241353342573023:0]
2025-12-24T02:11:27.420003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.420010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241353342572923:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.420014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.422733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:27.422850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:27.422831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241351382148402:2048] [node 1] ICS01 socket: 181 reason# EndOfStream
2025-12-24T02:11:27.422940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241351744843973:2048] [node 1] ICS07 socket disconnect 173 reason# EndOfStream
2025-12-24T02:11:27.422840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241351382148305:2048] [node 1] ICP30 unregister session Session# [4:7587241351382148402:2048] VirtualId# [4:7587241351382148399:0]
2025-12-24T02:11:27.422849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241351382148402:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.422910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.422936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.422946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241351744843973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.422989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.423010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241351744843973:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:27.423068Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:27.423137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353303133876:2048] [node 1] ICS01 socket: 183 reason# EndOfStream
2025-12-24T02:11:27.423143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353303133774:2048] [node 1] ICP30 unregister session Session# [6:7587241353303133876:2048] VirtualId# [6:7587241353303133874:0]
2025-12-24T02:11:27.423148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353303133876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.423188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.423212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.423473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241351744843871:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:27.423485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241351744843973:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:27.423490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241351744843871:2048] [node 1] ICP30 unregister session Session# [5:7587241351744843973:2048] VirtualId# [5:7587241351744843971:0]
2025-12-24T02:11:27.423548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.423553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241351744843871:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.423558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62209) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.430879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241353342572923:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:27.431916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:27.432002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241353342573009:2049] [node 2] ICS01 socket: 160 reason# EndOfStream
2025-12-24T02:11:27.432009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241353342572924:2049] [node 2] ICP30 unregister session Session# [3:7587241353342573009:2049] VirtualId# [3:7587241353342573004:0]
2025-12-24T02:11:27.432016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241353342573009:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.432047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.432065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.433150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241351744843871:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:27.439203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:27.439310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241351744843954:2049] [node 2] ICS01 socket: 192 reason# EndOfStream
2025-12-24T02:11:27.439319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241351744843872:2049] [node 2] ICP30 unregister session Session# [5:7587241351744843954:2049] VirtualId# [5:7587241351744843952:0]
2025-12-24T02:11:27.439327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241351744843954:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.439373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.439405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.440410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:27.440499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241351382148374:2049] [node 2] ICS07 socket disconnect 161 reason# EndOfStream
2025-12-24T02:11:27.440505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241351382148374:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.440566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.440591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241351382148374:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:27.441516Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241351382148306:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6124) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:27.441531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241351382148374:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:27.441537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241351382148306:2049] [node 2] ICP30 unregister session Session# [4:7587241351382148374:2049] VirtualId# [4:7587241351382148372:0]
2025-12-24T02:11:27.441591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.441598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241351382148306:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6124) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.441601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6124) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:27.446755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:27.446861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353303133856:2049] [node 2] ICS01 socket: 168 reason# EndOfStream
2025-12-24T02:11:27.446870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353303133775:2049] [node 2] ICP30 unregister session Session# [6:7587241353303133856:2049] VirtualId# [6:7587241353303133854:0]
2025-12-24T02:11:27.446877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353303133856:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.446927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.446952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.453915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:27.454022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241351744843968:2050] [node 3] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:11:27.454033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241351744843873:2050] [node 3] ICP30 unregister session Session# [5:7587241351744843968:2050] VirtualId# [5:7587241351744843963:0]
2025-12-24T02:11:27.454042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241351744843968:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.454102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:27.454128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:27.462601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:27.462738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353303133864:2050] [node 3] ICS01 socket: 162 reason# EndOfStream
2025-12-24T02:11:27.462751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353303133776:2050] [node 3] ICP30 unregister session Session# [6:7587241353303133864:2050] VirtualId# [6:7587241353303133861:0]
2025-12-24T02:11:27.462759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353303133864:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.462824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:27.462854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:27.471364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:27.471467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353303133872:2073] [node 4] ICS01 socket: 214 reason# EndOfStream
2025-12-24T02:11:27.471479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353303133814:2073] [node 4] ICP30 unregister session Session# [6:7587241353303133872:2073] VirtualId# [6:7587241353303133867:0]
2025-12-24T02:11:27.471487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353303133872:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.471555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:27.471580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:27.525894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350508460415:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.525941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350508460416:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.525948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350508460417:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.509470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353902869336:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.509480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353902869337:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.509485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353902869338:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.537457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353580010645:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.537487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353580010646:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.537494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353580010647:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.539073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241353195841617:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:27.539107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241353195841618:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.542712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241350476921607:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.542746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241350476921608:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.542724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352543671308:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.542763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352543671309:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.591020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350508460476:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:27.592169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353902869357:2063] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.592557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921667:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.591191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241353195841679:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:27.591288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241353195841680:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.593323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460479:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.593341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460480:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.593144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241353902869370:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.609474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869369:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.609843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241353195841695:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:27.609960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241353195841693:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:505
2025-12-24T02:11:27.609974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241353195841694:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:505
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.627369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241350476921672:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:27.627501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241350476921673:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:27.627623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241350476921674:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:27.628190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921671:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.658899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241350508460416:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39629330ns) {Configure:104(39627249ns) {ProcessConfigured:135(39232223ns) {StartInitialHandshake:157(39230386ns) {DropHandshakes:470(1600ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(201ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:477(1877ns) {SwitchToState:179(1300ns)}}}}}
2025-12-24T02:11:27.659351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350508460486:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.667667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460487:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.673642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352543671378:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:27.673890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352543671379:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:27.674046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352543671380:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:27.675378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671374:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.675447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671375:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.675490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671376:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.675504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671377:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.676114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869399:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.676596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.679572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921688:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.680620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353902869399:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.680663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353902869357:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.680818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671375:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.695199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353580010714:2083] [node 4] ICP01 ready to work
2025-12-24T02:11:27.695422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353580010715:2088] [node 6] ICP01 ready to work
2025-12-24T02:11:27.696936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921667:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.721765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671376:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.721801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671374:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.722159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353902869416:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.722194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353902869337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.723031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869419:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.723329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353902869416:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.723396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353902869337:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.723416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.723423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353902869337:2049] [node 2] ICP22 created new session: [6:7587241353902869420:2049]
2025-12-24T02:11:27.723430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353902869420:2049] [node 2] ICS09 handshake done sender: [6:7587241353902869416:2117] self: [6:7587241353902869418:0] peer: [2:7587241352543671381:0] socket: 193 qp: -1
2025-12-24T02:11:27.723439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353902869420:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.723448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.725396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921688:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.725442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241350476921608:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.722317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671382:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.722324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352543671382:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.723128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352543671382:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.724413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352543671378:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.724426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.724432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352543671378:2108] [node 6] ICP22 created new session: [2:7587241352543671395:2108]
2025-12-24T02:11:27.724436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241352543671395:2108] [node 6] ICS09 handshake done sender: [2:7587241352543671382:2115] self: [2:7587241352543671381:0] peer: [6:7587241353902869418:0] socket: 191 qp: -1
2025-12-24T02:11:27.724441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241352543671395:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:27.724448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.725759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460487:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.725797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460480:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.725825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460478:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.725829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241350508460478:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.726803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460496:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.729392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841693:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.734156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671384:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.734170Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352543671384:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.734522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671377:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.734313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869423:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.735193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352543671384:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.735318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352543671309:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.735344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.735351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352543671309:2049] [node 3] ICP22 created new session: [2:7587241352543671400:2049]
2025-12-24T02:11:27.735359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241352543671400:2049] [node 3] ICS09 handshake done sender: [2:7587241352543671384:2116] self: [2:7587241352543671383:0] peer: [3:7587241350476921690:0] socket: 192 qp: -1
2025-12-24T02:11:27.735366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241352543671400:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:27.735374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.735041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.735075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921692:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.736989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241350476921688:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.738566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241350476921608:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.738589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.738596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241350476921608:2049] [node 2] ICP22 created new session: [3:7587241350476921693:2049]
2025-12-24T02:11:27.738604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241350476921693:2049] [node 2] ICS09 handshake done sender: [3:7587241350476921688:2117] self: [3:7587241350476921690:0] peer: [2:7587241352543671383:0] socket: 194 qp: -1
2025-12-24T02:11:27.738613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241350476921693:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.738622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.741758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921691:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.743094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010723:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.743185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010724:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.743394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010723:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.743422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353580010647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.744004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010726:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.744323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353580010723:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.744376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353580010647:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.744389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.744395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353580010647:2050] [node 3] ICP22 created new session: [5:7587241353580010727:2050]
2025-12-24T02:11:27.744402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241353580010727:2050] [node 3] ICS09 handshake done sender: [5:7587241353580010723:2115] self: [5:7587241353580010725:0] peer: [3:7587241350476921681:0] socket: 205 qp: -1
2025-12-24T02:11:27.744409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241353580010727:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.744417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.743513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921682:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.743525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241350476921682:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.744149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241350476921682:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.745810Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869424:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.747108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350508460417:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.747290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460500:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.747663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921676:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.747674Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241350476921676:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.747949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241350476921673:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.747965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.747971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241350476921673:2109] [node 5] ICP22 created new session: [3:7587241350476921698:2109]
2025-12-24T02:11:27.747977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241350476921698:2109] [node 5] ICS09 handshake done sender: [3:7587241350476921682:2115] self: [3:7587241350476921681:0] peer: [5:7587241353580010725:0] socket: 181 qp: -1
2025-12-24T02:11:27.747984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241350476921698:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:27.747992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.748964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869425:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.749065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241350476921676:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.749144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241350476921672:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.749156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.749161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241350476921672:2108] [node 4] ICP22 created new session: [3:7587241350476921702:2108]
2025-12-24T02:11:27.749166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241350476921702:2108] [node 4] ICS09 handshake done sender: [3:7587241350476921676:2112] self: [3:7587241350476921675:0] peer: [4:7587241350508460498:0] socket: 179 qp: -1
2025-12-24T02:11:27.749170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241350476921702:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:27.749176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.751044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841694:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.751082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241353195841698:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:505
2025-12-24T02:11:27.751115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241353195841699:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:505
2025-12-24T02:11:27.751270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353902869369:2073] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.751297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353902869336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.751908Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350508460478:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.754988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353902869399:2100] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.755179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921671:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241350476921607:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.755092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841699:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841686:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755140Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241353195841686:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.755507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841698:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841682:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241353195841682:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.755059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353902869357:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.755077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.755083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353902869357:2063] [node 4] ICP22 created new session: [6:7587241353902869427:2063]
2025-12-24T02:11:27.755090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353902869427:2063] [node 4] ICS09 handshake done sender: [6:7587241353902869399:2100] self: [6:7587241353902869417:0] peer: [4:7587241350508460477:0] socket: 177 qp: -1
2025-12-24T02:11:27.755097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353902869427:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:27.755108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.755468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353902869425:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.755485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353902869338:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.756115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241350508460476:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.756136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.756142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350508460476:2107] [node 6] ICP22 created new session: [4:7587241350508460502:2107]
2025-12-24T02:11:27.756147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241350508460502:2107] [node 6] ICS09 handshake done sender: [4:7587241350508460478:2108] self: [4:7587241350508460477:0] peer: [6:7587241353902869417:0] socket: 171 qp: -1
2025-12-24T02:11:27.756153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241350508460502:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:27.756159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.756934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460479:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.756963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460501:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.757096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350508460487:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.757145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350508460417:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.757158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.757162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350508460417:2050] [node 3] ICP22 created new session: [4:7587241350508460505:2050]
2025-12-24T02:11:27.757166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241350508460505:2050] [node 3] ICS09 handshake done sender: [4:7587241350508460487:2114] self: [4:7587241350508460498:0] peer: [3:7587241350476921675:0] socket: 184 qp: -1
2025-12-24T02:11:27.757170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241350508460505:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.757175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.758853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350508460415:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.758778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241350476921684:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.758789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241350476921684:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.759589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921706:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.760144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010724:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.760473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353580010646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.760664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671388:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.760678Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352543671388:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.761363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352543671388:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.761468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352543671380:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.761488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.761494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352543671380:2110] [node 5] ICP22 created new session: [2:7587241352543671404:2110]
2025-12-24T02:11:27.761500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241352543671404:2110] [node 5] ICS09 handshake done sender: [2:7587241352543671388:2118] self: [2:7587241352543671387:0] peer: [5:7587241353580010730:0] socket: 189 qp: -1
2025-12-24T02:11:27.761507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241352543671404:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:27.761515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.760337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869430:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.760359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869431:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.760949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241353902869424:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.760517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241353195841686:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.760662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241353195841682:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.772002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241353580010646:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241353580010724:2116] held: no
2025-12-24T02:11:27.772264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010731:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.772278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010732:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.772306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.772602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353580010724:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.772680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353580010646:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.772700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.772707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353580010646:2049] [node 2] ICP22 created new session: [5:7587241353580010735:2049]
2025-12-24T02:11:27.772715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241353580010735:2049] [node 2] ICS09 handshake done sender: [5:7587241353580010724:2116] self: [5:7587241353580010730:0] peer: [2:7587241352543671387:0] socket: 206 qp: -1
2025-12-24T02:11:27.772723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241353580010735:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.772731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.775059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460496:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.775210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460500:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.775285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350508460416:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.778788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671386:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.778807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352543671386:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.779430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010731:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.779470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353580010714:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.778806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841684:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.778825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241353195841684:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.782862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241350476921684:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.785858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350508460493:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:27.785885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241350508460493:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.786637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010738:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.787049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350508460493:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.787369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241353195841680:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.787401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.787410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241353195841680:2108] [node 6] ICP22 created new session: [1:7587241353195841706:2108]
2025-12-24T02:11:27.787419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241353195841706:2108] [node 6] ICS09 handshake done sender: [1:7587241353195841686:2111] self: [1:7587241353195841685:0] peer: [6:7587241353902869426:0] socket: 166 qp: -1
2025-12-24T02:11:27.787437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241353195841706:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:27.787451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.787744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241353195841618:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.787755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.787759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241353195841618:2049] [node 3] ICP22 created new session: [1:7587241353195841708:2049]
2025-12-24T02:11:27.787764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241353195841708:2049] [node 3] ICS09 handshake done sender: [1:7587241353195841682:2109] self: [1:7587241353195841681:0] peer: [3:7587241350476921704:0] socket: 161 qp: -1
2025-12-24T02:11:27.787767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241353195841708:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:27.787772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.791351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352543671386:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.791862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352543671379:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.791884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.791900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352543671379:2109] [node 4] ICP22 created new session: [2:7587241352543671408:2109]
2025-12-24T02:11:27.791908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241352543671408:2109] [node 4] ICS09 handshake done sender: [2:7587241352543671386:2117] self: [2:7587241352543671385:0] peer: [4:7587241350508460510:0] socket: 201 qp: -1
2025-12-24T02:11:27.791914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241352543671408:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:27.791922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.792460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671407:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.791622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241353195841710:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:505
2025-12-24T02:11:27.792588Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241353195841684:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:27.792674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241353195841679:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.792687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.792694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241353195841679:2107] [node 4] ICP22 created new session: [1:7587241353195841714:2107]
2025-12-24T02:11:27.792700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241353195841714:2107] [node 4] ICS09 handshake done sender: [1:7587241353195841684:2110] self: [1:7587241353195841683:0] peer: [4:7587241350508460508:0] socket: 164 qp: -1
2025-12-24T02:11:27.792706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241353195841714:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:27.792714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.794085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241350476921674:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.794110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.794117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241350476921674:2110] [node 6] ICP22 created new session: [3:7587241350476921708:2110]
2025-12-24T02:11:27.794123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241350476921708:2110] [node 6] ICS09 handshake done sender: [3:7587241350476921684:2116] self: [3:7587241350476921683:0] peer: [6:7587241353902869429:0] socket: 211 qp: -1
2025-12-24T02:11:27.794128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241350476921708:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:27.794136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.795001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353580010731:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.794636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353902869425:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:27.796916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353580010714:2083] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.796935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:27.796944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353580010714:2083] [node 4] ICP22 created new session: [5:7587241353580010740:2083]
2025-12-24T02:11:27.796950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241353580010740:2083] [node 4] ICS09 handshake done sender: [5:7587241353580010731:2118] self: [5:7587241353580010737:0] peer: [4:7587241350508460492:0] socket: 187 qp: -1
2025-12-24T02:11:27.796957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241353580010740:2083] [node 4] ICS10 traffic start
2025-12-24T02:11:27.796966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:27.797235Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010739:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.799212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353902869338:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.799234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:27.799241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353902869338:2050] [node 3] ICP22 created new session: [6:7587241353902869433:2050]
2025-12-24T02:11:27.799247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353902869433:2050] [node 3] ICS09 handshake done sender: [6:7587241353902869425:2121] self: [6:7587241353902869429:0] peer: [3:7587241350476921683:0] socket: 212 qp: -1
2025-12-24T02:11:27.799255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353902869433:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:27.799263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:27.801956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841710:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.804033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241350476921607:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241350476921671:2111] held: no
2025-12-24T02:11:27.804164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241350476921705:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3340
2025-12-24T02:11:27.805040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353902869369:2073] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.805384Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241350476921671:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.805531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241350476921607:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.805544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.805550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241350476921607:2048] [node 1] ICP22 created new session: [3:7587241350476921710:2048]
2025-12-24T02:11:27.805557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241350476921710:2048] [node 1] ICS09 handshake done sender: [3:7587241350476921671:2111] self: [3:7587241350476921704:0] peer: [1:7587241353195841681:0] socket: 175 qp: -1
2025-12-24T02:11:27.805563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241350476921710:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.805570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.802863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350508460416:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28080) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241350508460500:2119] held: no
2025-12-24T02:11:27.802979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350508460415:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241350508460479:2109] held: no
2025-12-24T02:11:27.803128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241350508460486:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.803142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.803160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350508460486:2112] [node 5] ICP22 created new session: [4:7587241350508460514:2112]
2025-12-24T02:11:27.803167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241350508460514:2112] [node 5] ICS09 handshake done sender: [4:7587241350508460493:2117] self: [4:7587241350508460492:0] peer: [5:7587241353580010737:0] socket: 170 qp: -1
2025-12-24T02:11:27.803172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241350508460514:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:27.803180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.806064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.806199Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350508460513:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62533
2025-12-24T02:11:27.806370Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350508460500:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.806483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352543671407:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.806533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241352543671308:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.806531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010739:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.806824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353580010645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.806962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350508460479:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.806918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841688:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.806931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241353195841688:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.807325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241353195841697:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.807329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241353195841697:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.808174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352543671411:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45165
2025-12-24T02:11:27.808285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241353195841688:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:27.808488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352543671407:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:27.808638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241352543671308:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.808654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.808661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352543671308:2048] [node 1] ICP22 created new session: [2:7587241352543671412:2048]
2025-12-24T02:11:27.808669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241352543671412:2048] [node 1] ICS09 handshake done sender: [2:7587241352543671407:2119] self: [2:7587241352543671410:0] peer: [1:7587241353195841687:0] socket: 228 qp: -1
2025-12-24T02:11:27.808675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241352543671412:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.808686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.809809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241353195841697:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.809914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241353195841617:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.809935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.809942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241353195841617:2048] [node 2] ICP22 created new session: [1:7587241353195841719:2048]
2025-12-24T02:11:27.809949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241353195841719:2048] [node 2] ICS09 handshake done sender: [1:7587241353195841688:2112] self: [1:7587241353195841687:0] peer: [2:7587241352543671410:0] socket: 226 qp: -1
2025-12-24T02:11:27.809955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241353195841719:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:27.809963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.810352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241353195841695:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.810365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.810370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241353195841695:2113] [node 5] ICP22 created new session: [1:7587241353195841721:2113]
2025-12-24T02:11:27.810376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241353195841721:2113] [node 5] ICS09 handshake done sender: [1:7587241353195841697:2116] self: [1:7587241353195841696:0] peer: [5:7587241353580010742:0] socket: 227 qp: -1
2025-12-24T02:11:27.810380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241353195841721:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:27.810387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.810877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350508460416:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.810892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:27.810899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350508460416:2049] [node 2] ICP22 created new session: [4:7587241350508460516:2049]
2025-12-24T02:11:27.810906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241350508460516:2049] [node 2] ICS09 handshake done sender: [4:7587241350508460500:2119] self: [4:7587241350508460510:0] peer: [2:7587241352543671385:0] socket: 210 qp: -1
2025-12-24T02:11:27.810912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241350508460516:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:27.810918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:27.811129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350508460415:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.811135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.811139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350508460415:2048] [node 1] ICP22 created new session: [4:7587241350508460518:2048]
2025-12-24T02:11:27.811143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241350508460518:2048] [node 1] ICS09 handshake done sender: [4:7587241350508460479:2109] self: [4:7587241350508460508:0] peer: [1:7587241353195841683:0] socket: 169 qp: -1
2025-12-24T02:11:27.811148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241350508460518:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.811154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.812957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241353902869370:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:27.813035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353902869336:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.813046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.813052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353902869336:2048] [node 1] ICP22 created new session: [6:7587241353902869436:2048]
2025-12-24T02:11:27.813058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353902869436:2048] [node 1] ICS09 handshake done sender: [6:7587241353902869369:2073] self: [6:7587241353902869426:0] peer: [1:7587241353195841685:0] socket: 168 qp: -1
2025-12-24T02:11:27.813063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353902869436:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.813070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.813235Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241353902869336:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.813813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241353580010645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63745) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241353580010739:2122] held: no
2025-12-24T02:11:27.814024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010732:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.814058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353580010719:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.814061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241353580010719:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:27.814557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353580010743:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45073
2025-12-24T02:11:27.814922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353580010739:2122] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.815052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241353902869438:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63781
2025-12-24T02:11:27.815162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353580010645:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.815175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:27.815180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353580010645:2048] [node 1] ICP22 created new session: [5:7587241353580010746:2048]
2025-12-24T02:11:27.815187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241353580010746:2048] [node 1] ICS09 handshake done sender: [5:7587241353580010739:2122] self: [5:7587241353580010742:0] peer: [1:7587241353195841696:0] socket: 198 qp: -1
2025-12-24T02:11:27.815192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241353580010746:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:27.815198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:27.815644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353580010719:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:27.815845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241353902869424:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:27.815913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241353902869370:2071] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:27.815927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:27.815932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241353902869370:2071] [node 5] ICP22 created new session: [6:7587241353902869439:2071]
2025-12-24T02:11:27.815937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241353902869439:2071] [node 5] ICS09 handshake done sender: [6:7587241353902869424:2120] self: [6:7587241353902869435:0] peer: [5:7587241353580010718:0] socket: 209 qp: -1
2025-12-24T02:11:27.815942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241353902869439:2071] [node 5] ICS10 traffic start
2025-12-24T02:11:27.815949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:27.817066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241353580010715:2088] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:27.817082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:27.817089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353580010715:2088] [node 6] ICP22 created new session: [5:7587241353580010748:2088]
2025-12-24T02:11:27.817095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241353580010748:2088] [node 6] ICS09 handshake done sender: [5:7587241353580010719:2113] self: [5:7587241353580010718:0] peer: [6:7587241353902869435:0] socket: 160 qp: -1
2025-12-24T02:11:27.817100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241353580010748:2088] [node 6] ICS10 traffic start
2025-12-24T02:11:27.817107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:27.824685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:27.824790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241350476921710:2048] [node 1] ICS01 socket: 175 reason# EndOfStream
2025-12-24T02:11:27.824811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241350476921607:2048] [node 1] ICP30 unregister session Session# [3:7587241350476921710:2048] VirtualId# [3:7587241350476921704:0]
2025-12-24T02:11:27.824819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241350476921710:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.824854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.824877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.826832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:27.826911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241352543671412:2048] [node 1] ICS01 socket: 228 reason# EndOfStream
2025-12-24T02:11:27.826920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241352543671308:2048] [node 1] ICP30 unregister session Session# [2:7587241352543671412:2048] VirtualId# [2:7587241352543671410:0]
2025-12-24T02:11:27.826926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241352543671412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.826976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.827011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.828415Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:27.828495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241353580010746:2048] [node 1] ICS01 socket: 198 reason# EndOfStream
2025-12-24T02:11:27.828502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353580010645:2048] [node 1] ICP30 unregister session Session# [5:7587241353580010746:2048] VirtualId# [5:7587241353580010742:0]
2025-12-24T02:11:27.828508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241353580010746:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.828551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.828571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.828657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:27.828735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353902869436:2048] [node 1] ICS01 socket: 168 reason# EndOfStream
2025-12-24T02:11:27.828740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353902869336:2048] [node 1] ICP30 unregister session Session# [6:7587241353902869436:2048] VirtualId# [6:7587241353902869426:0]
2025-12-24T02:11:27.828745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353902869436:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.828771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.828793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.834763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:27.834871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241350508460518:2048] [node 1] ICS01 socket: 169 reason# EndOfStream
2025-12-24T02:11:27.834880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241350508460415:2048] [node 1] ICP30 unregister session Session# [4:7587241350508460518:2048] VirtualId# [4:7587241350508460508:0]
2025-12-24T02:11:27.834888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241350508460518:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.834946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:27.834983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:27.838805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:27.838894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241350476921693:2049] [node 2] ICS01 socket: 194 reason# EndOfStream
2025-12-24T02:11:27.838902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241350476921608:2049] [node 2] ICP30 unregister session Session# [3:7587241350476921693:2049] VirtualId# [3:7587241350476921690:0]
2025-12-24T02:11:27.838908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241350476921693:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.838956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.838980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.846947Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:27.847043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241353580010735:2049] [node 2] ICS01 socket: 206 reason# EndOfStream
2025-12-24T02:11:27.847052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353580010646:2049] [node 2] ICP30 unregister session Session# [5:7587241353580010735:2049] VirtualId# [5:7587241353580010730:0]
2025-12-24T02:11:27.847059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241353580010735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.847103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.847128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.847216Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:27.847288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353902869420:2049] [node 2] ICS01 socket: 193 reason# EndOfStream
2025-12-24T02:11:27.847293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353902869337:2049] [node 2] ICP30 unregister session Session# [6:7587241353902869420:2049] VirtualId# [6:7587241353902869418:0]
2025-12-24T02:11:27.847299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353902869420:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.847326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.847347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.849920Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:27.850016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241350508460516:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:11:27.850024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241350508460416:2049] [node 2] ICP30 unregister session Session# [4:7587241350508460516:2049] VirtualId# [4:7587241350508460510:0]
2025-12-24T02:11:27.850029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241350508460516:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.850086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:27.850116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:27.858963Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:27.859060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353902869433:2050] [node 3] ICS01 socket: 212 reason# EndOfStream
2025-12-24T02:11:27.859071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353902869338:2050] [node 3] ICP30 unregister session Session# [6:7587241353902869433:2050] VirtualId# [6:7587241353902869429:0]
2025-12-24T02:11:27.859079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353902869433:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.859119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:27.859143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:27.859232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:27.859303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241353580010727:2050] [node 3] ICS01 socket: 205 reason# EndOfStream
2025-12-24T02:11:27.859309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353580010647:2050] [node 3] ICP30 unregister session Session# [5:7587241353580010727:2050] VirtualId# [5:7587241353580010725:0]
2025-12-24T02:11:27.859314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241353580010727:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.859343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:27.859363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:27.878830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:27.878946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241353902869427:2063] [node 4] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:11:27.878957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241353902869357:2063] [node 4] ICP30 unregister session Session# [6:7587241353902869427:2063] VirtualId# [6:7587241353902869417:0]
2025-12-24T02:11:27.878965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241353902869427:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:27.879014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:27.879041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:27.918826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352198981795:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.918844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352198981796:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.918835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241352924829469:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:27.918848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241352924829470:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:27.933121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350504268592:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.933141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350504268593:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.933147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350504268594:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.934802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241350339900239:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.934812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241350339900240:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.934817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241350339900241:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.930849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241351853851595:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.930861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241351853851596:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.935788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353337874232:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:27.935799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353337874233:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:27.935804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353337874234:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:27.971142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353337874255:2061] [node 6] ICP01 ready to work
2025-12-24T02:11:27.971197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350504268627:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:27.972934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241350504268649:2068] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.974797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241352924829531:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:27.974908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241352924829530:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
2025-12-24T02:11:27.974967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241352924829532:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.974365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352198981862:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:27.975267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241352924829533:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
2025-12-24T02:11:27.974430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352198981858:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51298
2025-12-24T02:11:27.974442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352198981859:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51298
2025-12-24T02:11:27.974453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241352198981860:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51298
2025-12-24T02:11:27.978925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241353337874271:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:27.982723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241352924829534:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
2025-12-24T02:11:27.982957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829530:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.983372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241352924829469:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:27.989514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241350504268664:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
2025-12-24T02:11:27.989916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829534:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:27.991239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241351853851659:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:27.991341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241351853851660:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:27.991893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352198981867:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:27.992060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241352198981868:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:27.995295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241352924829542:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:27.995385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241352924829543:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:28.006920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352198981856:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.006951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352198981856:2107] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.006718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241350339900304:2069] [node 5] ICP01 ready to work
2025-12-24T02:11:28.006943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241354634867604:2075] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.012758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867606:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.012788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867607:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.014802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241352924829470:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24824169ns) {Configure:104(24823601ns) {ProcessConfigured:135(24496985ns) {StartInitialHandshake:157(24496516ns) {DropHandshakes:470(1164ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1583ns) {SwitchToState:179(331ns)}}}}}
2025-12-24T02:11:28.018805Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241351853851659:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27387391ns) {Configure:104(27386720ns) {ProcessConfigured:135(26990203ns) {StartInitialHandshake:157(26989833ns) {DropHandshakes:470(424ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1276ns) {SwitchToState:179(789ns)}}}}}
2025-12-24T02:11:28.020174Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241352198981867:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27983200ns) {Configure:104(27982795ns) {ProcessConfigured:135(27617360ns) {StartInitialHandshake:157(27617236ns) {DropHandshakes:470(330ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1350ns) {SwitchToState:179(781ns)}}}}}
2025-12-24T02:11:28.020952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352198981860:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.026004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241351853851658:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.029172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356493949174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51298
2025-12-24T02:11:28.029534Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352198981859:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.029575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352198981856:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.029656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352198981795:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.029671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.029679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352198981795:2048] [node 1] ICP22 created new session: [2:7587241356493949176:2048]
2025-12-24T02:11:28.029686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356493949176:2048] [node 1] ICS09 handshake done sender: [2:7587241352198981856:2107] self: [2:7587241352198981855:0] peer: [1:7587241352924829538:0] socket: 162 qp: -1
2025-12-24T02:11:28.029693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356493949176:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.029703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.025833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241353337874302:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:28.030867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353337874271:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.030907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353337874302:2102] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.034858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350504268642:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.034875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241350504268642:2093] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.035235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829533:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.035574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241357219796848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
2025-12-24T02:11:28.042938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350504268664:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.043028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350504268642:2093] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.043268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241350504268627:2064] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.043288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.043297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350504268627:2064] [node 5] ICP22 created new session: [4:7587241354799235969:2064]
2025-12-24T02:11:28.043305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241354799235969:2064] [node 5] ICS09 handshake done sender: [4:7587241350504268642:2093] self: [4:7587241350504268641:0] peer: [5:7587241357632841607:0] socket: 171 qp: -1
2025-12-24T02:11:28.043315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241354799235969:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:28.043326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.043639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350504268592:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.043076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241352924829530:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.045087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829536:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.045100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241352924829536:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.045434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241352924829469:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.045452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.045459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241352924829469:2048] [node 2] ICP22 created new session: [1:7587241357219796850:2048]
2025-12-24T02:11:28.045464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241357219796850:2048] [node 2] ICS09 handshake done sender: [1:7587241352924829530:2108] self: [1:7587241352924829538:0] peer: [2:7587241352198981855:0] socket: 177 qp: -1
2025-12-24T02:11:28.045470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241357219796850:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:28.045497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.046161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241352924829536:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.046227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241352924829531:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.046241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.046248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241352924829531:2107] [node 4] ICP22 created new session: [1:7587241357219796853:2107]
2025-12-24T02:11:28.046253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241357219796853:2107] [node 4] ICS09 handshake done sender: [1:7587241352924829536:2112] self: [1:7587241352924829535:0] peer: [4:7587241354799235971:0] socket: 181 qp: -1
2025-12-24T02:11:28.046258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241357219796853:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:28.046263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.050804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841608:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.050886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841609:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.047821Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241351853851658:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21694750ns) {Configure:104(21693769ns) {ProcessConfigured:135(21188955ns) {StartInitialHandshake:157(21188291ns) {DropHandshakes:470(1012ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1978ns) {SwitchToState:179(1005ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.055157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357632841609:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.055204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353337874234:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.055322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357632841608:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.055338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353337874232:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.056264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350504268592:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241350504268664:2109] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.058778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818970:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.058860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829545:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.058875Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241352924829545:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.062892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.063266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818958:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.062961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841613:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.064017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841614:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.063280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356148818958:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.064181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353337874271:2078] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.064261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353337874302:2102] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.064277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.064285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353337874302:2102] [node 4] ICP22 created new session: [5:7587241357632841615:2102]
2025-12-24T02:11:28.064293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357632841615:2102] [node 4] ICS09 handshake done sender: [5:7587241353337874271:2078] self: [5:7587241357632841607:0] peer: [4:7587241350504268641:0] socket: 175 qp: -1
2025-12-24T02:11:28.064301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357632841615:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:28.064326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.067347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235974:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
2025-12-24T02:11:28.067367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235975:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
2025-12-24T02:11:28.067402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
2025-12-24T02:11:28.067507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829532:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.066769Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241350339900241:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(59555425ns) {Configure:104(59554175ns) {ProcessConfigured:135(59145127ns) {StartInitialHandshake:157(59144075ns) {DropHandshakes:470(1308ns) {DropIncomingHandshake:442(231ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1497ns) {SwitchToState:179(891ns)}}}}}
2025-12-24T02:11:28.069079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241354634867607:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.069109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241350339900239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.070743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241352924829547:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.070750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241352924829547:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.070763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357632841612:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.070804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241353337874233:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.070962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352198981858:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.071307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241352198981866:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.071311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241352198981866:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.071055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356148818958:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.074741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818971:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.074800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241354634867606:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.074832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241350339900304:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.074791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241351853851659:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.074810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.074818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241351853851659:2108] [node 5] ICP22 created new session: [3:7587241356148818973:2108]
2025-12-24T02:11:28.074825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356148818973:2108] [node 5] ICS09 handshake done sender: [3:7587241356148818958:2110] self: [3:7587241351853851661:0] peer: [5:7587241357632841610:0] socket: 190 qp: -1
2025-12-24T02:11:28.074835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356148818973:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:28.074847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.075318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818970:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.075359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818976:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.079645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356493949174:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.081024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241357219796857:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21370
2025-12-24T02:11:28.081473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241352924829545:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.081589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241352924829547:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.081652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241352924829542:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.081670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.081675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241352924829542:2113] [node 5] ICP22 created new session: [1:7587241357219796860:2113]
2025-12-24T02:11:28.081680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241357219796860:2113] [node 5] ICS09 handshake done sender: [1:7587241352924829545:2115] self: [1:7587241352924829544:0] peer: [5:7587241357632841611:0] socket: 186 qp: -1
2025-12-24T02:11:28.081685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241357219796860:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:28.081691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.081896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241352924829543:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.081906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.081909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241352924829543:2114] [node 6] ICP22 created new session: [1:7587241357219796862:2114]
2025-12-24T02:11:28.081913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241357219796862:2114] [node 6] ICS09 handshake done sender: [1:7587241352924829547:2116] self: [1:7587241352924829546:0] peer: [6:7587241354634867615:0] socket: 187 qp: -1
2025-12-24T02:11:28.081916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241357219796862:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:28.081920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.078908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241353337874263:2074] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.078930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241353337874263:2074] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.079590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357632841609:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.079858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241353337874263:2074] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.079919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353337874234:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.079936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.079944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353337874234:2050] [node 3] ICP22 created new session: [5:7587241357632841620:2050]
2025-12-24T02:11:28.079951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357632841620:2050] [node 3] ICS09 handshake done sender: [5:7587241357632841609:2117] self: [5:7587241357632841610:0] peer: [3:7587241351853851661:0] socket: 207 qp: -1
2025-12-24T02:11:28.079959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357632841620:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.079966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.080334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241353337874255:2061] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.080340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.080344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353337874255:2061] [node 6] ICP22 created new session: [5:7587241357632841622:2061]
2025-12-24T02:11:28.080349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357632841622:2061] [node 6] ICS09 handshake done sender: [5:7587241353337874263:2074] self: [5:7587241353337874262:0] peer: [6:7587241354634867616:0] socket: 168 qp: -1
2025-12-24T02:11:28.080353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357632841622:2061] [node 6] ICS10 traffic start
2025-12-24T02:11:28.080358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.080429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867617:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.080483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867618:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.080517Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867619:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.080528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867620:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.082798Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241354634867618:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.083393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241350339900239:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241354634867607:2112] held: no
2025-12-24T02:11:28.083521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241354634867606:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.082975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818976:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.083271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818971:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.083339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241351853851659:2108] [node 5] ICP18 handshake (actor [3:7587241356148818976:2117]) is too old
2025-12-24T02:11:28.083353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241351853851595:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.084565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241354799235975:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.084595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350504268664:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.084570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241357219796847:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.084579Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241357219796847:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.083672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241354634867617:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.083686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241350339900304:2069] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.083716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.083722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241350339900304:2069] [node 5] ICP22 created new session: [6:7587241354634867622:2069]
2025-12-24T02:11:28.083727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241354634867622:2069] [node 5] ICS09 handshake done sender: [6:7587241354634867606:2111] self: [6:7587241354634867616:0] peer: [5:7587241353337874262:0] socket: 184 qp: -1
2025-12-24T02:11:28.083732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241354634867622:2069] [node 5] ICS10 traffic start
2025-12-24T02:11:28.083751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.084062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241350339900240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.084256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867621:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.089934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356493949166:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.089954Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356493949166:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.090747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241357219796857:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.090981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241350339900241:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.091014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241354634867607:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.091180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241350339900239:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.091192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.091198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241350339900239:2048] [node 1] ICP22 created new session: [6:7587241354634867627:2048]
2025-12-24T02:11:28.091203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241354634867627:2048] [node 1] ICS09 handshake done sender: [6:7587241354634867607:2112] self: [6:7587241354634867615:0] peer: [1:7587241352924829546:0] socket: 188 qp: -1
2025-12-24T02:11:28.091209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241354634867627:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.091215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.099193Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241353337874232:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241357632841608:2116] held: no
2025-12-24T02:11:28.099440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.100022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241357219796847:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.100112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241352924829470:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.100133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.100139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241352924829470:2049] [node 3] ICP22 created new session: [1:7587241357219796866:2049]
2025-12-24T02:11:28.100146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241357219796866:2049] [node 3] ICS09 handshake done sender: [1:7587241357219796847:2117] self: [1:7587241352924829539:0] peer: [3:7587241356148818978:0] socket: 180 qp: -1
2025-12-24T02:11:28.100152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241357219796866:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.100159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.100732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357632841608:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.100792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353337874232:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.100803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.100808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353337874232:2048] [node 1] ICP22 created new session: [5:7587241357632841625:2048]
2025-12-24T02:11:28.100813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357632841625:2048] [node 1] ICS09 handshake done sender: [5:7587241357632841608:2116] self: [5:7587241357632841611:0] peer: [1:7587241352924829544:0] socket: 199 qp: -1
2025-12-24T02:11:28.100818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357632841625:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.100823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.102773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350504268592:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.102796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.102805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350504268592:2048] [node 1] ICP22 created new session: [4:7587241354799235977:2048]
2025-12-24T02:11:28.102813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241354799235977:2048] [node 1] ICS09 handshake done sender: [4:7587241350504268664:2109] self: [4:7587241354799235971:0] peer: [1:7587241352924829535:0] socket: 182 qp: -1
2025-12-24T02:11:28.102820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241354799235977:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.102830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.104128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241352198981866:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.104266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241354799235974:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.104289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350504268593:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.104490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356493949168:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.104496Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356493949168:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.105131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356493949166:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.105230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356493949168:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.105293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352198981862:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.105308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.105314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352198981862:2111] [node 5] ICP22 created new session: [2:7587241356493949185:2111]
2025-12-24T02:11:28.105320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356493949185:2111] [node 5] ICS09 handshake done sender: [2:7587241352198981866:2114] self: [2:7587241352198981863:0] peer: [5:7587241357632841618:0] socket: 194 qp: -1
2025-12-24T02:11:28.105325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356493949185:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:28.105333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.105545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352198981867:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.105553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.105557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352198981867:2112] [node 6] ICP22 created new session: [2:7587241356493949187:2112]
2025-12-24T02:11:28.105561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356493949187:2112] [node 6] ICS09 handshake done sender: [2:7587241356493949166:2115] self: [2:7587241352198981869:0] peer: [6:7587241354634867624:0] socket: 195 qp: -1
2025-12-24T02:11:28.105564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356493949187:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:28.105568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.105733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352198981868:2113] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.105761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.105765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352198981868:2113] [node 4] ICP22 created new session: [2:7587241356493949189:2113]
2025-12-24T02:11:28.105769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356493949189:2113] [node 4] ICS09 handshake done sender: [2:7587241356493949168:2116] self: [2:7587241356493949167:0] peer: [4:7587241354799235979:0] socket: 202 qp: -1
2025-12-24T02:11:28.105772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356493949189:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:28.105777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.106780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241354634867619:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.110737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818960:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.110763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356148818960:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.114855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241351853851595:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241356148818971:2116] held: no
2025-12-24T02:11:28.115021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818979:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.115036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.115056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818980:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.115184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356148818971:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.115310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356148818960:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.116393Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350504268593:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241354799235975:2118] held: no
2025-12-24T02:11:28.115400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241351853851660:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.115415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.115424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241351853851660:2109] [node 6] ICP22 created new session: [3:7587241356148818984:2109]
2025-12-24T02:11:28.115430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356148818984:2109] [node 6] ICS09 handshake done sender: [3:7587241356148818960:2111] self: [3:7587241356148818959:0] peer: [6:7587241354634867625:0] socket: 191 qp: -1
2025-12-24T02:11:28.115436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356148818984:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:28.115458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.116987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241351853851595:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.117001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.117007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241351853851595:2048] [node 1] ICP22 created new session: [3:7587241356148818986:2048]
2025-12-24T02:11:28.117012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356148818986:2048] [node 1] ICS09 handshake done sender: [3:7587241356148818971:2116] self: [3:7587241356148818978:0] peer: [1:7587241352924829539:0] socket: 210 qp: -1
2025-12-24T02:11:28.117018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356148818986:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.117025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.119269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818979:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.119298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241351853851660:2109] [node 6] ICP18 handshake (actor [3:7587241356148818979:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.118974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241350339900240:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241354634867618:2118] held: no
2025-12-24T02:11:28.119204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867629:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.122822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235981:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.122967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867630:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.123150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241354634867604:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.123635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241350339900241:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241354634867617:2117] held: no
2025-12-24T02:11:28.123796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241354634867618:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.123837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241354634867617:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.123880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241350339900240:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.123892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.123896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241350339900240:2049] [node 2] ICP22 created new session: [6:7587241354634867632:2049]
2025-12-24T02:11:28.123902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241354634867632:2049] [node 2] ICS09 handshake done sender: [6:7587241354634867618:2118] self: [6:7587241354634867624:0] peer: [2:7587241352198981869:0] socket: 220 qp: -1
2025-12-24T02:11:28.123908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241354634867632:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.123914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.124098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241350339900241:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.124103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.124108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241350339900241:2050] [node 3] ICP22 created new session: [6:7587241354634867634:2050]
2025-12-24T02:11:28.124111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241354634867634:2050] [node 3] ICS09 handshake done sender: [6:7587241354634867617:2117] self: [6:7587241354634867625:0] peer: [3:7587241356148818959:0] socket: 200 qp: -1
2025-12-24T02:11:28.124115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241354634867634:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.124119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.128371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818969:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.128392Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356148818969:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.129935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356148818969:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.130011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241351853851658:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.130026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.130031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241351853851658:2107] [node 4] ICP22 created new session: [3:7587241356148818989:2107]
2025-12-24T02:11:28.130035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356148818989:2107] [node 4] ICS09 handshake done sender: [3:7587241356148818969:2114] self: [3:7587241356148818968:0] peer: [4:7587241354799235984:0] socket: 230 qp: -1
2025-12-24T02:11:28.130039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356148818989:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:28.130044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.127262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235982:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
2025-12-24T02:11:28.127809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241350504268652:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.127813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241350504268652:2097] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.128159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241354799235982:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.128260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241350504268594:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.129235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241350504268652:2097] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.129342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241350504268649:2068] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.129353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.129358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350504268649:2068] [node 6] ICP22 created new session: [4:7587241354799235986:2068]
2025-12-24T02:11:28.129363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241354799235986:2068] [node 6] ICS09 handshake done sender: [4:7587241350504268652:2097] self: [4:7587241350504268651:0] peer: [6:7587241354634867631:0] socket: 169 qp: -1
2025-12-24T02:11:28.129367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241354799235986:2068] [node 6] ICS10 traffic start
2025-12-24T02:11:28.129374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.130570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241354799235985:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44038
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.130931Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241353337874233:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241357632841612:2118] held: no
2025-12-24T02:11:28.130974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357632841627:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11015
2025-12-24T02:11:28.131092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357632841612:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.131175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241353337874233:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.131185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.131197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241353337874233:2049] [node 2] ICP22 created new session: [5:7587241357632841628:2049]
2025-12-24T02:11:28.131202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357632841628:2049] [node 2] ICS09 handshake done sender: [5:7587241357632841612:2118] self: [5:7587241357632841618:0] peer: [2:7587241352198981863:0] socket: 208 qp: -1
2025-12-24T02:11:28.131207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357632841628:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.131213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.131597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356148818981:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.131624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241351853851596:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.130832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241354799235975:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.134747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356493949170:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.134759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356493949170:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.135511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356493949170:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.135404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356148818992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15949
2025-12-24T02:11:28.135667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356148818981:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.135732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241351853851596:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.135742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.135748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241351853851596:2049] [node 2] ICP22 created new session: [3:7587241356148818993:2049]
2025-12-24T02:11:28.135753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356148818993:2049] [node 2] ICS09 handshake done sender: [3:7587241356148818981:2120] self: [3:7587241356148818991:0] peer: [2:7587241356493949169:0] socket: 173 qp: -1
2025-12-24T02:11:28.135758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356148818993:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.135764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.136782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241352198981796:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.136799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.136804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241352198981796:2049] [node 3] ICP22 created new session: [2:7587241356493949193:2049]
2025-12-24T02:11:28.136809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356493949193:2049] [node 3] ICS09 handshake done sender: [2:7587241356493949170:2117] self: [2:7587241356493949169:0] peer: [3:7587241356148818991:0] socket: 172 qp: -1
2025-12-24T02:11:28.136813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356493949193:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.136819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.143708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350504268593:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.143728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.143737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350504268593:2049] [node 2] ICP22 created new session: [4:7587241354799235988:2049]
2025-12-24T02:11:28.143743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241354799235988:2049] [node 2] ICS09 handshake done sender: [4:7587241354799235975:2118] self: [4:7587241354799235979:0] peer: [2:7587241356493949167:0] socket: 211 qp: -1
2025-12-24T02:11:28.143751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241354799235988:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.143759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.144876Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350504268594:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241354799235982:2121] held: no
2025-12-24T02:11:28.145054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241354799235982:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.145107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241350504268594:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.145116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.145121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241350504268594:2050] [node 3] ICP22 created new session: [4:7587241354799235990:2050]
2025-12-24T02:11:28.145126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241354799235990:2050] [node 3] ICS09 handshake done sender: [4:7587241354799235982:2121] self: [4:7587241354799235984:0] peer: [3:7587241356148818968:0] socket: 179 qp: -1
2025-12-24T02:11:28.145131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241354799235990:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.145138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.147013Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241354634867604:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241354634867619:2119] held: no
2025-12-24T02:11:28.147245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241354634867636:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28401
2025-12-24T02:11:28.147398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241354634867619:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.147457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241354634867604:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.147467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.147471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241354634867604:2075] [node 4] ICP22 created new session: [6:7587241354634867637:2075]
2025-12-24T02:11:28.147477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241354634867637:2075] [node 4] ICS09 handshake done sender: [6:7587241354634867619:2119] self: [6:7587241354634867631:0] peer: [4:7587241350504268651:0] socket: 221 qp: -1
2025-12-24T02:11:28.147482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241354634867637:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:28.147489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.155896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:28.155992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241356493949176:2048] [node 1] ICS01 socket: 162 reason# EndOfStream
2025-12-24T02:11:28.156000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241352198981795:2048] [node 1] ICP30 unregister session Session# [2:7587241356493949176:2048] VirtualId# [2:7587241352198981855:0]
2025-12-24T02:11:28.156007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241356493949176:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.156038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.156059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.156453Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:28.156524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241356148818986:2048] [node 1] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:11:28.156531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241351853851595:2048] [node 1] ICP30 unregister session Session# [3:7587241356148818986:2048] VirtualId# [3:7587241356148818978:0]
2025-12-24T02:11:28.156535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241356148818986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.156558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.156575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.162892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:28.162990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241354799235977:2048] [node 1] ICS07 socket disconnect 182 reason# EndOfStream
2025-12-24T02:11:28.162997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241354799235977:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.163052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.163073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241354799235977:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:28.163298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:28.163371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357632841625:2048] [node 1] ICS01 socket: 199 reason# EndOfStream
2025-12-24T02:11:28.163378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353337874232:2048] [node 1] ICP30 unregister session Session# [5:7587241357632841625:2048] VirtualId# [5:7587241357632841611:0]
2025-12-24T02:11:28.163383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357632841625:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.163420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.163444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.163477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:28.163560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241354634867627:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-12-24T02:11:28.163565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241350339900239:2048] [node 1] ICP30 unregister session Session# [6:7587241354634867627:2048] VirtualId# [6:7587241354634867615:0]
2025-12-24T02:11:28.163569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241354634867627:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.163593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.163613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.164144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241350504268592:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:28.164156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241354799235977:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:28.164160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241350504268592:2048] [node 1] ICP30 unregister session Session# [4:7587241354799235977:2048] VirtualId# [4:7587241354799235971:0]
2025-12-24T02:11:28.164174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.164181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241350504268592:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:28.164184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31315) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:28.174823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241350504268592:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:28.179058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:28.179160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241356148818993:2049] [node 2] ICS01 socket: 173 reason# EndOfStream
2025-12-24T02:11:28.179169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241351853851596:2049] [node 2] ICP30 unregister session Session# [3:7587241356148818993:2049] VirtualId# [3:7587241356148818991:0]
2025-12-24T02:11:28.179176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241356148818993:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.179215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.179238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.189541Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:28.189634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357632841628:2049] [node 2] ICS01 socket: 208 reason# EndOfStream
2025-12-24T02:11:28.189644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353337874233:2049] [node 2] ICP30 unregister session Session# [5:7587241357632841628:2049] VirtualId# [5:7587241357632841618:0]
2025-12-24T02:11:28.189652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357632841628:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.189705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.189733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.190822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:28.190931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241354634867632:2049] [node 2] ICS01 socket: 220 reason# EndOfStream
2025-12-24T02:11:28.190941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241350339900240:2049] [node 2] ICP30 unregister session Session# [6:7587241354634867632:2049] VirtualId# [6:7587241354634867624:0]
2025-12-24T02:11:28.190950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241354634867632:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.190997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.191024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.191070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:28.191140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241354799235988:2049] [node 2] ICS01 socket: 211 reason# EndOfStream
2025-12-24T02:11:28.191145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241350504268593:2049] [node 2] ICP30 unregister session Session# [4:7587241354799235988:2049] VirtualId# [4:7587241354799235979:0]
2025-12-24T02:11:28.191151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241354799235988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.191177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.191197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.210756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:28.210876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241354634867634:2050] [node 3] ICS01 socket: 200 reason# EndOfStream
2025-12-24T02:11:28.210889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241350339900241:2050] [node 3] ICP30 unregister session Session# [6:7587241354634867634:2050] VirtualId# [6:7587241354634867625:0]
2025-12-24T02:11:28.210897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241354634867634:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.210951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:28.210978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:28.215400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:28.215507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357632841620:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:11:28.215515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241353337874234:2050] [node 3] ICP30 unregister session Session# [5:7587241357632841620:2050] VirtualId# [5:7587241357632841610:0]
2025-12-24T02:11:28.215523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357632841620:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.215591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:28.215621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:28.230833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:28.230971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241354634867637:2075] [node 4] ICS01 socket: 221 reason# EndOfStream
2025-12-24T02:11:28.230984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241354634867604:2075] [node 4] ICP30 unregister session Session# [6:7587241354634867637:2075] VirtualId# [6:7587241354634867631:0]
2025-12-24T02:11:28.230993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241354634867637:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.231060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:28.231089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:28.278906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241354870191267:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:28.278915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241354870191268:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:28.278777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241354953606757:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.278793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241354953606758:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:28.283126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241356034958563:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.283134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241356034958564:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.283139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241356034958565:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:28.278498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241355047496357:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.278507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241355047496358:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.290869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241357220534790:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.290886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241357220534791:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.290891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241357220534792:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.301375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496367:2057] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.301710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241356034958581:2062] [node 5] ICP01 ready to work
2025-12-24T02:11:28.302493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241354870191283:2062] [node 4] ICP01 ready to work
2025-12-24T02:11:28.303097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241354870191295:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:28.303110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241355047496370:2059] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.311604Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606791:2075] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
2025-12-24T02:11:28.314356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241354953606777:2063] [node 6] ICP01 ready to work
2025-12-24T02:11:28.314445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241354953606776:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:28.314794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241354870191297:2067] [node 6] ICP01 ready to work
2025-12-24T02:11:28.322891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241356034958612:2070] [node 6] ICP01 ready to work
2025-12-24T02:11:28.322908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241354953606818:2077] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.326437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241355047496402:2083] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.330685Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241354953606776:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16137472ns) {Configure:104(16137016ns) {ProcessConfigured:135(15862113ns) {StartInitialHandshake:157(15861569ns) {DropHandshakes:470(395ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(36ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(838ns) {SwitchToState:179(277ns)}}}}}
2025-12-24T02:11:28.330981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606824:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.331122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241355047496370:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27948920ns) {Configure:104(27948133ns) {ProcessConfigured:135(27506897ns) {StartInitialHandshake:157(27506472ns) {DropHandshakes:470(631ns) {DropIncomingHandshake:442(95ns) DropOutgoingHandshake:457(123ns)} GenerateSessionVirtualId:988(118ns) PrepareNewSessionHandshake:477(619ns) {SwitchToState:179(386ns)}}}}}
2025-12-24T02:11:28.331149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496399:2086] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.331628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496397:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.335428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958624:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.335475Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958625:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.335166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241354870191340:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-12-24T02:11:28.342986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241355047496387:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:28.346137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496398:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.342852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241354870191336:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-12-24T02:11:28.343198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241354870191337:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50029
2025-12-24T02:11:28.323859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241355530472992:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.323900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241355530472993:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:28.324046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241355530473027:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:28.324159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241355530473028:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:28.342729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241355530472991:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.342736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241357220534811:2059] [node 6] ICP01 ready to work
2025-12-24T02:11:28.350802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473050:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.354803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241356034958612:2070] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31777104ns) {Configure:104(31775815ns) {ProcessConfigured:135(31403368ns) {StartInitialHandshake:157(31402651ns) {DropHandshakes:470(600ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(186ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:477(898ns) {SwitchToState:179(504ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.362952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534832:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.363776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241357220534864:2074] [node 4] ICP01 ready to work
2025-12-24T02:11:28.363863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534859:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.363926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534860:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.363938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.364706Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241356034958564:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(58193308ns) {Configure:104(58192701ns) {ProcessConfigured:135(57864305ns) {StartInitialHandshake:157(57864161ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(685ns) {SwitchToState:179(240ns)}}}}}
2025-12-24T02:11:28.364756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958624:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.364930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241356034958563:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.366791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241357220534790:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48144531ns) {Configure:104(48143309ns) {ProcessConfigured:135(47821176ns) {StartInitialHandshake:157(47820847ns) {DropHandshakes:470(388ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(858ns) {SwitchToState:179(404ns)}}}}}
2025-12-24T02:11:28.366841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191298:2073] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.366855Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241354870191298:2073] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.367387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534859:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.367417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241357220534790:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.368963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241354953606777:2063] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(54506624ns) {Configure:104(54505668ns) {ProcessConfigured:135(54176956ns) {StartInitialHandshake:157(54176681ns) {DropHandshakes:470(378ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(1083ns) {SwitchToState:179(775ns)}}}}}
2025-12-24T02:11:28.370780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191337:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.371104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496397:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.371136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496398:2085] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.371113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958625:2099] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.371147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241356034958565:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.376466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958645:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.375167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496409:2092] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.375180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241355047496409:2092] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.375520Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496367:2057] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.375574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241355047496358:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.375588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241355047496357:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.379654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473070:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.378867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241355047496409:2092] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.379112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241355047496402:2083] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.379128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.379134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241355047496402:2083] [node 4] ICP22 created new session: [3:7587241355047496444:2083]
2025-12-24T02:11:28.379140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241355047496444:2083] [node 4] ICS09 handshake done sender: [3:7587241355047496409:2092] self: [3:7587241355047496408:0] peer: [4:7587241356034958644:0] socket: 195 qp: -1
2025-12-24T02:11:28.379147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241355047496444:2083] [node 4] ICS10 traffic start
2025-12-24T02:11:28.379156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.383035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191340:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.386774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241354870191298:2073] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.390714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191305:2078] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.390727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191303:2077] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.390734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241354870191305:2078] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.390742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241354870191303:2077] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.390791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606784:2071] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.390810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241354953606784:2071] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.396983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241355530473050:2097] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.394898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241354953606818:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(71837943ns) {Configure:104(71837145ns) {ProcessConfigured:135(71342493ns) {StartInitialHandshake:157(71341796ns) {DropHandshakes:470(440ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1139ns) {SwitchToState:179(587ns)}}}}}
2025-12-24T02:11:28.395585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496447:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.406996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241356034958625:2099] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.403759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241354870191283:2062] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.403786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.403796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241354870191283:2062] [node 4] ICP22 created new session: [1:7587241354870191352:2062]
2025-12-24T02:11:28.403804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241354870191352:2062] [node 4] ICS09 handshake done sender: [1:7587241354870191298:2073] self: [1:7587241354870191293:0] peer: [4:7587241356034958643:0] socket: 170 qp: -1
2025-12-24T02:11:28.403813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241354870191352:2062] [node 4] ICS10 traffic start
2025-12-24T02:11:28.403825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.407570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241354870191305:2078] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.407700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241354870191268:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.407721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.407734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241354870191268:2049] [node 3] ICP22 created new session: [1:7587241354870191356:2049]
2025-12-24T02:11:28.407742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241354870191356:2049] [node 3] ICS09 handshake done sender: [1:7587241354870191305:2078] self: [1:7587241354870191304:0] peer: [3:7587241355047496440:0] socket: 171 qp: -1
2025-12-24T02:11:28.407749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241354870191356:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.407759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.406772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534861:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.409141Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241355047496357:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241355047496398:2085] held: no
2025-12-24T02:11:28.409326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.409498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241355047496398:2085] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.409546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241355047496357:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.409556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.409561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241355047496357:2048] [node 1] ICP22 created new session: [3:7587241355047496450:2048]
2025-12-24T02:11:28.409566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241355047496450:2048] [node 1] ICS09 handshake done sender: [3:7587241355047496398:2085] self: [3:7587241355047496440:0] peer: [1:7587241354870191304:0] socket: 189 qp: -1
2025-12-24T02:11:28.409571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241355047496450:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.409576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.411466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606824:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.411889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473072:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.415297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241355047496402:2083] [node 4] ICP35 event processing took too much time PendingConnection:279(36073901ns) {HandleHandshakeStatus:329(36072805ns) {DropIncomingHandshake:442(2867ns) DropOutgoingHandshake:457(240ns) SwitchToState:179(86ns) ProcessPendingSessionEvents:495(24334857ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24315746ns max# 24246685ns}}}
2025-12-24T02:11:28.415490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496399:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.414758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241356034958565:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241356034958625:2099] held: no
2025-12-24T02:11:28.414871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241356034958565:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.414883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.414890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241356034958565:2050] [node 3] ICP22 created new session: [4:7587241356034958647:2050]
2025-12-24T02:11:28.414896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241356034958647:2050] [node 3] ICS09 handshake done sender: [4:7587241356034958625:2099] self: [4:7587241356034958644:0] peer: [3:7587241355047496408:0] socket: 199 qp: -1
2025-12-24T02:11:28.414904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241356034958647:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.414912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.415023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958646:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.415023Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241354870191303:2077] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.415186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191336:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.415210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241354870191283:2062] [node 4] ICP18 handshake (actor [1:7587241354870191336:2105]) is too old
2025-12-24T02:11:28.418976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241354953606784:2071] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.419085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241354953606758:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.419099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.419107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241354953606758:2049] [node 3] ICP22 created new session: [2:7587241354953606840:2049]
2025-12-24T02:11:28.419113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241354953606840:2049] [node 3] ICS09 handshake done sender: [2:7587241354953606784:2071] self: [2:7587241354953606783:0] peer: [3:7587241355047496439:0] socket: 169 qp: -1
2025-12-24T02:11:28.419119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241354953606840:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.419127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.419490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473074:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.419518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.419254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606842:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
2025-12-24T02:11:28.419302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606843:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
2025-12-24T02:11:28.418632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496407:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.418654Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241355047496407:2091] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.422848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241354870191295:2068] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.422864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.422872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241354870191295:2068] [node 5] ICP22 created new session: [1:7587241354870191360:2068]
2025-12-24T02:11:28.422878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241354870191360:2068] [node 5] ICS09 handshake done sender: [1:7587241354870191303:2077] self: [1:7587241354870191296:0] peer: [5:7587241357220534872:0] socket: 172 qp: -1
2025-12-24T02:11:28.422884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241354870191360:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:28.422891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.425671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534832:2085] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.425707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241357220534791:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.425786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241357220534864:2074] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.425807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534873:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.425820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534874:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.422899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241356034958563:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241356034958624:2098] held: no
2025-12-24T02:11:28.425027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241356034958624:2098] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.426206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958583:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.426210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241356034958583:2063] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.426524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241356034958563:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.426535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.426540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241356034958563:2048] [node 1] ICP22 created new session: [4:7587241356034958650:2048]
2025-12-24T02:11:28.426546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241356034958650:2048] [node 1] ICS09 handshake done sender: [4:7587241356034958624:2098] self: [4:7587241356034958643:0] peer: [1:7587241354870191293:0] socket: 184 qp: -1
2025-12-24T02:11:28.426552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241356034958650:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.426560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.426827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357220534859:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.426920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241355530473073:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.426960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241355530473028:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.427199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241356034958583:2063] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.426988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534860:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.427000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241357220534790:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.427011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.427018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241357220534790:2048] [node 1] ICP22 created new session: [5:7587241357220534877:2048]
2025-12-24T02:11:28.427023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357220534877:2048] [node 1] ICS09 handshake done sender: [5:7587241357220534859:2112] self: [5:7587241357220534872:0] peer: [1:7587241354870191296:0] socket: 201 qp: -1
2025-12-24T02:11:28.427027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357220534877:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.427045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.427324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958641:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.427331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241356034958641:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.427745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241356034958581:2062] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.427754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.427759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241356034958581:2062] [node 5] ICP22 created new session: [4:7587241356034958653:2062]
2025-12-24T02:11:28.427765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241356034958653:2062] [node 5] ICS09 handshake done sender: [4:7587241356034958583:2063] self: [4:7587241356034958582:0] peer: [5:7587241357220534876:0] socket: 166 qp: -1
2025-12-24T02:11:28.427771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241356034958653:2062] [node 5] ICS10 traffic start
2025-12-24T02:11:28.427776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.428512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241356034958641:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.428595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241356034958612:2070] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.428606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.428611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241356034958612:2070] [node 6] ICP22 created new session: [4:7587241356034958657:2070]
2025-12-24T02:11:28.428616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241356034958657:2070] [node 6] ICS09 handshake done sender: [4:7587241356034958641:2115] self: [4:7587241356034958621:0] peer: [6:7587241355530473075:0] socket: 219 qp: -1
2025-12-24T02:11:28.428620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241356034958657:2070] [node 6] ICS10 traffic start
2025-12-24T02:11:28.428627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.430758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241355530472993:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.430785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241355530473070:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.430795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241355530472993:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(33730347ns) {IncomingHandshake:261(33729636ns) {DropIncomingHandshake:442(182ns) PrepareNewSessionHandshake:477(619ns) {SwitchToState:179(426ns)} GenerateSessionVirtualId:988(340ns) IssueIncomingHandshakeReply:193(33720481ns) {SwitchToState:179(277ns)}}}
2025-12-24T02:11:28.430843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241355530473072:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.430867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241355530473027:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.430879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241355530472991:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.430763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606823:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.430774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241354953606823:2103] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.431073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534863:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.431081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241357220534863:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.431050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191324:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.431060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241354870191324:2093] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.431618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241354870191324:2093] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.431693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241354870191297:2067] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.431705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.431710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241354870191297:2067] [node 6] ICP22 created new session: [1:7587241354870191363:2067]
2025-12-24T02:11:28.431715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241354870191363:2067] [node 6] ICS09 handshake done sender: [1:7587241354870191324:2093] self: [1:7587241354870191323:0] peer: [6:7587241355530473077:0] socket: 198 qp: -1
2025-12-24T02:11:28.431720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241354870191363:2067] [node 6] ICS10 traffic start
2025-12-24T02:11:28.431725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.435022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606843:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.435203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606842:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.435251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
2025-12-24T02:11:28.435265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241354953606757:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.434917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241355047496454:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53053
2025-12-24T02:11:28.438856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241355047496407:2091] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.438867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241355530473074:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.438904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241355530472992:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.439055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958659:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.439687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241354953606758:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20485343ns) {HandleHandshakeStatus:329(20484818ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(101ns) ProcessPendingSessionEvents:495(8817216ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8783573ns max# 8695047ns}}}
2025-12-24T02:11:28.439735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241354953606758:2049] [node 3] ICP18 handshake (actor [2:7587241354953606842:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.441600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241355047496397:2084] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.441817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606836:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.441826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241354953606836:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.442087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241354953606823:2103] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.442405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606791:2075] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.442661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606845:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.442733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241354953606836:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.440068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473080:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.441645Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473081:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.442735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473079:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.442946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357220534863:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.442818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241354953606776:2059] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.442836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.442842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241354953606776:2059] [node 5] ICP22 created new session: [2:7587241354953606850:2059]
2025-12-24T02:11:28.442848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241354953606850:2059] [node 5] ICS09 handshake done sender: [2:7587241354953606823:2103] self: [2:7587241354953606803:0] peer: [5:7587241357220534875:0] socket: 192 qp: -1
2025-12-24T02:11:28.442854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241354953606850:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:28.442861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.443097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241354953606777:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.443108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.443114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241354953606777:2063] [node 6] ICP22 created new session: [2:7587241354953606852:2063]
2025-12-24T02:11:28.443118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241354953606852:2063] [node 6] ICS09 handshake done sender: [2:7587241354953606836:2115] self: [2:7587241354953606802:0] peer: [6:7587241355530473078:0] socket: 222 qp: -1
2025-12-24T02:11:28.443122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241354953606852:2063] [node 6] ICS10 traffic start
2025-12-24T02:11:28.443127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.443064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241357220534873:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.443089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241357220534811:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.443101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.443106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241357220534811:2059] [node 6] ICP22 created new session: [5:7587241357220534882:2059]
2025-12-24T02:11:28.443112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357220534882:2059] [node 6] ICS09 handshake done sender: [5:7587241357220534863:2115] self: [5:7587241357220534862:0] peer: [6:7587241355530473076:0] socket: 206 qp: -1
2025-12-24T02:11:28.443116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357220534882:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:28.443152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.443627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241357220534792:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.443802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241354870191322:2092] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.443813Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241354870191322:2092] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.445581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958660:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.442789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496447:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.444618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241355047496442:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.444623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241355047496442:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.445004Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241355047496358:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241355047496397:2084] held: no
2025-12-24T02:11:28.446798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241355047496358:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.446809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.446816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241355047496358:2049] [node 2] ICP22 created new session: [3:7587241355047496457:2049]
2025-12-24T02:11:28.446822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241355047496457:2049] [node 2] ICS09 handshake done sender: [3:7587241355047496397:2084] self: [3:7587241355047496439:0] peer: [2:7587241354953606783:0] socket: 182 qp: -1
2025-12-24T02:11:28.446829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241355047496457:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.446829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241355047496442:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.446836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.447067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241355047496370:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.447084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.447088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241355047496370:2059] [node 6] ICP22 created new session: [3:7587241355047496459:2059]
2025-12-24T02:11:28.447093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241355047496459:2059] [node 6] ICS09 handshake done sender: [3:7587241355047496407:2091] self: [3:7587241355047496371:0] peer: [6:7587241355530473071:0] socket: 193 qp: -1
2025-12-24T02:11:28.447096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241355047496459:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:28.447102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.447544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241355047496387:2064] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.447566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.447571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241355047496387:2064] [node 5] ICP22 created new session: [3:7587241355047496461:2064]
2025-12-24T02:11:28.447577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241355047496461:2064] [node 5] ICS09 handshake done sender: [3:7587241355047496442:2118] self: [3:7587241355047496441:0] peer: [5:7587241357220534884:0] socket: 194 qp: -1
2025-12-24T02:11:28.447582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241355047496461:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:28.447593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.446966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958660:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.447004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241356034958564:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.447472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241354953606837:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.447480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241354953606837:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.448087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241354953606837:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.448151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241354953606818:2077] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.448161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.448168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241354953606818:2077] [node 4] ICP22 created new session: [2:7587241354953606857:2077]
2025-12-24T02:11:28.448172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241354953606857:2077] [node 4] ICS09 handshake done sender: [2:7587241354953606837:2116] self: [2:7587241354953606819:0] peer: [4:7587241356034958661:0] socket: 191 qp: -1
2025-12-24T02:11:28.448176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241354953606857:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:28.448183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.450948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534885:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.450958Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241356034958564:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241356034958660:2120] held: no
2025-12-24T02:11:28.451178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241356034958659:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.451200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241356034958581:2062] [node 5] ICP18 handshake (actor [4:7587241356034958659:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.454076Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241354870191322:2092] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.454284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241354953606859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10806
2025-12-24T02:11:28.454447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241354953606843:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.454516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241354953606757:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.454528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.454535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241354953606757:2048] [node 1] ICP22 created new session: [2:7587241354953606860:2048]
2025-12-24T02:11:28.454541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241354953606860:2048] [node 1] ICS09 handshake done sender: [2:7587241354953606843:2118] self: [2:7587241354953606846:0] peer: [1:7587241354870191299:0] socket: 221 qp: -1
2025-12-24T02:11:28.454547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241354953606860:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.454555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.454861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241354870191267:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.454879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.454885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241354870191267:2048] [node 2] ICP22 created new session: [1:7587241354870191366:2048]
2025-12-24T02:11:28.454891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241354870191366:2048] [node 2] ICS09 handshake done sender: [1:7587241354870191322:2092] self: [1:7587241354870191299:0] peer: [2:7587241354953606846:0] socket: 164 qp: -1
2025-12-24T02:11:28.454896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241354870191366:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:28.454904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.460381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241355530472992:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241355530473074:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.460500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473083:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.460288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958662:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.460948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241356034958660:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.461282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241356034958564:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.461295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.461302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241356034958564:2049] [node 2] ICP22 created new session: [4:7587241356034958663:2049]
2025-12-24T02:11:28.461309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241356034958663:2049] [node 2] ICS09 handshake done sender: [4:7587241356034958660:2120] self: [4:7587241356034958661:0] peer: [2:7587241354953606819:0] socket: 213 qp: -1
2025-12-24T02:11:28.461316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241356034958663:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.461323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.460609Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241355530473027:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19399) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241355530473070:2116] held: no
2025-12-24T02:11:28.461130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241355530473072:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.461808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241355530473073:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.461176Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241357220534791:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13866) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241357220534861:2114] held: no
2025-12-24T02:11:28.461363Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357220534832:2085] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.466117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241357220534864:2074] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.466136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.466143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241357220534864:2074] [node 4] ICP22 created new session: [5:7587241357220534888:2074]
2025-12-24T02:11:28.466149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357220534888:2074] [node 4] ICS09 handshake done sender: [5:7587241357220534832:2085] self: [5:7587241357220534876:0] peer: [4:7587241356034958582:0] socket: 167 qp: -1
2025-12-24T02:11:28.466156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357220534888:2074] [node 4] ICS10 traffic start
2025-12-24T02:11:28.466164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.466327Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241357220534864:2074] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:28.466567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534886:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.467672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241355530473050:2097] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.467801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241355530473074:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.467861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241357220534887:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51019
2025-12-24T02:11:28.470834Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241357220534792:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241357220534873:2119] held: no
2025-12-24T02:11:28.470939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357220534873:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.471274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241355530472991:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.471290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.471296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241355530472991:2048] [node 1] ICP22 created new session: [6:7587241355530473085:2048]
2025-12-24T02:11:28.471303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241355530473085:2048] [node 1] ICS09 handshake done sender: [6:7587241355530473072:2117] self: [6:7587241355530473077:0] peer: [1:7587241354870191323:0] socket: 218 qp: -1
2025-12-24T02:11:28.471308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241355530473085:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.471328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.471720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241355530473028:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.471727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.471733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241355530473028:2069] [node 4] ICP22 created new session: [6:7587241355530473087:2069]
2025-12-24T02:11:28.471738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241355530473087:2069] [node 4] ICS09 handshake done sender: [6:7587241355530473073:2118] self: [6:7587241355530473075:0] peer: [4:7587241356034958621:0] socket: 223 qp: -1
2025-12-24T02:11:28.471753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241355530473087:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:28.471757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.472037Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241355530472993:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15823) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241355530473050:2097] held: no
2025-12-24T02:11:28.472111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241355530472993:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.472115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.472120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241355530472993:2050] [node 3] ICP22 created new session: [6:7587241355530473089:2050]
2025-12-24T02:11:28.472125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241355530473089:2050] [node 3] ICS09 handshake done sender: [6:7587241355530473050:2097] self: [6:7587241355530473071:0] peer: [3:7587241355047496371:0] socket: 196 qp: -1
2025-12-24T02:11:28.472129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241355530473089:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.472134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.471061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241357220534861:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.472317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241355530472992:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.472350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.472354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241355530472992:2049] [node 2] ICP22 created new session: [6:7587241355530473091:2049]
2025-12-24T02:11:28.472358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241355530473091:2049] [node 2] ICS09 handshake done sender: [6:7587241355530473074:2119] self: [6:7587241355530473078:0] peer: [2:7587241354953606802:0] socket: 224 qp: -1
2025-12-24T02:11:28.472362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241355530473091:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.472366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.472539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241355530473082:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10791
2025-12-24T02:11:28.478936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241357220534791:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.478956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.478967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241357220534791:2049] [node 2] ICP22 created new session: [5:7587241357220534890:2049]
2025-12-24T02:11:28.478974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357220534890:2049] [node 2] ICS09 handshake done sender: [5:7587241357220534861:2114] self: [5:7587241357220534875:0] peer: [2:7587241354953606803:0] socket: 205 qp: -1
2025-12-24T02:11:28.478981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357220534890:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.478989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.479193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241357220534792:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.479198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.479203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241357220534792:2050] [node 3] ICP22 created new session: [5:7587241357220534892:2050]
2025-12-24T02:11:28.479207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241357220534892:2050] [node 3] ICS09 handshake done sender: [5:7587241357220534873:2119] self: [5:7587241357220534884:0] peer: [3:7587241355047496441:0] socket: 225 qp: -1
2025-12-24T02:11:28.479213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241357220534892:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.479218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.480519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241355530473070:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.481149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241355530473027:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.481160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.481166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241355530473027:2059] [node 5] ICP22 created new session: [6:7587241355530473093:2059]
2025-12-24T02:11:28.481171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241355530473093:2059] [node 5] ICS09 handshake done sender: [6:7587241355530473070:2116] self: [6:7587241355530473076:0] peer: [5:7587241357220534862:0] socket: 208 qp: -1
2025-12-24T02:11:28.481177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241355530473093:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:28.481182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.482937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241356034958665:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8556
2025-12-24T02:11:28.503104Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:28.503201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241355047496450:2048] [node 1] ICS01 socket: 189 reason# EndOfStream
2025-12-24T02:11:28.503211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241355047496357:2048] [node 1] ICP30 unregister session Session# [3:7587241355047496450:2048] VirtualId# [3:7587241355047496440:0]
2025-12-24T02:11:28.503219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241355047496450:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.503266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.503291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.503325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:28.503382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241357220534877:2048] [node 1] ICS07 socket disconnect 201 reason# EndOfStream
2025-12-24T02:11:28.503386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357220534877:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.503412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.503430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241357220534877:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:28.503962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241357220534790:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:28.503974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357220534877:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:28.503978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241357220534790:2048] [node 1] ICP30 unregister session Session# [5:7587241357220534877:2048] VirtualId# [5:7587241357220534872:0]
2025-12-24T02:11:28.504029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.504035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241357220534790:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:28.504038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28099) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:28.506833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:28.506928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241355530473085:2048] [node 1] ICS01 socket: 218 reason# EndOfStream
2025-12-24T02:11:28.506935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241355530472991:2048] [node 1] ICP30 unregister session Session# [6:7587241355530473085:2048] VirtualId# [6:7587241355530473077:0]
2025-12-24T02:11:28.506941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241355530473085:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.506992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.507017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.507176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:28.507244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241356034958650:2048] [node 1] ICS01 socket: 184 reason# EndOfStream
2025-12-24T02:11:28.507250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241356034958563:2048] [node 1] ICP30 unregister session Session# [4:7587241356034958650:2048] VirtualId# [4:7587241356034958643:0]
2025-12-24T02:11:28.507255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241356034958650:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.507288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:28.507317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:28.518767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241357220534790:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:28.518891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:28.518993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241355047496457:2049] [node 2] ICS01 socket: 182 reason# EndOfStream
2025-12-24T02:11:28.519002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241355047496358:2049] [node 2] ICP30 unregister session Session# [3:7587241355047496457:2049] VirtualId# [3:7587241355047496439:0]
2025-12-24T02:11:28.519014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241355047496457:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.519078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.519105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.520098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:28.520181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357220534890:2049] [node 2] ICS01 socket: 205 reason# EndOfStream
2025-12-24T02:11:28.520190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241357220534791:2049] [node 2] ICP30 unregister session Session# [5:7587241357220534890:2049] VirtualId# [5:7587241357220534875:0]
2025-12-24T02:11:28.520195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357220534890:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.520242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.520268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.520685Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:28.520768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241355530473091:2049] [node 2] ICS01 socket: 224 reason# EndOfStream
2025-12-24T02:11:28.520775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241355530472992:2049] [node 2] ICP30 unregister session Session# [6:7587241355530473091:2049] VirtualId# [6:7587241355530473078:0]
2025-12-24T02:11:28.520781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241355530473091:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.520816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.520842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.526760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:28.526859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241356034958663:2049] [node 2] ICS01 socket: 213 reason# EndOfStream
2025-12-24T02:11:28.526867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241356034958564:2049] [node 2] ICP30 unregister session Session# [4:7587241356034958663:2049] VirtualId# [4:7587241356034958661:0]
2025-12-24T02:11:28.526873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241356034958663:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.526919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:28.526945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:28.552482Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:28.552580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241357220534892:2050] [node 3] ICS01 socket: 225 reason# EndOfStream
2025-12-24T02:11:28.552592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241357220534792:2050] [node 3] ICP30 unregister session Session# [5:7587241357220534892:2050] VirtualId# [5:7587241357220534884:0]
2025-12-24T02:11:28.552600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241357220534892:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.552652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:28.552677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:28.554751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:28.554842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241355530473089:2050] [node 3] ICS01 socket: 196 reason# EndOfStream
2025-12-24T02:11:28.554849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241355530472993:2050] [node 3] ICP30 unregister session Session# [6:7587241355530473089:2050] VirtualId# [6:7587241355530473071:0]
2025-12-24T02:11:28.554856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241355530473089:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.554905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:28.554930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:28.586772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:28.586881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241355530473087:2069] [node 4] ICS01 socket: 223 reason# EndOfStream
2025-12-24T02:11:28.586904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241355530473028:2069] [node 4] ICP30 unregister session Session# [6:7587241355530473087:2069] VirtualId# [6:7587241355530473075:0]
2025-12-24T02:11:28.586915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241355530473087:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:28.586976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:28.587003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:28.650777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241356907992356:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.650814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241356907992357:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:28.658724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241356113181318:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.658841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241354001229849:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.658856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241354001229850:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.658861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241354001229851:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:28.658922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241356141673997:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:28.658932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241356141673998:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:28.670715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241356113181319:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.662795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241358205330479:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.662812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241358205330481:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:28.662812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241358205330480:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.670964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241356278234642:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:28.670976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241356278234643:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:28.670979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241356278234644:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.740087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992416:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.740892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356113181378:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.739158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241354001229907:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.744071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992417:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
2025-12-24T02:11:28.741964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234669:2068] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.746479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241356278234686:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:28.746571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241356278234687:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:28.743060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241356141674058:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:28.750852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241356141674068:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.778351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241356141674067:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:28.783069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241356907992421:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.783883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330540:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.783975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241356141673998:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32992029ns) {Configure:104(32991478ns) {ProcessConfigured:135(32685928ns) {StartInitialHandshake:157(32685412ns) {DropHandshakes:470(401ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(1021ns) {SwitchToState:179(434ns)}}}}}
2025-12-24T02:11:28.785406Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241356141674068:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34435612ns) {Configure:104(34435203ns) {ProcessConfigured:135(33985941ns) {StartInitialHandshake:157(33985642ns) {DropHandshakes:470(407ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(552ns) {SwitchToState:179(228ns)}}}}}
2025-12-24T02:11:28.783173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241356907992422:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:28.786790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241356141674069:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6957
2025-12-24T02:11:28.795651Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241356907992356:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48717246ns) {Configure:104(48716652ns) {ProcessConfigured:135(48344803ns) {StartInitialHandshake:157(48344456ns) {DropHandshakes:470(413ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1105ns) {SwitchToState:179(716ns)}}}}}
2025-12-24T02:11:28.810719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241356113181380:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:28.810910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241356113181384:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:28.811007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241356113181385:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:28.811177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181378:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:28.811099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241354001229916:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.812633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356113181396:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:28.812368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241356907992421:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28647780ns) {Configure:104(26188151ns) {ProcessConfigured:135(25875916ns) {StartInitialHandshake:157(25874940ns) {DropHandshakes:470(241ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(778ns) {SwitchToState:179(366ns)}}}}}
2025-12-24T02:11:28.812415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992416:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.812585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992433:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.814951Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241356113181318:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28650370ns) {Configure:104(28649602ns) {ProcessConfigured:135(28202362ns) {StartInitialHandshake:157(28201954ns) {DropHandshakes:470(556ns) {DropIncomingHandshake:442(120ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:477(1118ns) {SwitchToState:179(767ns)}}}}}
2025-12-24T02:11:28.814901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330550:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.815447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330551:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.815119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241356907992426:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.822971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234722:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.822993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234721:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.823307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241358205330553:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:28.826190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330540:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.826235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241358205330479:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.826368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674060:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.826380Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241356141674060:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.826906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674069:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.826947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241356141674060:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.830797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241356278234669:2068] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.830858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241356278234687:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.831031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992437:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
2025-12-24T02:11:28.831048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241354001229909:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.831063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241354001229909:2105] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.834830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241356907992357:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51603204ns) {Configure:104(51601697ns) {ProcessConfigured:135(51153120ns) {StartInitialHandshake:157(51152063ns) {DropHandshakes:470(1410ns) {DropIncomingHandshake:442(324ns) DropOutgoingHandshake:457(338ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:477(1155ns) {SwitchToState:179(623ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.850927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241358205330558:2116] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.850839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241356141674058:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.850866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.850879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241356141674058:2107] [node 4] ICP22 created new session: [1:7587241356141674080:2107]
2025-12-24T02:11:28.850888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241356141674080:2107] [node 4] ICS09 handshake done sender: [1:7587241356141674060:2108] self: [1:7587241356141674059:0] peer: [4:7587241358205330556:0] socket: 184 qp: -1
2025-12-24T02:11:28.850899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241356141674080:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:28.850941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.851471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241356141674078:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6957
2025-12-24T02:11:28.851513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241356141674079:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6957
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.854827Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241356113181319:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28034765ns) {Configure:104(28034460ns) {ProcessConfigured:135(27603323ns) {StartInitialHandshake:157(27602674ns) {DropHandshakes:470(238ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1302ns) {SwitchToState:179(793ns)}}}}}
2025-12-24T02:11:28.855672Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992417:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.855713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241356907992356:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.856256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992437:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.854898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234724:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.851643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330561:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.858867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241358205330540:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.858944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674078:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.858979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241356141674058:2107] [node 4] ICP18 handshake (actor [1:7587241356141674078:2116]) is too old
2025-12-24T02:11:28.859363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674062:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.859375Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241356141674062:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.863013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241358205330479:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.863037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.863047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241358205330479:2048] [node 1] ICP22 created new session: [4:7587241358205330562:2048]
2025-12-24T02:11:28.863056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241358205330562:2048] [node 1] ICS09 handshake done sender: [4:7587241358205330540:2108] self: [4:7587241358205330556:0] peer: [1:7587241356141674059:0] socket: 188 qp: -1
2025-12-24T02:11:28.863065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241358205330562:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.863076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.863038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229929:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:28.863649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241354001229909:2105] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.863735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241354001229907:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.863748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.863754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241354001229907:2104] [node 6] ICP22 created new session: [5:7587241354001229930:2104]
2025-12-24T02:11:28.863769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241354001229930:2104] [node 6] ICS09 handshake done sender: [5:7587241354001229909:2105] self: [5:7587241354001229908:0] peer: [6:7587241356278234723:0] socket: 174 qp: -1
2025-12-24T02:11:28.863774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241354001229930:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:28.863796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.867031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241356141674084:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6957
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.870832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181396:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.870841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356113181400:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:28.870875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241356113181318:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.879255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674084:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.886729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674072:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.886753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241356141674072:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.882803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241356278234724:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.883065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:28.891008Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241358205330479:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27907898ns) {HandleHandshakeStatus:329(27906891ns) {DropIncomingHandshake:442(515ns) DropOutgoingHandshake:457(1690ns) SwitchToState:179(94ns) ProcessPendingSessionEvents:495(123940ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 109889ns max# 42596ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.894784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241356907992426:2111] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(75977982ns) {Configure:104(75977020ns) {ProcessConfigured:135(75576266ns) {StartInitialHandshake:157(75575163ns) {DropHandshakes:470(366ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(1604ns) {SwitchToState:179(1231ns)}}}}}
2025-12-24T02:11:28.894834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992433:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.895160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.895445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241356278234669:2068] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.895166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330551:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.895437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241358205330479:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.899022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234727:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.899128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241356278234687:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.899146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.899153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241356278234687:2077] [node 5] ICP22 created new session: [6:7587241356278234728:2077]
2025-12-24T02:11:28.899161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241356278234728:2077] [node 5] ICS09 handshake done sender: [6:7587241356278234669:2068] self: [6:7587241356278234723:0] peer: [5:7587241354001229908:0] socket: 180 qp: -1
2025-12-24T02:11:28.899168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241356278234728:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:28.899177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.903394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181400:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.902887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330566:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.906759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674074:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.906779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241356141674074:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.907893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992443:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
2025-12-24T02:11:28.910821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241356141674062:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.910885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241356141674072:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.911813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241356141673998:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.911833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.911841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241356141673998:2049] [node 3] ICP22 created new session: [1:7587241356141674088:2049]
2025-12-24T02:11:28.911850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241356141674088:2049] [node 3] ICS09 handshake done sender: [1:7587241356141674072:2113] self: [1:7587241356141674070:0] peer: [3:7587241356113181401:0] socket: 189 qp: -1
2025-12-24T02:11:28.911856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241356141674088:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.911864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.911674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241356278234722:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.911702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241356278234721:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.911895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241356278234643:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.914736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241356278234642:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.911761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241354001229929:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.911802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241354001229850:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.914781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241356278234642:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31893154ns) {IncomingHandshake:261(31892749ns) {DropIncomingHandshake:442(251ns) PrepareNewSessionHandshake:477(676ns) {SwitchToState:179(131ns)} GenerateSessionVirtualId:988(248ns) IssueIncomingHandshakeReply:193(31887591ns) {SwitchToState:179(852ns)}}}
2025-12-24T02:11:28.911522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356907992417:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.914737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241356907992356:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.914751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.914760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241356907992356:2048] [node 1] ICP22 created new session: [2:7587241356907992444:2048]
2025-12-24T02:11:28.914767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356907992444:2048] [node 1] ICS09 handshake done sender: [2:7587241356907992417:2108] self: [2:7587241356907992439:0] peer: [1:7587241356141674061:0] socket: 186 qp: -1
2025-12-24T02:11:28.914773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356907992444:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.914790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.915174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241356141673997:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.915195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.915201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241356141673997:2048] [node 2] ICP22 created new session: [1:7587241356141674090:2048]
2025-12-24T02:11:28.915214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241356141674090:2048] [node 2] ICS09 handshake done sender: [1:7587241356141674062:2109] self: [1:7587241356141674061:0] peer: [2:7587241356907992439:0] socket: 185 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.918067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330566:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.915222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241356141674090:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:28.915232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.916027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234733:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.916139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229934:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:28.918923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674079:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.918986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241356141674074:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.922870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992432:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.922883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356907992432:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.922892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241354001229927:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.922937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241354001229851:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.922818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181393:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.922834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356113181393:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.923038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330550:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.923068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241358205330481:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.923239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241356278234724:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.923337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241356278234642:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.923357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.923364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241356278234642:2048] [node 1] ICP22 created new session: [6:7587241356278234734:2048]
2025-12-24T02:11:28.923371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241356278234734:2048] [node 1] ICS09 handshake done sender: [6:7587241356278234724:2118] self: [6:7587241356278234725:0] peer: [1:7587241356141674073:0] socket: 219 qp: -1
2025-12-24T02:11:28.923378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241356278234734:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.923398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:28.922838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241356141673997:2048] [node 2] ICP18 handshake (actor [1:7587241356141674079:2117]) is too old
2025-12-24T02:11:28.923406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241356141674067:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.923423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.923428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241356141674067:2110] [node 6] ICP22 created new session: [1:7587241356141674094:2110]
2025-12-24T02:11:28.923434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241356141674094:2110] [node 6] ICS09 handshake done sender: [1:7587241356141674074:2114] self: [1:7587241356141674073:0] peer: [6:7587241356278234725:0] socket: 218 qp: -1
2025-12-24T02:11:28.923439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241356141674094:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:28.923445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.924033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992429:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.924045Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356907992429:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.926849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356907992429:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.925116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241356278234644:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.926791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181387:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.926800Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356113181387:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.927602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241356907992422:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.927621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.927627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241356907992422:2110] [node 6] ICP22 created new session: [2:7587241356907992448:2110]
2025-12-24T02:11:28.927632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356907992448:2110] [node 6] ICS09 handshake done sender: [2:7587241356907992429:2113] self: [2:7587241356907992428:0] peer: [6:7587241356278234732:0] socket: 195 qp: -1
2025-12-24T02:11:28.927636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356907992448:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:28.927651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.927762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356113181387:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.927903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241356113181384:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.927914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.927920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241356113181384:2109] [node 5] ICP22 created new session: [3:7587241356113181404:2109]
2025-12-24T02:11:28.927924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356113181404:2109] [node 5] ICS09 handshake done sender: [3:7587241356113181387:2111] self: [3:7587241356113181386:0] peer: [5:7587241354001229935:0] socket: 199 qp: -1
2025-12-24T02:11:28.927929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356113181404:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:28.927935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.928653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234736:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.929708Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356113181393:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.929824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241356113181380:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.929839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.929844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241356113181380:2108] [node 6] ICP22 created new session: [3:7587241356113181408:2108]
2025-12-24T02:11:28.929849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356113181408:2108] [node 6] ICS09 handshake done sender: [3:7587241356113181393:2113] self: [3:7587241356113181391:0] peer: [6:7587241356278234731:0] socket: 200 qp: -1
2025-12-24T02:11:28.929853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356113181408:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:28.929858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.928984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241356141674096:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6957
2025-12-24T02:11:28.930045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674096:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.930072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241356141673998:2049] [node 3] ICP18 handshake (actor [1:7587241356141674096:2119]) is too old
2025-12-24T02:11:28.930822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181389:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.930834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241356113181389:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.931049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241354001229934:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.931081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241354001229849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.935258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241356278234721:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:28.938926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356907992432:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.942846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241356141674076:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.942863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241356141674076:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.946000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241356907992421:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.946025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.946033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241356907992421:2109] [node 5] ICP22 created new session: [2:7587241356907992453:2109]
2025-12-24T02:11:28.946041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356907992453:2109] [node 5] ICS09 handshake done sender: [2:7587241356907992432:2114] self: [2:7587241356907992425:0] peer: [5:7587241354001229933:0] socket: 205 qp: -1
2025-12-24T02:11:28.946048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356907992453:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:28.946087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.946672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241356113181318:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241356113181396:2114] held: no
2025-12-24T02:11:28.946954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356113181411:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:28.946991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241356113181412:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:28.947458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356113181389:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.947537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241356113181385:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.947553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.947569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241356113181385:2110] [node 4] ICP22 created new session: [3:7587241356113181414:2110]
2025-12-24T02:11:28.947574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356113181414:2110] [node 4] ICS09 handshake done sender: [3:7587241356113181389:2112] self: [3:7587241356113181388:0] peer: [4:7587241358205330568:0] socket: 198 qp: -1
2025-12-24T02:11:28.947579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356113181414:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:28.947585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.947770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356113181396:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:28.948002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241356113181318:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.948014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:28.948018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241356113181318:2048] [node 1] ICP22 created new session: [3:7587241356113181416:2048]
2025-12-24T02:11:28.948023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241356113181416:2048] [node 1] ICS09 handshake done sender: [3:7587241356113181396:2114] self: [3:7587241356113181401:0] peer: [1:7587241356141674070:0] socket: 190 qp: -1
2025-12-24T02:11:28.948027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241356113181416:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:28.948033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.950314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234737:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.950583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241356278234727:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.950595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241356278234643:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.950606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:28.950612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241356278234643:2049] [node 2] ICP22 created new session: [6:7587241356278234738:2049]
2025-12-24T02:11:28.950617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241356278234738:2049] [node 2] ICS09 handshake done sender: [6:7587241356278234721:2116] self: [6:7587241356278234732:0] peer: [2:7587241356907992428:0] socket: 196 qp: -1
2025-12-24T02:11:28.950628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241356278234738:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:28.950635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:28.951046Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241358205330481:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241358205330550:2112] held: no
2025-12-24T02:11:28.951609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241356907992422:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(23897947ns) {HandleHandshakeStatus:329(23897042ns) {DropIncomingHandshake:442(2963ns) DropOutgoingHandshake:457(101ns) SwitchToState:179(61ns) ProcessPendingSessionEvents:495(20438675ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 20421517ns max# 20356740ns}}}
2025-12-24T02:11:28.954763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241356907992451:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12624
2025-12-24T02:11:28.955829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241356278234643:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:28.956618Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241356278234644:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2802) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241356278234722:2117] held: no
2025-12-24T02:11:28.956733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241356278234722:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.956855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241356278234644:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.956869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.956874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241356278234644:2050] [node 3] ICP22 created new session: [6:7587241356278234740:2050]
2025-12-24T02:11:28.956879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241356278234740:2050] [node 3] ICS09 handshake done sender: [6:7587241356278234722:2117] self: [6:7587241356278234731:0] peer: [3:7587241356113181391:0] socket: 213 qp: -1
2025-12-24T02:11:28.956885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241356278234740:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.956890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.957059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241356278234686:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.958780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330555:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.958790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241358205330555:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.970332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992438:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.970346Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356907992438:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:28.969199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241356113181412:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.969242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241356113181319:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.970902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356907992438:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.971139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241356907992357:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.971148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.971154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241356907992357:2049] [node 3] ICP22 created new session: [2:7587241356907992456:2049]
2025-12-24T02:11:28.971159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241356907992456:2049] [node 3] ICS09 handshake done sender: [2:7587241356907992438:2117] self: [2:7587241356907992423:0] peer: [3:7587241356113181418:0] socket: 234 qp: -1
2025-12-24T02:11:28.971164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241356907992456:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:28.971169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.970829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241356141674076:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:28.971025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241356141674068:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.971042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:28.971050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241356141674068:2111] [node 5] ICP22 created new session: [1:7587241356141674098:2111]
2025-12-24T02:11:28.971056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241356141674098:2111] [node 5] ICS09 handshake done sender: [1:7587241356141674076:2115] self: [1:7587241356141674071:0] peer: [5:7587241354001229936:0] socket: 231 qp: -1
2025-12-24T02:11:28.971061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241356141674098:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:28.971069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:28.975854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330569:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.982991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229937:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:28.983050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229938:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:28.987756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241356278234743:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33612
2025-12-24T02:11:28.988071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241356278234727:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:28.988723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241356278234686:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.988739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:28.988745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241356278234686:2075] [node 4] ICP22 created new session: [6:7587241356278234744:2075]
2025-12-24T02:11:28.988750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241356278234744:2075] [node 4] ICS09 handshake done sender: [6:7587241356278234727:2120] self: [6:7587241356278234742:0] peer: [4:7587241358205330554:0] socket: 223 qp: -1
2025-12-24T02:11:28.988754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241356278234744:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:28.988760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:28.987201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241358205330571:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:28.987898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241358205330555:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:28.988182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241358205330553:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:28.988197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:28.988204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241358205330553:2114] [node 6] ICP22 created new session: [4:7587241358205330573:2114]
2025-12-24T02:11:28.988211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241358205330573:2114] [node 6] ICS09 handshake done sender: [4:7587241358205330555:2115] self: [4:7587241358205330554:0] peer: [6:7587241356278234742:0] socket: 191 qp: -1
2025-12-24T02:11:28.988217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241358205330573:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:28.988226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:28.988681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241358205330550:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:28.989956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241358205330481:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:28.989964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:28.989968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241358205330481:2050] [node 3] ICP22 created new session: [4:7587241358205330575:2050]
2025-12-24T02:11:28.989973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241358205330575:2050] [node 3] ICS09 handshake done sender: [4:7587241358205330550:2112] self: [4:7587241358205330568:0] peer: [3:7587241356113181388:0] socket: 207 qp: -1
2025-12-24T02:11:28.989978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241358205330575:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:28.989982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:28.990872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992451:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.991003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330569:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.991042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241358205330480:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:28.991006Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241356141674068:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(19875094ns) {HandleHandshakeStatus:329(19874379ns) {DropIncomingHandshake:442(4641ns) DropOutgoingHandshake:457(262ns) SwitchToState:179(102ns) ProcessPendingSessionEvents:495(19791969ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19763007ns max# 19694137ns}}}
2025-12-24T02:11:28.991410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241356907992357:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20172201ns) {HandleHandshakeStatus:329(20172021ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(109ns) SwitchToState:179(43ns) ProcessPendingSessionEvents:495(20122672ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20097524ns max# 20027519ns}}}
2025-12-24T02:11:28.991445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241356907992357:2049] [node 3] ICP18 handshake (actor [2:7587241356907992451:2120]) is too old
2025-12-24T02:11:28.994896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241356907992440:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:28.994907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241356907992440:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.006352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241356113181319:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241356113181412:2119] held: no
2025-12-24T02:11:29.003690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241354001229850:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241354001229929:2116] held: no
2025-12-24T02:11:29.003833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241354001229929:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.004826Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241356907992440:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.005340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241354001229850:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.004911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241356907992426:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.005349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.004928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.005354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241354001229850:2049] [node 2] ICP22 created new session: [5:7587241358296197237:2049]
2025-12-24T02:11:29.004936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241356907992426:2111] [node 4] ICP22 created new session: [2:7587241361202959755:2111]
2025-12-24T02:11:29.005359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358296197237:2049] [node 2] ICS09 handshake done sender: [5:7587241354001229929:2116] self: [5:7587241354001229933:0] peer: [2:7587241356907992425:0] socket: 216 qp: -1
2025-12-24T02:11:29.005364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358296197237:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.004944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241361202959755:2111] [node 4] ICS09 handshake done sender: [2:7587241356907992440:2118] self: [2:7587241356907992436:0] peer: [4:7587241358205330577:0] socket: 245 qp: -1
2025-12-24T02:11:29.005371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.004950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241361202959755:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:29.005658Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241354001229849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241354001229934:2117] held: no
2025-12-24T02:11:29.005815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241354001229927:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.006004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:29.006050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241354001229939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:29.006542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241354001229934:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.006666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241354001229849:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.006674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.006679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241354001229849:2048] [node 1] ICP22 created new session: [5:7587241358296197240:2048]
2025-12-24T02:11:29.006685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358296197240:2048] [node 1] ICS09 handshake done sender: [5:7587241354001229934:2117] self: [5:7587241354001229936:0] peer: [1:7587241356141674071:0] socket: 233 qp: -1
2025-12-24T02:11:29.006704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358296197240:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.006711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.004961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.007002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241354001229851:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.007016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.007021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241354001229851:2050] [node 3] ICP22 created new session: [5:7587241358296197242:2050]
2025-12-24T02:11:29.007026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358296197242:2050] [node 3] ICS09 handshake done sender: [5:7587241354001229927:2115] self: [5:7587241354001229935:0] peer: [3:7587241356113181386:0] socket: 208 qp: -1
2025-12-24T02:11:29.007031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358296197242:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.007037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.006960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241358205330480:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20529) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241358205330569:2120] held: no
2025-12-24T02:11:29.008569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241354001229940:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.008613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241354001229916:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.010789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241358205330560:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.010802Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241358205330560:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.011241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241362500297874:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18440
2025-12-24T02:11:29.011160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241360408148715:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:29.014899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241358205330569:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.014939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241356113181412:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.015006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241358205330480:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.015023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.015031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241358205330480:2049] [node 2] ICP22 created new session: [4:7587241362500297875:2049]
2025-12-24T02:11:29.015037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241362500297875:2049] [node 2] ICS09 handshake done sender: [4:7587241358205330569:2120] self: [4:7587241358205330577:0] peer: [2:7587241356907992436:0] socket: 240 qp: -1
2025-12-24T02:11:29.015051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241362500297875:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.015060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.015059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241356113181319:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.015074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.015081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241356113181319:2049] [node 2] ICP22 created new session: [3:7587241360408148717:2049]
2025-12-24T02:11:29.015087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241360408148717:2049] [node 2] ICS09 handshake done sender: [3:7587241356113181412:2119] self: [3:7587241356113181418:0] peer: [2:7587241356907992423:0] socket: 243 qp: -1
2025-12-24T02:11:29.015094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241360408148717:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.015107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.016488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241358205330560:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.016589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241358205330558:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.016606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.016612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241358205330558:2116] [node 5] ICP22 created new session: [4:7587241362500297878:2116]
2025-12-24T02:11:29.016620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241362500297878:2116] [node 5] ICS09 handshake done sender: [4:7587241358205330560:2117] self: [4:7587241358205330559:0] peer: [5:7587241358296197244:0] socket: 179 qp: -1
2025-12-24T02:11:29.016625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241362500297878:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:29.016631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.016217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241360408148716:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61962
2025-12-24T02:11:29.018346Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241356113181384:2109] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.019137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241354001229916:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2120) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241354001229940:2121] held: no
2025-12-24T02:11:29.019271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358296197245:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7999
2025-12-24T02:11:29.019449Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241354001229940:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.019491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241354001229916:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.019499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.019505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241354001229916:2110] [node 4] ICP22 created new session: [5:7587241358296197246:2110]
2025-12-24T02:11:29.019510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358296197246:2110] [node 4] ICS09 handshake done sender: [5:7587241354001229940:2121] self: [5:7587241358296197244:0] peer: [4:7587241358205330559:0] socket: 210 qp: -1
2025-12-24T02:11:29.019516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358296197246:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:29.019522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.046789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:29.046923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241356113181416:2048] [node 1] ICS01 socket: 190 reason# EndOfStream
2025-12-24T02:11:29.046936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241356113181318:2048] [node 1] ICP30 unregister session Session# [3:7587241356113181416:2048] VirtualId# [3:7587241356113181401:0]
2025-12-24T02:11:29.046946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241356113181416:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.046994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.047020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.047059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:29.047128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241356907992444:2048] [node 1] ICS07 socket disconnect 186 reason# EndOfStream
2025-12-24T02:11:29.047132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241356907992444:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.047158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.047177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241356907992444:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:29.047762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241356907992356:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.047774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241356907992444:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.047779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241356907992356:2048] [node 1] ICP30 unregister session Session# [2:7587241356907992444:2048] VirtualId# [2:7587241356907992439:0]
2025-12-24T02:11:29.047795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.047802Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241356907992356:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.047805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.054436Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:29.054543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241356278234734:2048] [node 1] ICS07 socket disconnect 219 reason# EndOfStream
2025-12-24T02:11:29.054552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241356278234734:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.054604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.054625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241356278234734:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:29.055091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241356278234642:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.055104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241356278234734:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.055108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241356278234642:2048] [node 1] ICP30 unregister session Session# [6:7587241356278234734:2048] VirtualId# [6:7587241356278234725:0]
2025-12-24T02:11:29.055158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.055164Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241356278234642:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.055167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.054826Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:29.054902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241358205330562:2048] [node 1] ICS07 socket disconnect 188 reason# EndOfStream
2025-12-24T02:11:29.054907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241358205330562:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.054952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.054971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241358205330562:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:29.055717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241358205330479:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.055728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241358205330562:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.055732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241358205330479:2048] [node 1] ICP30 unregister session Session# [4:7587241358205330562:2048] VirtualId# [4:7587241358205330556:0]
2025-12-24T02:11:29.055779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.055784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241358205330479:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.055787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11547) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.055162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:29.055227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358296197240:2048] [node 1] ICS01 socket: 233 reason# EndOfStream
2025-12-24T02:11:29.055233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241354001229849:2048] [node 1] ICP30 unregister session Session# [5:7587241358296197240:2048] VirtualId# [5:7587241354001229936:0]
2025-12-24T02:11:29.055238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358296197240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.055265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.055290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.066921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241356278234642:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:29.069810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241358205330479:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:29.075374Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:29.075478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241360408148717:2049] [node 2] ICS01 socket: 243 reason# EndOfStream
2025-12-24T02:11:29.075489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241356113181319:2049] [node 2] ICP30 unregister session Session# [3:7587241360408148717:2049] VirtualId# [3:7587241356113181418:0]
2025-12-24T02:11:29.075497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241360408148717:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.075533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.075556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.080904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:29.080992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358296197237:2049] [node 2] ICS01 socket: 216 reason# EndOfStream
2025-12-24T02:11:29.081001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241354001229850:2049] [node 2] ICP30 unregister session Session# [5:7587241358296197237:2049] VirtualId# [5:7587241354001229933:0]
2025-12-24T02:11:29.081007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358296197237:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.081046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.081069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.082731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:29.082801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241356278234738:2049] [node 2] ICS01 socket: 196 reason# EndOfStream
2025-12-24T02:11:29.082811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241356278234643:2049] [node 2] ICP30 unregister session Session# [6:7587241356278234738:2049] VirtualId# [6:7587241356278234732:0]
2025-12-24T02:11:29.082817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241356278234738:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.082864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.082888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.087479Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:29.087577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241362500297875:2049] [node 2] ICS01 socket: 240 reason# EndOfStream
2025-12-24T02:11:29.087586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241358205330480:2049] [node 2] ICP30 unregister session Session# [4:7587241362500297875:2049] VirtualId# [4:7587241358205330577:0]
2025-12-24T02:11:29.087592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241362500297875:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.087642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.087668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.106823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:29.106919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241358205330575:2050] [node 3] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:11:29.106929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241358205330481:2050] [node 3] ICP30 unregister session Session# [4:7587241358205330575:2050] VirtualId# [4:7587241358205330568:0]
2025-12-24T02:11:29.106936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241358205330575:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.106986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:29.107011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:29.109414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:29.109510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241358296197242:2050] [node 3] ICS07 socket disconnect 208 reason# EndOfStream
2025-12-24T02:11:29.109517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358296197242:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.109564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:29.109586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241358296197242:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:29.110729Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:29.110795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241356278234740:2050] [node 3] ICS01 socket: 213 reason# EndOfStream
2025-12-24T02:11:29.110801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241356278234644:2050] [node 3] ICP30 unregister session Session# [6:7587241356278234740:2050] VirtualId# [6:7587241356278234731:0]
2025-12-24T02:11:29.110806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241356278234740:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.110842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:29.110864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:29.122929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241354001229851:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2802) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.122950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358296197242:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.122958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241354001229851:2050] [node 3] ICP30 unregister session Session# [5:7587241358296197242:2050] VirtualId# [5:7587241354001229935:0]
2025-12-24T02:11:29.123021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:29.123029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241354001229851:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.123032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2802) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.141668Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:29.141790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241356278234744:2075] [node 4] ICS01 socket: 223 reason# EndOfStream
2025-12-24T02:11:29.141800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241356278234686:2075] [node 4] ICP30 unregister session Session# [6:7587241356278234744:2075] VirtualId# [6:7587241356278234742:0]
2025-12-24T02:11:29.141808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241356278234744:2075] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.141866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:29.141894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:29.198869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241360820662355:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:29.198903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241360820662356:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:29.202829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358466738083:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.202840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358466738084:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.209348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358466738113:2076] [node 4] ICP01 ready to work
2025-12-24T02:11:29.209905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358466738119:2079] [node 5] ICP01 ready to work
2025-12-24T02:11:29.207898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241360820662384:2074] [node 6] ICP01 ready to work
2025-12-24T02:11:29.209868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241360820662395:2081] [node 5] ICP01 ready to work
2025-12-24T02:11:29.226729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241360820662404:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:29.226881Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241360820662395:2081] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.226915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7587241360820662395:2081] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.226919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.234974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738157:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.238946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241358466738113:2076] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.239031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241358466738113:2076] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.239034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.238737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7587241360820662395:2081] [node 5] ICP33 wake up from error state
2025-12-24T02:11:29.246731Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241358466738119:2079] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.246827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241358466738119:2079] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.246830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64455) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.250909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241358466738113:2076] [node 4] ICP33 wake up from error state
2025-12-24T02:11:29.206934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241362346319931:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.206950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241362346319932:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:29.210940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241362346319972:2081] [node 4] ICP01 ready to work
2025-12-24T02:11:29.210972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241362346319973:2083] [node 6] ICP01 ready to work
2025-12-24T02:11:29.225810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241362346319970:2076] [node 5] ICP01 ready to work
2025-12-24T02:11:29.239189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241362346319972:2081] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(28199775ns) {Configure:104(28199216ns) {ProcessConfigured:135(19003842ns) {StartInitialHandshake:157(19002537ns) {DropHandshakes:470(561ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(970ns) {SwitchToState:179(538ns)}}}}}
2025-12-24T02:11:29.239408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241362346319931:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28390138ns) {Configure:104(28389729ns) {ProcessConfigured:135(19189560ns) {StartInitialHandshake:157(19189387ns) {DropHandshakes:470(225ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(38ns) PrepareNewSessionHandshake:477(520ns) {SwitchToState:179(372ns)}}}}}
2025-12-24T02:11:29.206835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241362294892588:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.206844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241362294892589:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.206849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241362294892590:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:29.234124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241362294892589:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(22624113ns) {Configure:104(22622464ns) {ProcessConfigured:135(22237454ns) {StartInitialHandshake:157(22237136ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(94ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1392ns) {SwitchToState:179(730ns)}}}}}
2025-12-24T02:11:29.238960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241362294892588:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24573265ns) {Configure:104(24572206ns) {ProcessConfigured:135(24285183ns) {StartInitialHandshake:157(24284458ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:477(873ns) {SwitchToState:179(75ns)}}}}}
2025-12-24T02:11:29.258729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241358466738119:2079] [node 5] ICP33 wake up from error state
2025-12-24T02:11:29.266793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241362346319973:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40900194ns) {Configure:104(40899743ns) {ProcessConfigured:135(40606119ns) {StartInitialHandshake:157(40605565ns) {DropHandshakes:470(423ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(174ns) PrepareNewSessionHandshake:477(1160ns) {SwitchToState:179(319ns)}}}}}
2025-12-24T02:11:29.267304Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241362346319972:2081] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.267398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241362346319972:2081] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.267401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.266887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241360820662356:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56100970ns) {Configure:104(56100221ns) {ProcessConfigured:135(55855795ns) {StartInitialHandshake:157(55855507ns) {DropHandshakes:470(335ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(1385ns) {SwitchToState:179(517ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.270888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358834554185:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:29.271474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554248:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.271662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241360820662404:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(44656978ns) {Configure:104(44656657ns) {ProcessConfigured:135(44311213ns) {StartInitialHandshake:157(44311022ns) {DropHandshakes:470(203ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(557ns) PrepareNewSessionHandshake:477(1112ns) {SwitchToState:179(723ns)}}}}}
2025-12-24T02:11:29.271831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241358466738084:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(44828284ns) {Configure:104(44827336ns) {ProcessConfigured:135(44476294ns) {StartInitialHandshake:157(44476181ns) {DropHandshakes:470(413ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(527ns) {SwitchToState:179(297ns)}}}}}
2025-12-24T02:11:29.271944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738165:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.271955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738166:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.274727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358466738136:2086] [node 6] ICP01 ready to work
2025-12-24T02:11:29.278766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7587241362346319972:2081] [node 4] ICP33 wake up from error state
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.282844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662435:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
2025-12-24T02:11:29.282876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662436:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
2025-12-24T02:11:29.282892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662437:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
2025-12-24T02:11:29.283489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892662:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.283510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892663:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.283065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241362346319970:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32258188ns) {Configure:104(32254447ns) {ProcessConfigured:135(31776480ns) {StartInitialHandshake:157(31775649ns) {DropHandshakes:470(996ns) {DropIncomingHandshake:442(207ns) DropOutgoingHandshake:457(214ns)} GenerateSessionVirtualId:988(2705ns) PrepareNewSessionHandshake:477(1464ns) {SwitchToState:179(664ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.287134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320011:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.287154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320012:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.287177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320010:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.290716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358834554247:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:29.290723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358834554246:2083] [node 6] ICP01 ready to work
2025-12-24T02:11:29.290762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358834554184:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.287752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359783486697:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.290582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359783486760:2080] [node 6] ICP01 ready to work
2025-12-24T02:11:29.290725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359783486696:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.291300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359783486759:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:29.290780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241362294892657:2079] [node 4] ICP01 ready to work
2025-12-24T02:11:29.291414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241362294892658:2083] [node 5] ICP01 ready to work
2025-12-24T02:11:29.294559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662435:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.294787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320010:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.294857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241362346319931:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.298813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241358466738136:2086] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23933708ns) {Configure:104(23932580ns) {ProcessConfigured:135(23520574ns) {StartInitialHandshake:157(23519456ns) {DropHandshakes:470(687ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(208ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1619ns) {SwitchToState:179(1119ns)}}}}}
2025-12-24T02:11:29.291059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358834554183:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.298812Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662389:2078] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.298827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241360820662389:2078] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.299453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241360820662389:2078] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.299589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241360820662355:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.299607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.299616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241360820662355:2048] [node 2] ICP22 created new session: [1:7587241360820662440:2048]
2025-12-24T02:11:29.299623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241360820662440:2048] [node 2] ICS09 handshake done sender: [1:7587241360820662389:2078] self: [1:7587241360820662388:0] peer: [2:7587241362346320013:0] socket: 173 qp: -1
2025-12-24T02:11:29.299631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241360820662440:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:29.299643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.310867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241358466738113:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48050048ns) {Configure:104(48049185ns) {ProcessConfigured:135(48038808ns) {StartInitialHandshake:157(48037838ns) {DropHandshakes:470(1364ns) {DropIncomingHandshake:442(206ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(1951ns) {SwitchToState:179(163ns)}}}}}
2025-12-24T02:11:29.311057Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241359783486697:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23162805ns) {Configure:104(23161774ns) {ProcessConfigured:135(22695561ns) {StartInitialHandshake:157(22695301ns) {DropHandshakes:470(1681ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(661ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(953ns) {SwitchToState:179(276ns)}}}}}
2025-12-24T02:11:29.311190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486768:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.314894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554255:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
2025-12-24T02:11:29.314918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554256:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.315883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662437:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.316013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662443:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
2025-12-24T02:11:29.316026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662444:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
2025-12-24T02:11:29.316045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241360820662355:2048] [node 2] ICP18 handshake (actor [1:7587241360820662437:2117]) is too old
2025-12-24T02:11:29.317201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241362346319931:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16943) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241362346320010:2114] held: no
2025-12-24T02:11:29.315686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241359783486696:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24847868ns) {Configure:104(24847496ns) {ProcessConfigured:135(24618408ns) {StartInitialHandshake:157(24618237ns) {DropHandshakes:470(1192ns) {DropIncomingHandshake:442(204ns) DropOutgoingHandshake:457(326ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(668ns) {SwitchToState:179(243ns)}}}}}
2025-12-24T02:11:29.318939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486768:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.319117Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241359783486760:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28365171ns) {Configure:104(28364314ns) {ProcessConfigured:135(28040075ns) {StartInitialHandshake:157(28039590ns) {DropHandshakes:470(1079ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(322ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(836ns) {SwitchToState:179(424ns)}}}}}
2025-12-24T02:11:29.319141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241359783486696:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.319370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241358834554246:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28496225ns) {Configure:104(28495647ns) {ProcessConfigured:135(28127920ns) {StartInitialHandshake:157(28127632ns) {DropHandshakes:470(534ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(110ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(890ns) {SwitchToState:179(307ns)}}}}}
2025-12-24T02:11:29.319645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662434:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.319655Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241360820662434:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.320336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241360820662434:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.320441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241360820662404:2084] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.320458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.320465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241360820662404:2084] [node 4] ICP22 created new session: [1:7587241360820662446:2084]
2025-12-24T02:11:29.320472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241360820662446:2084] [node 4] ICS09 handshake done sender: [1:7587241360820662434:2114] self: [1:7587241360820662426:0] peer: [4:7587241359783486773:0] socket: 214 qp: -1
2025-12-24T02:11:29.320478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241360820662446:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:29.320488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.322795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241358834554247:2096] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31889756ns) {Configure:104(31889180ns) {ProcessConfigured:135(31537321ns) {StartInitialHandshake:157(31536850ns) {DropHandshakes:470(387ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(922ns) {SwitchToState:179(300ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.323319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486774:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
2025-12-24T02:11:29.324142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359783486768:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.324997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241359783486696:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.325014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.325021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359783486696:2048] [node 1] ICP22 created new session: [4:7587241359783486775:2048]
2025-12-24T02:11:29.325028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359783486775:2048] [node 1] ICS09 handshake done sender: [4:7587241359783486768:2112] self: [4:7587241359783486773:0] peer: [1:7587241360820662426:0] socket: 216 qp: -1
2025-12-24T02:11:29.325033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359783486775:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.325053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.323029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241362294892657:2079] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32097311ns) {Configure:104(32096882ns) {ProcessConfigured:135(31716944ns) {StartInitialHandshake:157(31716668ns) {DropHandshakes:470(399ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(725ns) {SwitchToState:179(534ns)}}}}}
2025-12-24T02:11:29.323137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892668:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.326905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320016:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.327118Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241362346320010:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.327267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241362294892662:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.327302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241362294892588:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.327200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241362346319931:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.327214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.327219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241362346319931:2048] [node 1] ICP22 created new session: [2:7587241362346320018:2048]
2025-12-24T02:11:29.327226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241362346320018:2048] [node 1] ICS09 handshake done sender: [2:7587241362346320010:2114] self: [2:7587241362346320013:0] peer: [1:7587241360820662388:0] socket: 194 qp: -1
2025-12-24T02:11:29.327234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241362346320018:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.327241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.327480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662387:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.327491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241360820662387:2077] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.328014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358834554248:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.328049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358834554185:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.328119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241360820662387:2077] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.328426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.328248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241360820662384:2074] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.328264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.328273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241360820662384:2074] [node 6] ICP22 created new session: [1:7587241360820662449:2074]
2025-12-24T02:11:29.328280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241360820662449:2074] [node 6] ICS09 handshake done sender: [1:7587241360820662387:2077] self: [1:7587241360820662386:0] peer: [6:7587241362294892669:0] socket: 172 qp: -1
2025-12-24T02:11:29.328287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241360820662449:2074] [node 6] ICS10 traffic start
2025-12-24T02:11:29.328303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.324718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738171:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.328199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738162:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.328203Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358466738162:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.330755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320017:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.330813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320012:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.331117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738165:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.331158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241358466738083:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.328737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241362294892662:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.328857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241362294892588:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.328867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.328872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241362294892588:2048] [node 1] ICP22 created new session: [6:7587241362294892671:2048]
2025-12-24T02:11:29.328877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362294892671:2048] [node 1] ICS09 handshake done sender: [6:7587241362294892662:2113] self: [6:7587241362294892669:0] peer: [1:7587241360820662386:0] socket: 177 qp: -1
2025-12-24T02:11:29.328882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362294892671:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.328896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.333887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359783486698:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:29.342904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241362294892663:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.342964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241362294892589:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.346722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662432:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.346744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241360820662432:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.347656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738173:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.347846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241360820662432:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.348071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358466738165:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.348199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241358466738083:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.348220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.348229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358466738083:2048] [node 1] ICP22 created new session: [3:7587241358466738174:2048]
2025-12-24T02:11:29.348236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358466738174:2048] [node 1] ICS09 handshake done sender: [3:7587241358466738165:2116] self: [3:7587241358466738172:0] peer: [1:7587241360820662405:0] socket: 200 qp: -1
2025-12-24T02:11:29.348244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358466738174:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.348264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.347413Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241358834554184:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56230063ns) {Configure:104(56229643ns) {ProcessConfigured:135(55967294ns) {StartInitialHandshake:157(55966554ns) {DropHandshakes:470(330ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(461ns) PrepareNewSessionHandshake:477(1345ns) {SwitchToState:179(657ns)}}}}}
2025-12-24T02:11:29.350471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241360820662356:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.350494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.350500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241360820662356:2049] [node 3] ICP22 created new session: [1:7587241360820662452:2049]
2025-12-24T02:11:29.350506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241360820662452:2049] [node 3] ICS09 handshake done sender: [1:7587241360820662432:2113] self: [1:7587241360820662405:0] peer: [3:7587241358466738172:0] socket: 197 qp: -1
2025-12-24T02:11:29.350513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241360820662452:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:29.350530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.354825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358466738162:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.354645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320006:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.354663Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241362346320006:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.354728Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241362346319932:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(80024782ns) {Configure:104(80016989ns) {ProcessConfigured:135(79728496ns) {StartInitialHandshake:157(79727428ns) {DropHandshakes:470(573ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(1525ns) {SwitchToState:179(548ns)}}}}}
2025-12-24T02:11:29.355033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320017:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.355084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241362346319972:2081] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.354934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358466738119:2079] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.354952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.354958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358466738119:2079] [node 5] ICP22 created new session: [3:7587241358466738178:2079]
2025-12-24T02:11:29.354963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358466738178:2079] [node 5] ICS09 handshake done sender: [3:7587241358466738162:2114] self: [3:7587241358466738161:0] peer: [5:7587241358834554263:0] socket: 196 qp: -1
2025-12-24T02:11:29.354971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358466738178:2079] [node 5] ICS10 traffic start
2025-12-24T02:11:29.354985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.358962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554265:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.359195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554266:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
2025-12-24T02:11:29.359356Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358834554248:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.359449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358834554185:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.359465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.359471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358834554185:2050] [node 3] ICP22 created new session: [5:7587241358834554267:2050]
2025-12-24T02:11:29.362831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486767:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.362847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241359783486767:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.358913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.359179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241362294892668:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.362800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241362294892658:2083] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.362890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241362294892674:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.362911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241362294892590:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.362893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738171:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.363899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892678:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.366750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320014:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.366763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241362346320014:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.366761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359783486767:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.359478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358834554267:2050] [node 3] ICS09 handshake done sender: [5:7587241358834554248:2110] self: [5:7587241358834554263:0] peer: [3:7587241358466738161:0] socket: 198 qp: -1
2025-12-24T02:11:29.359485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358834554267:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.359501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.363315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358834554260:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.363321Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241358834554260:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.366871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358834554256:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.366926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358834554260:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:29.367128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241358834554246:2083] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.367140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.367146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358834554246:2083] [node 6] ICP22 created new session: [5:7587241358834554270:2083]
2025-12-24T02:11:29.367153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358834554270:2083] [node 6] ICS09 handshake done sender: [5:7587241358834554260:2115] self: [5:7587241358834554252:0] peer: [6:7587241362294892676:0] socket: 192 qp: -1
2025-12-24T02:11:29.367160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358834554270:2083] [node 6] ICS10 traffic start
2025-12-24T02:11:29.367168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.367290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358834554247:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.367632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241362294892668:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.367701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241360820662443:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.367843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241360820662395:2081] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.367863Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241360820662384:2074] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.367779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241362294892658:2083] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.367796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.367803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241362294892658:2083] [node 5] ICP22 created new session: [6:7587241362294892679:2083]
2025-12-24T02:11:29.367810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362294892679:2083] [node 5] ICS09 handshake done sender: [6:7587241362294892668:2117] self: [6:7587241362294892676:0] peer: [5:7587241358834554252:0] socket: 224 qp: -1
2025-12-24T02:11:29.367815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362294892679:2083] [node 5] ICS10 traffic start
2025-12-24T02:11:29.367832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.368366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358834554258:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.368376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241358834554258:2114] [node 1] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.368042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241359783486697:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.368057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.368062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359783486697:2049] [node 2] ICP22 created new session: [4:7587241359783486783:2049]
2025-12-24T02:11:29.368068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359783486783:2049] [node 2] ICS09 handshake done sender: [4:7587241359783486767:2111] self: [4:7587241359783486761:0] peer: [2:7587241362346320021:0] socket: 217 qp: -1
2025-12-24T02:11:29.368072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359783486783:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.368077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.362933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241358466738084:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.366783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738169:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.366789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358466738169:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.370825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738182:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.370784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486779:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
2025-12-24T02:11:29.371596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358834554258:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.371860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241358834554183:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.371874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.371882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358834554183:2048] [node 1] ICP22 created new session: [5:7587241358834554275:2048]
2025-12-24T02:11:29.371889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358834554275:2048] [node 1] ICS09 handshake done sender: [5:7587241358834554258:2114] self: [5:7587241358834554257:0] peer: [1:7587241360820662454:0] socket: 219 qp: -1
2025-12-24T02:11:29.371895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358834554275:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.371912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.373706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358834554255:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.373764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358834554184:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.373601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320023:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.374780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738182:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.374992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241360820662404:2084] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.378759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320023:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.378824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241362346320014:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.379420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320011:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.379597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738157:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.379654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358466738169:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:29.379771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358466738136:2086] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.379788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.379796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358466738136:2086] [node 6] ICP22 created new session: [3:7587241358466738186:2086]
2025-12-24T02:11:29.379803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358466738186:2086] [node 6] ICS09 handshake done sender: [3:7587241358466738169:2118] self: [3:7587241358466738168:0] peer: [6:7587241362294892677:0] socket: 232 qp: -1
2025-12-24T02:11:29.379809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358466738186:2086] [node 6] ICS10 traffic start
2025-12-24T02:11:29.379820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.382733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486765:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.382751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241359783486765:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.382801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486781:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
2025-12-24T02:11:29.382770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892681:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.382982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486781:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.382748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241362346320022:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40484
2025-12-24T02:11:29.382791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241362346320008:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.382797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241362346320008:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.383032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241362346320006:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:29.390489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241362294892681:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.390530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241362294892657:2079] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.386716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241362346319932:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.386741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.386750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241362346319932:2049] [node 3] ICP22 created new session: [2:7587241362346320028:2049]
2025-12-24T02:11:29.386757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241362346320028:2049] [node 3] ICS09 handshake done sender: [2:7587241362346320014:2117] self: [2:7587241362346319979:0] peer: [3:7587241358466738181:0] socket: 225 qp: -1
2025-12-24T02:11:29.386764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241362346320028:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:29.386772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.387093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241362346319973:2083] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.387102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.387104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241362346319973:2083] [node 6] ICP22 created new session: [2:7587241362346320030:2083]
2025-12-24T02:11:29.387109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241362346320030:2083] [node 6] ICS09 handshake done sender: [2:7587241362346320006:2112] self: [2:7587241362346319987:0] peer: [6:7587241362294892673:0] socket: 191 qp: -1
2025-12-24T02:11:29.387113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241362346320030:2083] [node 6] ICS10 traffic start
2025-12-24T02:11:29.387118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.393353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241360820662455:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12098
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.395000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486772:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.395017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241359783486772:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.394836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358466738188:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6735
2025-12-24T02:11:29.394937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241360820662443:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.397627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241360820662395:2081] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.397641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.397650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241360820662395:2081] [node 5] ICP22 created new session: [1:7587241360820662456:2081]
2025-12-24T02:11:29.397658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241360820662456:2081] [node 5] ICS09 handshake done sender: [1:7587241360820662443:2118] self: [1:7587241360820662454:0] peer: [5:7587241358834554257:0] socket: 221 qp: -1
2025-12-24T02:11:29.397667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241360820662456:2081] [node 5] ICS10 traffic start
2025-12-24T02:11:29.397683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.395431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358466738171:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.395493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241358466738084:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.395509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.395515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358466738084:2049] [node 2] ICP22 created new session: [3:7587241358466738189:2049]
2025-12-24T02:11:29.395522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358466738189:2049] [node 2] ICS09 handshake done sender: [3:7587241358466738171:2120] self: [3:7587241358466738181:0] peer: [2:7587241362346319979:0] socket: 226 qp: -1
2025-12-24T02:11:29.395527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358466738189:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.395533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.399001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359783486779:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.399049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241359783486698:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:29.399088Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241362294892590:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241362294892674:2119] held: no
2025-12-24T02:11:29.399270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892684:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.403528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241362294892674:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.403654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241362294892590:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.403672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.403679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241362294892590:2050] [node 3] ICP22 created new session: [6:7587241362294892686:2050]
2025-12-24T02:11:29.403685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362294892686:2050] [node 3] ICS09 handshake done sender: [6:7587241362294892674:2119] self: [6:7587241362294892677:0] peer: [3:7587241358466738168:0] socket: 233 qp: -1
2025-12-24T02:11:29.403691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362294892686:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.403701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.406795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241362346320017:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.406816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359783486765:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.407094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241358834554246:2083] [node 6] ICP35 event processing took too much time PendingConnection:279(39839039ns) {HandleHandshakeStatus:329(39838162ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(311ns) SwitchToState:179(92ns) ProcessPendingSessionEvents:495(24089986ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24058376ns max# 23957666ns}}}
2025-12-24T02:11:29.407288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241358834554246:2083] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.406959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241359783486759:2092] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.406976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.406983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359783486759:2092] [node 5] ICP22 created new session: [4:7587241359783486789:2092]
2025-12-24T02:11:29.406989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359783486789:2092] [node 5] ICS09 handshake done sender: [4:7587241359783486765:2110] self: [4:7587241359783486764:0] peer: [5:7587241358834554272:0] socket: 211 qp: -1
2025-12-24T02:11:29.406994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359783486789:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:29.407013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.410739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241358834554247:2096] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63536) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241358834554256:2113] held: no
2025-12-24T02:11:29.410834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241362346319972:2081] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.410856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.410864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241362346319972:2081] [node 4] ICP22 created new session: [2:7587241362346320032:2081]
2025-12-24T02:11:29.410871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241362346320032:2081] [node 4] ICS09 handshake done sender: [2:7587241362346320017:2119] self: [2:7587241362346320021:0] peer: [4:7587241359783486761:0] socket: 227 qp: -1
2025-12-24T02:11:29.410878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241362346320032:2081] [node 4] ICS10 traffic start
2025-12-24T02:11:29.410902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.414829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241362294892589:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241362294892663:2114] held: no
2025-12-24T02:11:29.415116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241362346320008:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.415839Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241362346319973:2083] [node 6] ICP35 event processing took too much time PendingConnection:279(28635694ns) {HandleHandshakeStatus:329(28635210ns) {DropIncomingHandshake:442(2328ns) DropOutgoingHandshake:457(70ns) SwitchToState:179(180ns) ProcessPendingSessionEvents:495(28589292ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28570528ns max# 28488449ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.417641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892688:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.417815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241362346319970:2076] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.417839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.417846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241362346319970:2076] [node 5] ICP22 created new session: [2:7587241362346320035:2076]
2025-12-24T02:11:29.417853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241362346320035:2076] [node 5] ICS09 handshake done sender: [2:7587241362346320008:2113] self: [2:7587241362346320005:0] peer: [5:7587241358834554278:0] socket: 206 qp: -1
2025-12-24T02:11:29.417859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241362346320035:2076] [node 5] ICS10 traffic start
2025-12-24T02:11:29.417868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.418605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241362294892681:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.416958Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241358466738136:2086] [node 6] ICP35 event processing took too much time PendingConnection:279(37081430ns) {HandleHandshakeStatus:329(37080916ns) {DropIncomingHandshake:442(2671ns) DropOutgoingHandshake:457(219ns) SwitchToState:179(54ns) ProcessPendingSessionEvents:495(21786927ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21765149ns max# 21682986ns}}}
2025-12-24T02:11:29.418441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359783486772:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:29.418666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241362294892657:2079] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.418680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.418684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241362294892657:2079] [node 4] ICP22 created new session: [6:7587241362294892689:2079]
2025-12-24T02:11:29.418689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362294892689:2079] [node 4] ICS09 handshake done sender: [6:7587241362294892681:2121] self: [6:7587241362294892682:0] peer: [4:7587241359783486762:0] socket: 243 qp: -1
2025-12-24T02:11:29.418709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362294892689:2079] [node 4] ICS10 traffic start
2025-12-24T02:11:29.418716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.419053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362294892685:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47712
2025-12-24T02:11:29.419168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241359783486760:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.419183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.419189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359783486760:2080] [node 6] ICP22 created new session: [4:7587241359783486792:2080]
2025-12-24T02:11:29.419195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359783486792:2080] [node 6] ICS09 handshake done sender: [4:7587241359783486772:2114] self: [4:7587241359783486762:0] peer: [6:7587241362294892682:0] socket: 175 qp: -1
2025-12-24T02:11:29.419202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359783486792:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:29.419211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.423505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241362294892663:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.424059Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241358466738084:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(28449573ns) {HandleHandshakeStatus:329(28449379ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(1368ns) SwitchToState:179(208ns) ProcessPendingSessionEvents:495(28366459ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28341494ns max# 28263985ns}}}
2025-12-24T02:11:29.426800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358466738170:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.426814Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358466738170:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.428122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358466738170:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.428213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241358466738119:2079] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.428226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358466738113:2076] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.428241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.428248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358466738113:2076] [node 4] ICP22 created new session: [3:7587241358466738192:2076]
2025-12-24T02:11:29.428255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358466738192:2076] [node 4] ICS09 handshake done sender: [3:7587241358466738170:2119] self: [3:7587241358466738160:0] peer: [4:7587241359783486787:0] socket: 176 qp: -1
2025-12-24T02:11:29.428262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358466738192:2076] [node 4] ICS10 traffic start
2025-12-24T02:11:29.428269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.429268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241359783486698:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20250) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241359783486779:2117] held: no
2025-12-24T02:11:29.427029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241362294892589:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.427048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.427055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241362294892589:2049] [node 2] ICP22 created new session: [6:7587241362294892691:2049]
2025-12-24T02:11:29.427061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362294892691:2049] [node 2] ICS09 handshake done sender: [6:7587241362294892663:2114] self: [6:7587241362294892673:0] peer: [2:7587241362346319987:0] socket: 207 qp: -1
2025-12-24T02:11:29.427067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362294892691:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.427077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.430866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241358834554184:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241358834554255:2112] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.431299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554279:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
2025-12-24T02:11:29.431689Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358834554256:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.431745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358834554247:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.431757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.431764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358834554247:2096] [node 4] ICP22 created new session: [5:7587241358834554281:2096]
2025-12-24T02:11:29.431770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358834554281:2096] [node 4] ICS09 handshake done sender: [5:7587241358834554256:2113] self: [5:7587241358834554272:0] peer: [4:7587241359783486764:0] socket: 218 qp: -1
2025-12-24T02:11:29.431776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358834554281:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:29.431783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.432033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358834554280:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51195
2025-12-24T02:11:29.432197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358834554255:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.432331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358834554184:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.432341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.432345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358834554184:2049] [node 2] ICP22 created new session: [5:7587241358834554283:2049]
2025-12-24T02:11:29.432350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358834554283:2049] [node 2] ICS09 handshake done sender: [5:7587241358834554255:2112] self: [5:7587241358834554278:0] peer: [2:7587241362346320005:0] socket: 210 qp: -1
2025-12-24T02:11:29.432354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358834554283:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.432360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.435990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241359783486760:2080] [node 6] ICP35 event processing took too much time PendingConnection:279(16717306ns) {HandleHandshakeStatus:329(16716767ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(122ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(11170970ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11149875ns max# 11076793ns}}}
2025-12-24T02:11:29.436017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486794:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
2025-12-24T02:11:29.436179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359783486795:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12536
2025-12-24T02:11:29.439104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241359783486760:2080] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.439124Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359783486779:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.439294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241359783486698:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.439317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.439323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359783486698:2050] [node 3] ICP22 created new session: [4:7587241359783486796:2050]
2025-12-24T02:11:29.439334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359783486796:2050] [node 3] ICS09 handshake done sender: [4:7587241359783486779:2117] self: [4:7587241359783486787:0] peer: [3:7587241358466738160:0] socket: 237 qp: -1
2025-12-24T02:11:29.439340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359783486796:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.439350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.459805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:29.459919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241362346320018:2048] [node 1] ICS01 socket: 194 reason# EndOfStream
2025-12-24T02:11:29.459930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241362346319931:2048] [node 1] ICP30 unregister session Session# [2:7587241362346320018:2048] VirtualId# [2:7587241362346320013:0]
2025-12-24T02:11:29.459939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241362346320018:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.459974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.460004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.460467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:29.460535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241358466738174:2048] [node 1] ICS01 socket: 200 reason# EndOfStream
2025-12-24T02:11:29.460540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241358466738083:2048] [node 1] ICP30 unregister session Session# [3:7587241358466738174:2048] VirtualId# [3:7587241358466738172:0]
2025-12-24T02:11:29.460545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241358466738174:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.460568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.460584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.469875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:29.469957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358834554275:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:11:29.469963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358834554183:2048] [node 1] ICP30 unregister session Session# [5:7587241358834554275:2048] VirtualId# [5:7587241358834554257:0]
2025-12-24T02:11:29.469971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358834554275:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.470010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.470034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.469808Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:29.469910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362294892671:2048] [node 1] ICS01 socket: 177 reason# EndOfStream
2025-12-24T02:11:29.469922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241362294892588:2048] [node 1] ICP30 unregister session Session# [6:7587241362294892671:2048] VirtualId# [6:7587241362294892669:0]
2025-12-24T02:11:29.469929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362294892671:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.469986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.470012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.478932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:29.479027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241359783486775:2048] [node 1] ICS01 socket: 216 reason# EndOfStream
2025-12-24T02:11:29.479036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241359783486696:2048] [node 1] ICP30 unregister session Session# [4:7587241359783486775:2048] VirtualId# [4:7587241359783486773:0]
2025-12-24T02:11:29.479043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241359783486775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.479086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:29.479112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:29.498819Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:29.498936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241358466738189:2049] [node 2] ICS07 socket disconnect 226 reason# EndOfStream
2025-12-24T02:11:29.498944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241358466738189:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.499006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.499027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241358466738189:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:29.509380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241358466738084:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:29.509402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241358466738189:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:29.513731Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:29.513825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241359783486783:2049] [node 2] ICS01 socket: 217 reason# EndOfStream
2025-12-24T02:11:29.513832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241359783486697:2049] [node 2] ICP30 unregister session Session# [4:7587241359783486783:2049] VirtualId# [4:7587241359783486761:0]
2025-12-24T02:11:29.513839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241359783486783:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.513876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.513899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.514485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241358466738084:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.514503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241358466738189:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.514508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241358466738084:2049] [node 2] ICP30 unregister session Session# [3:7587241358466738189:2049] VirtualId# [3:7587241358466738181:0]
2025-12-24T02:11:29.514564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.514571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241358466738084:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.514574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9374) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.514943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:29.515020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362294892691:2049] [node 2] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:11:29.515027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241362294892589:2049] [node 2] ICP30 unregister session Session# [6:7587241362294892691:2049] VirtualId# [6:7587241362294892673:0]
2025-12-24T02:11:29.515033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362294892691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.515074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.515094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.514578Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:29.514655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358834554283:2049] [node 2] ICS01 socket: 210 reason# EndOfStream
2025-12-24T02:11:29.514663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358834554184:2049] [node 2] ICP30 unregister session Session# [5:7587241358834554283:2049] VirtualId# [5:7587241358834554278:0]
2025-12-24T02:11:29.514669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358834554283:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.514755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:29.514780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:29.552524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:29.552628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362294892686:2050] [node 3] ICS01 socket: 233 reason# EndOfStream
2025-12-24T02:11:29.552639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241362294892590:2050] [node 3] ICP30 unregister session Session# [6:7587241362294892686:2050] VirtualId# [6:7587241362294892677:0]
2025-12-24T02:11:29.552647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362294892686:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.552703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:29.552726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:29.552823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:29.552895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241358834554267:2050] [node 3] ICS07 socket disconnect 198 reason# EndOfStream
2025-12-24T02:11:29.552900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358834554267:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.552927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:29.552947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241358834554267:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:29.553645Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241358834554185:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20250) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:29.553658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358834554267:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:29.553663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358834554185:2050] [node 3] ICP30 unregister session Session# [5:7587241358834554267:2050] VirtualId# [5:7587241358834554263:0]
2025-12-24T02:11:29.553721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:29.553729Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241358834554185:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.553733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20250) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:29.562752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241358834554185:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:29.590932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:29.591041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362294892689:2079] [node 4] ICS01 socket: 243 reason# EndOfStream
2025-12-24T02:11:29.591051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241362294892657:2079] [node 4] ICP30 unregister session Session# [6:7587241362294892689:2079] VirtualId# [6:7587241362294892682:0]
2025-12-24T02:11:29.591060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362294892689:2079] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:29.591109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:29.591137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:29.666896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241358657448710:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:29.666911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241358657448711:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:29.678802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358887707942:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.678820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358887707943:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.678783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241361899172475:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.678800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241361899172476:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:29.670808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359558365896:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.670817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359558365897:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.670822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359558365898:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:29.690359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358632311890:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.690381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358632311891:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.690387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358632311892:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:29.718616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241358358133222:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:29.718669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241358358133223:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:29.718676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241358358133224:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.780016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241358657448775:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:29.780141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241358657448776:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:29.782948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172535:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.782602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241358358133241:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:29.782747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241358358133243:2061] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.784270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708002:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:29.785359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708003:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:29.791086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358632311937:2084] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.807018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241358632311958:2088] [node 6] ICP01 ready to work
2025-12-24T02:11:29.809098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311955:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.815072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365957:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.815100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365958:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.823047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358887708005:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:29.823164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311956:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:29.830959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241361899172543:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:29.831035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241358657448788:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.847074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365959:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.847104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365960:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.859043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708002:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.859323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358887708016:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:29.859453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241358887708017:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.879316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133300:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.882831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311955:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.886554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241358358133300:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.886612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241358358133243:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.890162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311962:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.890181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241358632311962:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.894466Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358632311962:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.898722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133302:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.899021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241358358133300:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.898746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241358657448786:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51809
2025-12-24T02:11:29.898881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241358657448787:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51809
2025-12-24T02:11:29.902316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241358657448791:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51809
2025-12-24T02:11:29.902321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241358358133243:2061] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.902343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.902352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241358358133243:2061] [node 5] ICP22 created new session: [6:7587241358358133303:2061]
2025-12-24T02:11:29.902359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241358358133303:2061] [node 5] ICS09 handshake done sender: [6:7587241358358133300:2115] self: [6:7587241358358133301:0] peer: [5:7587241358632311961:0] socket: 220 qp: -1
2025-12-24T02:11:29.902370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241358358133303:2061] [node 5] ICS10 traffic start
2025-12-24T02:11:29.902383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.895376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359558365968:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:29.902190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172535:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.895533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365957:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.902387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241361899172548:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:29.902544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241361899172549:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:29.902872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311956:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.902891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241358632311958:2088] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.902909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.902917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358632311958:2088] [node 6] ICP22 created new session: [5:7587241358632311968:2088]
2025-12-24T02:11:29.902923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358632311968:2088] [node 6] ICS09 handshake done sender: [5:7587241358632311962:2113] self: [5:7587241358632311961:0] peer: [6:7587241358358133301:0] socket: 207 qp: -1
2025-12-24T02:11:29.902928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358632311968:2088] [node 6] ICS10 traffic start
2025-12-24T02:11:29.902935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.903550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448786:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.908788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448787:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.908446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708014:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:29.908495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708015:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:29.920212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358632311890:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.934650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133305:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.932362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365958:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.932398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365959:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.933398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241359558365974:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:29.933688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241359558365896:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.938736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448780:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.938760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241358657448780:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.939265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448778:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.939268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241358657448778:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.939785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241358657448794:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51809
2025-12-24T02:11:29.939818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241358657448795:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51809
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.940388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365960:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.941335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241359558365898:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.941395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365981:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.940075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708003:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.942779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708015:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.943048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241358632311891:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20124636ns) {Configure:104(20123707ns) {ProcessConfigured:135(19739954ns) {StartInitialHandshake:157(19739209ns) {DropHandshakes:470(1222ns) {DropIncomingHandshake:442(248ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(1749ns) {SwitchToState:179(914ns)}}}}}
2025-12-24T02:11:29.943641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311976:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:29.943704Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311977:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:29.943849Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241361899172549:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24969966ns) {Configure:104(24969206ns) {ProcessConfigured:135(24523442ns) {StartInitialHandshake:157(24522534ns) {DropHandshakes:470(1240ns) {DropIncomingHandshake:442(225ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(526ns) PrepareNewSessionHandshake:477(1744ns) {SwitchToState:179(1185ns)}}}}}
2025-12-24T02:11:29.946767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241358657448780:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:29.946892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708009:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.946903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358887708009:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.947498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358887708009:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.950925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241361899172475:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.954902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311976:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.954991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358632311956:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.957067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358632311890:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.957085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.957091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358632311890:2048] [node 1] ICP22 created new session: [5:7587241358632311978:2048]
2025-12-24T02:11:29.957096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241358632311978:2048] [node 1] ICS09 handshake done sender: [5:7587241358632311956:2110] self: [5:7587241358632311972:0] peer: [1:7587241358657448779:0] socket: 204 qp: -1
2025-12-24T02:11:29.957101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241358632311978:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.957107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.957270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358632311937:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.955061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358887708005:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.955089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.955099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358887708005:2109] [node 4] ICP22 created new session: [3:7587241358887708029:2109]
2025-12-24T02:11:29.955105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358887708029:2109] [node 4] ICS09 handshake done sender: [3:7587241358887708009:2111] self: [3:7587241358887708008:0] peer: [4:7587241359558365982:0] socket: 209 qp: -1
2025-12-24T02:11:29.955112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358887708029:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:29.955123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:29.955395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708014:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.956044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365983:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.956165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359558365960:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.956207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241359558365898:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.956217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.956222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359558365898:2050] [node 3] ICP22 created new session: [4:7587241359558365984:2050]
2025-12-24T02:11:29.956227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359558365984:2050] [node 3] ICS09 handshake done sender: [4:7587241359558365960:2111] self: [4:7587241359558365982:0] peer: [3:7587241358887708008:0] socket: 214 qp: -1
2025-12-24T02:11:29.956232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359558365984:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.956238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.958925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241358657448778:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:29.958895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241358887707943:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.955908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133307:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.959060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241358358133305:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.959094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241358358133224:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.969773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448791:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.969825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448794:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.969838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241358657448776:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.969855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:29.969869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241358657448776:2109] [node 5] ICP22 created new session: [1:7587241358657448798:2109]
2025-12-24T02:11:29.969878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241358657448798:2109] [node 5] ICS09 handshake done sender: [1:7587241358657448780:2111] self: [1:7587241358657448779:0] peer: [5:7587241358632311972:0] socket: 195 qp: -1
2025-12-24T02:11:29.969887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241358657448798:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:29.969899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:29.970380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448771:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.970385Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241358657448771:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.970915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241358657448775:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.974719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708019:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.974742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358887708019:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.970938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:29.970944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241358657448775:2108] [node 4] ICP22 created new session: [1:7587241358657448800:2108]
2025-12-24T02:11:29.970951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241358657448800:2108] [node 4] ICS09 handshake done sender: [1:7587241358657448778:2110] self: [1:7587241358657448777:0] peer: [4:7587241359558365978:0] socket: 194 qp: -1
2025-12-24T02:11:29.970958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241358657448800:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:29.970967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.977781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133309:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.977751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172540:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.977765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241361899172540:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.978148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172557:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-12-24T02:11:29.978163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172558:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-12-24T02:11:29.978265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172556:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-12-24T02:11:29.977412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359558365958:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:29.977337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241358657448776:2109] [node 5] ICP18 handshake (actor [1:7587241358657448794:2118]) is too old
2025-12-24T02:11:29.977417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241358657448775:2108] [node 4] ICP18 handshake (actor [1:7587241358657448791:2117]) is too old
2025-12-24T02:11:29.979263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241361899172540:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.979405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358887708019:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:29.980835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241358358133305:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:29.981227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241358358133224:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.981245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.981252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241358358133224:2050] [node 3] ICP22 created new session: [6:7587241358358133311:2050]
2025-12-24T02:11:29.981258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241358358133311:2050] [node 3] ICS09 handshake done sender: [6:7587241358358133305:2117] self: [6:7587241358358133308:0] peer: [3:7587241358887708018:0] socket: 192 qp: -1
2025-12-24T02:11:29.981264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241358358133311:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:29.981271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:29.979081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365971:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.979094Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241359558365971:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.979521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241359558365896:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.979536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.979544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359558365896:2048] [node 1] ICP22 created new session: [4:7587241359558365986:2048]
2025-12-24T02:11:29.979551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241359558365986:2048] [node 1] ICS09 handshake done sender: [4:7587241359558365958:2109] self: [4:7587241359558365978:0] peer: [1:7587241358657448777:0] socket: 203 qp: -1
2025-12-24T02:11:29.979558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241359558365986:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.979566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.981101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241358657448771:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.982234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448795:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.982458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241358657448710:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.982482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.982488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241358657448710:2048] [node 2] ICP22 created new session: [1:7587241358657448805:2048]
2025-12-24T02:11:29.982494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241358657448805:2048] [node 2] ICS09 handshake done sender: [1:7587241358657448771:2107] self: [1:7587241358657448770:0] peer: [2:7587241361899172554:0] socket: 186 qp: -1
2025-12-24T02:11:29.982500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241358657448805:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:29.982506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.983021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241361899172476:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.983036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:29.983043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241361899172476:2049] [node 3] ICP22 created new session: [2:7587241361899172560:2049]
2025-12-24T02:11:29.983049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241361899172560:2049] [node 3] ICS09 handshake done sender: [2:7587241361899172540:2109] self: [2:7587241361899172539:0] peer: [3:7587241358887708032:0] socket: 210 qp: -1
2025-12-24T02:11:29.983054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241361899172560:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:29.983062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.984092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.987852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241358358133307:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.987889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241358358133223:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.987622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358887708017:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.987643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.987649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358887708017:2113] [node 6] ICP22 created new session: [3:7587241358887708036:2113]
2025-12-24T02:11:29.987654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358887708036:2113] [node 6] ICS09 handshake done sender: [3:7587241358887708019:2116] self: [3:7587241358887708018:0] peer: [6:7587241358358133308:0] socket: 224 qp: -1
2025-12-24T02:11:29.987658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358887708036:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:29.987666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.988353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241361899172535:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.988486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311982:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.988739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708034:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.988806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241361899172475:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25034) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241361899172535:2107] held: no
2025-12-24T02:11:29.988898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241361899172475:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.988905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:29.988910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241361899172475:2048] [node 1] ICP22 created new session: [2:7587241361899172564:2048]
2025-12-24T02:11:29.988914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241361899172564:2048] [node 1] ICS09 handshake done sender: [2:7587241361899172535:2107] self: [2:7587241361899172554:0] peer: [1:7587241358657448770:0] socket: 188 qp: -1
2025-12-24T02:11:29.988920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241361899172564:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:29.988926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:29.989109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172557:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.989131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172556:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.989155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172547:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.989158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241361899172547:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:29.989852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172562:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
2025-12-24T02:11:29.989948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241361899172476:2049] [node 3] ICP18 handshake (actor [2:7587241361899172556:2116]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.990248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311983:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.990391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241358887708035:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:29.990485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358887708014:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.990578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241358632311984:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:29.988202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241359558365896:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:29.990646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241358887707943:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.990658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.990662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358887707943:2049] [node 2] ICP22 created new session: [3:7587241358887708039:2049]
2025-12-24T02:11:29.990668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241358887708039:2049] [node 2] ICS09 handshake done sender: [3:7587241358887708014:2114] self: [3:7587241358887708032:0] peer: [2:7587241361899172539:0] socket: 218 qp: -1
2025-12-24T02:11:29.990672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241358887708039:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.990678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:29.992950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241361899172547:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.993947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133315:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.993965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241358358133316:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:29.994123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241358358133307:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:29.994183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241358358133223:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:29.994195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:29.994201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241358358133223:2049] [node 2] ICP22 created new session: [6:7587241358358133317:2049]
2025-12-24T02:11:29.994205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241358358133317:2049] [node 2] ICS09 handshake done sender: [6:7587241358358133307:2118] self: [6:7587241358358133314:0] peer: [2:7587241361899172546:0] socket: 237 qp: -1
2025-12-24T02:11:29.994211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241358358133317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:29.994218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.993276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241361899172543:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:29.993287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:29.993293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241361899172543:2110] [node 6] ICP22 created new session: [2:7587241361899172570:2110]
2025-12-24T02:11:29.993298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241361899172570:2110] [node 6] ICS09 handshake done sender: [2:7587241361899172547:2113] self: [2:7587241361899172546:0] peer: [6:7587241358358133314:0] socket: 236 qp: -1
2025-12-24T02:11:29.993302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241361899172570:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:29.993309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:29.994679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241361899172567:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35616
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:29.995298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708035:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.995320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241358887707942:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:29.995452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172562:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:29.995525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241361899172543:2110] [node 6] ICP18 handshake (actor [2:7587241361899172562:2119]) is too old
2025-12-24T02:11:29.994953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241359558365991:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:29.995021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359558365971:2116] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.001015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241363182675338:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64784
2025-12-24T02:11:30.000397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448782:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.000409Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241358657448782:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.002848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365991:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.002878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241359558365897:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.000187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311984:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.000204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358632311976:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.004165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172567:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.004197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172551:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.004201Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241361899172551:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.005143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241358657448782:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.005246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241358657448711:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.005268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.005281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241358657448711:2049] [node 3] ICP22 created new session: [1:7587241362952416105:2049]
2025-12-24T02:11:30.005287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241362952416105:2049] [node 3] ICS09 handshake done sender: [1:7587241358657448782:2112] self: [1:7587241358657448781:0] peer: [3:7587241358887708041:0] socket: 223 qp: -1
2025-12-24T02:11:30.005292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241362952416105:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:30.005299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.005287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358632311937:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.005306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.005312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358632311937:2084] [node 4] ICP22 created new session: [5:7587241362927279281:2084]
2025-12-24T02:11:30.005318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241362927279281:2084] [node 4] ICS09 handshake done sender: [5:7587241358632311976:2117] self: [5:7587241358632311980:0] peer: [4:7587241359558365970:0] socket: 232 qp: -1
2025-12-24T02:11:30.005325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241362927279281:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:30.005333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.006408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358887708035:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.004731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241359558365968:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.004751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.004758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359558365968:2114] [node 5] ICP22 created new session: [4:7587241363853333289:2114]
2025-12-24T02:11:30.004765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241363853333289:2114] [node 5] ICS09 handshake done sender: [4:7587241359558365971:2116] self: [4:7587241359558365970:0] peer: [5:7587241358632311980:0] socket: 225 qp: -1
2025-12-24T02:11:30.004771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241363853333289:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:30.004780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.006470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241358887707942:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.006485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.006490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358887707942:2048] [node 1] ICP22 created new session: [3:7587241363182675339:2048]
2025-12-24T02:11:30.006495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241363182675339:2048] [node 1] ICS09 handshake done sender: [3:7587241358887708035:2120] self: [3:7587241358887708041:0] peer: [1:7587241358657448781:0] socket: 246 qp: -1
2025-12-24T02:11:30.006501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241363182675339:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.006507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.007144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241358358133310:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.012566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358632311892:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.012993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241363853333291:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19701
2025-12-24T02:11:30.013165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241361899172551:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.013394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241358632311983:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.013418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241358632311891:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.014305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241361899172549:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.014321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.014332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241361899172549:2112] [node 4] ICP22 created new session: [2:7587241366194139870:2112]
2025-12-24T02:11:30.014337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241366194139870:2112] [node 4] ICS09 handshake done sender: [2:7587241361899172551:2114] self: [2:7587241361899172550:0] peer: [4:7587241363853333288:0] socket: 197 qp: -1
2025-12-24T02:11:30.014342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241366194139870:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:30.014348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.014978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241358887708023:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.014989Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241358887708023:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.016123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241358358133315:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.016158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241358358133222:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.016209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241358358133241:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.016497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241358887708023:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.016572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241358887708016:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.016588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.016593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241358887708016:2112] [node 5] ICP22 created new session: [3:7587241363182675342:2112]
2025-12-24T02:11:30.016598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241363182675342:2112] [node 5] ICS09 handshake done sender: [3:7587241358887708023:2118] self: [3:7587241358887708022:0] peer: [5:7587241362927279283:0] socket: 253 qp: -1
2025-12-24T02:11:30.016603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241363182675342:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:30.016610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.017055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241358887707942:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:30.013343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359558365991:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.013509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241359558365897:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.013520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.013526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359558365897:2049] [node 2] ICP22 created new session: [4:7587241363853333292:2049]
2025-12-24T02:11:30.013531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241363853333292:2049] [node 2] ICS09 handshake done sender: [4:7587241359558365991:2121] self: [4:7587241363853333288:0] peer: [2:7587241361899172550:0] socket: 255 qp: -1
2025-12-24T02:11:30.013537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241363853333292:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.013544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.019072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241361899172553:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.019080Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241361899172553:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.020450Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241358657448790:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.020461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241358657448790:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.022159Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241358657448790:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.022291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241358632311892:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4349) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241358632311984:2121] held: no
2025-12-24T02:11:30.022451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241362927279285:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:30.020985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241359558365976:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.020994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241359558365976:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.021546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241359558365976:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.021765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241359558365974:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.021777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.021782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241359558365974:2115] [node 6] ICP22 created new session: [4:7587241363853333295:2115]
2025-12-24T02:11:30.021788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241363853333295:2115] [node 6] ICS09 handshake done sender: [4:7587241359558365976:2118] self: [4:7587241359558365975:0] peer: [6:7587241362653100616:0] socket: 248 qp: -1
2025-12-24T02:11:30.021792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241363853333295:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:30.021800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.022523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241358657448788:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.022537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.022542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241358657448788:2113] [node 6] ICP22 created new session: [1:7587241362952416108:2113]
2025-12-24T02:11:30.022547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241362952416108:2113] [node 6] ICS09 handshake done sender: [1:7587241358657448790:2116] self: [1:7587241358657448789:0] peer: [6:7587241362653100615:0] socket: 227 qp: -1
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.022904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362653100617:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:30.022552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241362952416108:2113] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.022560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.022996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241362927279286:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14622
2025-12-24T02:11:30.023043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358632311984:2121] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.023108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241362653100618:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33860
2025-12-24T02:11:30.023147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241361899172553:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.023243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241358358133310:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.023365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241361899172548:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.023380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.023386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241361899172548:2111] [node 5] ICP22 created new session: [2:7587241366194139873:2111]
2025-12-24T02:11:30.023391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241366194139873:2111] [node 5] ICS09 handshake done sender: [2:7587241361899172553:2115] self: [2:7587241361899172552:0] peer: [5:7587241362927279284:0] socket: 251 qp: -1
2025-12-24T02:11:30.023397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241366194139873:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:30.023404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.024061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358632311892:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.024076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.024081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358632311892:2050] [node 3] ICP22 created new session: [5:7587241362927279287:2050]
2025-12-24T02:11:30.024087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241362927279287:2050] [node 3] ICS09 handshake done sender: [5:7587241358632311984:2121] self: [5:7587241362927279283:0] peer: [3:7587241358887708022:0] socket: 254 qp: -1
2025-12-24T02:11:30.024092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241362927279287:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.024099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.024379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241358632311983:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.024383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241358358133315:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.025658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241358358133222:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.025673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.025679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241358358133222:2048] [node 1] ICP22 created new session: [6:7587241362653100619:2048]
2025-12-24T02:11:30.025685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362653100619:2048] [node 1] ICS09 handshake done sender: [6:7587241358358133310:2120] self: [6:7587241362653100615:0] peer: [1:7587241358657448789:0] socket: 244 qp: -1
2025-12-24T02:11:30.025690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362653100619:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.025697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.025843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241358358133222:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:30.026031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241358358133241:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.026038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.026043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241358358133241:2059] [node 4] ICP22 created new session: [6:7587241362653100621:2059]
2025-12-24T02:11:30.026049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241362653100621:2059] [node 4] ICS09 handshake done sender: [6:7587241358358133315:2121] self: [6:7587241362653100616:0] peer: [4:7587241359558365975:0] socket: 219 qp: -1
2025-12-24T02:11:30.026053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241362653100621:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:30.026059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.026175Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241358358133241:2059] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:30.027727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241358632311891:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.027742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.027748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241358632311891:2049] [node 2] ICP22 created new session: [5:7587241362927279289:2049]
2025-12-24T02:11:30.027753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241362927279289:2049] [node 2] ICS09 handshake done sender: [5:7587241358632311983:2120] self: [5:7587241362927279284:0] peer: [2:7587241361899172552:0] socket: 252 qp: -1
2025-12-24T02:11:30.027759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241362927279289:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.027766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.027900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241358632311891:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:30.038824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:30.038931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241361899172564:2048] [node 1] ICS01 socket: 188 reason# EndOfStream
2025-12-24T02:11:30.038941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241361899172475:2048] [node 1] ICP30 unregister session Session# [2:7587241361899172564:2048] VirtualId# [2:7587241361899172554:0]
2025-12-24T02:11:30.038949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241361899172564:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.038992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.039015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.042765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:30.042857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241363182675339:2048] [node 1] ICS01 socket: 246 reason# EndOfStream
2025-12-24T02:11:30.042866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241358887707942:2048] [node 1] ICP30 unregister session Session# [3:7587241363182675339:2048] VirtualId# [3:7587241358887708041:0]
2025-12-24T02:11:30.042874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241363182675339:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.042939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.042968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.045432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:30.045495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241358632311978:2048] [node 1] ICS07 socket disconnect 204 reason# EndOfStream
2025-12-24T02:11:30.045499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241358632311978:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.045522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.045541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241358632311978:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.045234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:30.045318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241359558365986:2048] [node 1] ICS01 socket: 203 reason# EndOfStream
2025-12-24T02:11:30.045326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241359558365896:2048] [node 1] ICP30 unregister session Session# [4:7587241359558365986:2048] VirtualId# [4:7587241359558365978:0]
2025-12-24T02:11:30.045332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241359558365986:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.045374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.045396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.046756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:30.046839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362653100619:2048] [node 1] ICS01 socket: 244 reason# EndOfStream
2025-12-24T02:11:30.046846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241358358133222:2048] [node 1] ICP30 unregister session Session# [6:7587241362653100619:2048] VirtualId# [6:7587241362653100615:0]
2025-12-24T02:11:30.046852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362653100619:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.046904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.046929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.050898Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241358632311890:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25034) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.050916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241358632311978:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.050922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358632311890:2048] [node 1] ICP30 unregister session Session# [5:7587241358632311978:2048] VirtualId# [5:7587241358632311972:0]
2025-12-24T02:11:30.050960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.050966Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241358632311890:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.050970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25034) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.052276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:30.052356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241358887708039:2049] [node 2] ICS01 socket: 218 reason# EndOfStream
2025-12-24T02:11:30.052362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241358887707943:2049] [node 2] ICP30 unregister session Session# [3:7587241358887708039:2049] VirtualId# [3:7587241358887708032:0]
2025-12-24T02:11:30.052367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241358887708039:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.052400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.052418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.062003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:30.062080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241363853333292:2049] [node 2] ICS01 socket: 255 reason# EndOfStream
2025-12-24T02:11:30.062090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241359558365897:2049] [node 2] ICP30 unregister session Session# [4:7587241363853333292:2049] VirtualId# [4:7587241363853333288:0]
2025-12-24T02:11:30.062095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241363853333292:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.062119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.061777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:30.061874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241362927279289:2049] [node 2] ICS01 socket: 252 reason# EndOfStream
2025-12-24T02:11:30.061883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358632311891:2049] [node 2] ICP30 unregister session Session# [5:7587241362927279289:2049] VirtualId# [5:7587241362927279284:0]
2025-12-24T02:11:30.061890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241362927279289:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.061937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.061965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.061764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:30.061867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241358358133317:2049] [node 2] ICS07 socket disconnect 237 reason# EndOfStream
2025-12-24T02:11:30.061876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241358358133317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.061933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.061955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241358358133317:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.062428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241358358133223:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8331) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.062443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241358358133317:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.062448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241358358133223:2049] [node 2] ICP30 unregister session Session# [6:7587241358358133317:2049] VirtualId# [6:7587241358358133314:0]
2025-12-24T02:11:30.062501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.062508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241358358133223:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.062511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8331) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.062143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.066740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241358632311890:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:30.074756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241358358133223:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:30.078793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:30.078903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241362927279287:2050] [node 3] ICS01 socket: 254 reason# EndOfStream
2025-12-24T02:11:30.078914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241358632311892:2050] [node 3] ICP30 unregister session Session# [5:7587241362927279287:2050] VirtualId# [5:7587241362927279283:0]
2025-12-24T02:11:30.078922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241362927279287:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.078980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:30.079007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:30.082739Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:30.082829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241358358133311:2050] [node 3] ICS01 socket: 192 reason# EndOfStream
2025-12-24T02:11:30.082837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241358358133224:2050] [node 3] ICP30 unregister session Session# [6:7587241358358133311:2050] VirtualId# [6:7587241358358133308:0]
2025-12-24T02:11:30.082844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241358358133311:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.082903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:30.082928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:30.097889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:30.097997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241362653100621:2059] [node 4] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:11:30.098009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241358358133241:2059] [node 4] ICP30 unregister session Session# [6:7587241362653100621:2059] VirtualId# [6:7587241362653100616:0]
2025-12-24T02:11:30.098017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241362653100621:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.098070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:30.098097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:30.138891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241366678093607:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:30.138907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241366678093608:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:30.138863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363700070690:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.138880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363700070691:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:30.142888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241366717750879:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.142902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241366717750880:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.142906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241366717750881:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.150325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241364524854778:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.150345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241364524854779:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.150351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241364524854780:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.146787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241362634981503:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.146803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241362634981504:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.184825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241363701187404:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.184846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241363701187405:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.186715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241363701187403:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.205544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241363701187446:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:30.207066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241363701187456:2093] [node 5] ICP01 ready to work
2025-12-24T02:11:30.206979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241366678093669:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:30.207037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241366678093667:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47692
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.209445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241366717750941:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:30.209501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750940:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
2025-12-24T02:11:30.210222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187469:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.211006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241364524854833:2098] [node 4] ICP01 ready to work
2025-12-24T02:11:30.210907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241366678093674:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:30.218788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241366717750940:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.219809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241362634981565:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:30.219895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241362634981566:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.221357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187477:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.226796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241364524854832:2095] [node 6] ICP01 ready to work
2025-12-24T02:11:30.229594Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854845:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.229612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854846:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.227295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241366678093677:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.230099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241366717750941:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20554588ns) {Configure:104(20554106ns) {ProcessConfigured:135(20159342ns) {StartInitialHandshake:157(20159110ns) {DropHandshakes:470(412ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(79ns) PrepareNewSessionHandshake:477(1728ns) {SwitchToState:179(1476ns)}}}}}
2025-12-24T02:11:30.230249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241366717750947:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:30.230640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241366717750947:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.231977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750949:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
2025-12-24T02:11:30.231394Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981573:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.231439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981574:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.234952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241363701187469:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.243026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241363701187403:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.243125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241363701187477:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.243206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241363701187405:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.243652Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241366678093674:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32675168ns) {Configure:104(32674520ns) {ProcessConfigured:135(32333475ns) {StartInitialHandshake:157(32332953ns) {DropHandshakes:470(403ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(115ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(21ns)}}}}}
2025-12-24T02:11:30.243770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093671:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.243773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241366678093671:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.246882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241363701187450:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.246899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241363701187450:2091] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.247039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981573:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.247107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363700070757:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.253017Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.250823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241366678093671:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.250957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241366678093669:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.250975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.250984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241366678093669:2108] [node 6] ICP22 created new session: [1:7587241366678093688:2108]
2025-12-24T02:11:30.250992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241366678093688:2108] [node 6] ICS09 handshake done sender: [1:7587241366678093671:2109] self: [1:7587241366678093670:0] peer: [6:7587241363701187483:0] socket: 198 qp: -1
2025-12-24T02:11:30.251001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241366678093688:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:30.251029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.250870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363700070755:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:30.251060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363700070758:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.258794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093667:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.261051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981580:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.261093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981581:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.263015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981581:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.263157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981568:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.263165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241362634981568:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.263204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241366717750949:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.263663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241366717750879:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.262850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854846:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.262932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241364524854780:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.263808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854857:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.269693Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750953:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
2025-12-24T02:11:30.269756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750952:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
2025-12-24T02:11:30.268083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363700070766:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59966
2025-12-24T02:11:30.268258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363700070767:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59966
2025-12-24T02:11:30.268331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363700070768:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59966
2025-12-24T02:11:30.267662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187486:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.270802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093679:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.270819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241366678093679:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.270822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241366717750881:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51939734ns) {Configure:104(51938141ns) {ProcessConfigured:135(51617492ns) {StartInitialHandshake:157(51616976ns) {DropHandshakes:470(1031ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(312ns)} GenerateSessionVirtualId:988(676ns) PrepareNewSessionHandshake:477(1075ns) {SwitchToState:179(443ns)}}}}}
2025-12-24T02:11:30.270849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854845:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.274091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241366678093692:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47692
2025-12-24T02:11:30.273452Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241364524854833:2098] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(46522951ns) {Configure:104(46522166ns) {ProcessConfigured:135(46187481ns) {StartInitialHandshake:157(46187006ns) {DropHandshakes:470(602ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(157ns) PrepareNewSessionHandshake:477(976ns) {SwitchToState:179(226ns)}}}}}
2025-12-24T02:11:30.273493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241364524854833:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.270880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241363701187450:2091] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.271184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241363701187446:2089] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.271200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.271210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241363701187446:2089] [node 4] ICP22 created new session: [6:7587241363701187487:2089]
2025-12-24T02:11:30.271217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241363701187487:2089] [node 4] ICS09 handshake done sender: [6:7587241363701187450:2091] self: [6:7587241363701187449:0] peer: [4:7587241366717750948:0] socket: 193 qp: -1
2025-12-24T02:11:30.271225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241363701187487:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:30.271236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.271450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241363701187469:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.271522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241363701187403:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.271534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.271558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241363701187403:2048] [node 1] ICP22 created new session: [6:7587241363701187489:2048]
2025-12-24T02:11:30.271564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241363701187489:2048] [node 1] ICS09 handshake done sender: [6:7587241363701187469:2103] self: [6:7587241363701187483:0] peer: [1:7587241366678093670:0] socket: 201 qp: -1
2025-12-24T02:11:30.271569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241363701187489:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.271587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.270982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981570:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.270996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241362634981570:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.274574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981574:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.274796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241362634981504:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.274539Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070768:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.275904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363700070771:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59966
2025-12-24T02:11:30.274850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241366717750946:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.274862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241366717750946:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.276241Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241363700070755:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24122931ns) {Configure:104(24121302ns) {ProcessConfigured:135(23908190ns) {StartInitialHandshake:157(23907873ns) {DropHandshakes:470(306ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(515ns) PrepareNewSessionHandshake:477(693ns) {SwitchToState:179(32ns)}}}}}
2025-12-24T02:11:30.278654Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070767:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.275000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981580:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.276338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981583:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.276385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241362634981503:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.276502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241366717750953:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.276553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241366717750946:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.280224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070752:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.280235Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363700070752:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.279308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093692:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.279553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241366678093679:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.279634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241366678093677:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.279648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.279677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241366678093677:2111] [node 4] ICP22 created new session: [1:7587241366678093695:2111]
2025-12-24T02:11:30.279684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241366678093695:2111] [node 4] ICS09 handshake done sender: [1:7587241366678093679:2112] self: [1:7587241366678093678:0] peer: [4:7587241366717750951:0] socket: 212 qp: -1
2025-12-24T02:11:30.279691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241366678093695:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:30.279701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.282813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093683:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.282822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241366678093683:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.283808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070766:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.283846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241363700070690:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.282801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241362634981568:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.284288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241366717750941:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.284308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.284317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241366717750941:2107] [node 5] ICP22 created new session: [4:7587241366717750957:2107]
2025-12-24T02:11:30.284323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241366717750957:2107] [node 5] ICS09 handshake done sender: [4:7587241366717750946:2111] self: [4:7587241366717750942:0] peer: [5:7587241364524854861:0] socket: 217 qp: -1
2025-12-24T02:11:30.284331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241366717750957:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:30.284338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.284547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241366717750940:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.286988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854862:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.287158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.287124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750959:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
2025-12-24T02:11:30.287343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241364524854857:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.287425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241364524854833:2098] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.287445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.287450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241364524854833:2098] [node 4] ICP22 created new session: [5:7587241364524854864:2098]
2025-12-24T02:11:30.287456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241364524854864:2098] [node 4] ICS09 handshake done sender: [5:7587241364524854857:2116] self: [5:7587241364524854861:0] peer: [4:7587241366717750942:0] socket: 223 qp: -1
2025-12-24T02:11:30.287463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241364524854864:2098] [node 4] ICS10 traffic start
2025-12-24T02:11:30.287481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.291098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241362634981565:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.291130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.291138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241362634981565:2107] [node 6] ICP22 created new session: [3:7587241362634981588:2107]
2025-12-24T02:11:30.291145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241362634981588:2107] [node 6] ICS09 handshake done sender: [3:7587241362634981568:2109] self: [3:7587241362634981567:0] peer: [6:7587241363701187484:0] socket: 204 qp: -1
2025-12-24T02:11:30.291152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241362634981588:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:30.291163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.291473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241362634981583:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.292475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241366717750880:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.294877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070771:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.294949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363700070752:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.295036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187493:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.296333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187494:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.296645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241362634981593:2118] [node 4] ICP01 ready to work
2025-12-24T02:11:30.295430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093681:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.295442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241366678093681:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.297114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241362634981593:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.297691Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187405:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241363701187477:2111] held: no
2025-12-24T02:11:30.297796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241363701187477:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.297874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241363701187405:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.297890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.297903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241363701187405:2050] [node 3] ICP22 created new session: [6:7587241363701187495:2050]
2025-12-24T02:11:30.297910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241363701187495:2050] [node 3] ICS09 handshake done sender: [6:7587241363701187477:2111] self: [6:7587241363701187484:0] peer: [3:7587241362634981567:0] socket: 207 qp: -1
2025-12-24T02:11:30.297916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241363701187495:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.297922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.295640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363700070691:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.295667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.295674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363700070691:2049] [node 3] ICP22 created new session: [2:7587241363700070777:2049]
2025-12-24T02:11:30.295682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363700070777:2049] [node 3] ICS09 handshake done sender: [2:7587241363700070752:2108] self: [2:7587241363700070751:0] peer: [3:7587241362634981585:0] socket: 209 qp: -1
2025-12-24T02:11:30.295689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363700070777:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:30.295699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.299153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241366678093683:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.299359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241362634981570:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.298990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241363701187494:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.299032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241363701187456:2093] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.302931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241366717750947:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.302950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.302956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241366717750947:2110] [node 6] ICP22 created new session: [4:7587241366717750961:2110]
2025-12-24T02:11:30.302962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241366717750961:2110] [node 6] ICS09 handshake done sender: [4:7587241366717750940:2108] self: [4:7587241366717750948:0] peer: [6:7587241363701187449:0] socket: 200 qp: -1
2025-12-24T02:11:30.302969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241366717750961:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:30.302977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.303148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241366717750954:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.303153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241366717750954:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.303459Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241366717750949:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.303936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241366717750954:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.304014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241366717750881:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.304029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.304035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241366717750881:2050] [node 3] ICP22 created new session: [4:7587241366717750964:2050]
2025-12-24T02:11:30.304042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241366717750964:2050] [node 3] ICS09 handshake done sender: [4:7587241366717750954:2115] self: [4:7587241366717750945:0] peer: [3:7587241362634981594:0] socket: 235 qp: -1
2025-12-24T02:11:30.304047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241366717750964:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.304053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.304294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241366717750879:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.304305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.304310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241366717750879:2048] [node 1] ICP22 created new session: [4:7587241366717750966:2048]
2025-12-24T02:11:30.304315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241366717750966:2048] [node 1] ICS09 handshake done sender: [4:7587241366717750949:2112] self: [4:7587241366717750951:0] peer: [1:7587241366678093678:0] socket: 216 qp: -1
2025-12-24T02:11:30.304320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241366717750966:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.304326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.307478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854867:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.307953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070760:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.307964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363700070760:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.308460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363700070779:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59966
2025-12-24T02:11:30.308738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241364524854846:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.308802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241364524854780:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.308817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.308823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241364524854780:2050] [node 3] ICP22 created new session: [5:7587241364524854869:2050]
2025-12-24T02:11:30.308829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241364524854869:2050] [node 3] ICS09 handshake done sender: [5:7587241364524854846:2111] self: [5:7587241364524854858:0] peer: [3:7587241362634981569:0] socket: 208 qp: -1
2025-12-24T02:11:30.308833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241364524854869:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.308840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.306959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241362634981566:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.306982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.306988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241362634981566:2108] [node 5] ICP22 created new session: [3:7587241362634981598:2108]
2025-12-24T02:11:30.306994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241362634981598:2108] [node 5] ICS09 handshake done sender: [3:7587241362634981570:2110] self: [3:7587241362634981569:0] peer: [5:7587241364524854858:0] socket: 205 qp: -1
2025-12-24T02:11:30.306998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241362634981598:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:30.307004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.309696Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981596:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.309827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981597:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.311215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854854:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.311229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241364524854854:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.311386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363700070760:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.314993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241366717750968:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.317241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241366678093681:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.317357Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241366717750953:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.315160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241363700070691:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19469737ns) {HandleHandshakeStatus:329(19468885ns) {DropIncomingHandshake:442(4504ns) DropOutgoingHandshake:457(147ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(15876032ns) {unordered ForwardSessionEventToSession:612 num# 30 sum# 15866341ns max# 15838650ns}}}
2025-12-24T02:11:30.315280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363700070757:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.315290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.315312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363700070757:2109] [node 4] ICP22 created new session: [2:7587241363700070783:2109]
2025-12-24T02:11:30.315318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363700070783:2109] [node 4] ICS09 handshake done sender: [2:7587241363700070760:2111] self: [2:7587241363700070759:0] peer: [4:7587241366717750960:0] socket: 231 qp: -1
2025-12-24T02:11:30.315323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363700070783:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:30.315329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.321664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187456:2093] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19561) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241363701187494:2119] held: no
2025-12-24T02:11:30.319428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854868:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.319714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241364524854854:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.319776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241364524854832:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.319790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.319795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241364524854832:2095] [node 6] ICP22 created new session: [5:7587241364524854872:2095]
2025-12-24T02:11:30.319802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241364524854872:2095] [node 6] ICS09 handshake done sender: [5:7587241364524854854:2115] self: [5:7587241364524854853:0] peer: [6:7587241363701187497:0] socket: 248 qp: -1
2025-12-24T02:11:30.319806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241364524854872:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:30.319812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.319626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363700070766:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.320773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241363700070690:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.320785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.320791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363700070690:2048] [node 1] ICP22 created new session: [2:7587241363700070786:2048]
2025-12-24T02:11:30.320796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363700070786:2048] [node 1] ICS09 handshake done sender: [2:7587241363700070766:2114] self: [2:7587241363700070775:0] peer: [1:7587241366678093680:0] socket: 221 qp: -1
2025-12-24T02:11:30.320801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363700070786:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.320822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.316969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241362634981595:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53370
2025-12-24T02:11:30.326820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241366678093608:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.326841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.326849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241366678093608:2049] [node 3] ICP22 created new session: [1:7587241366678093701:2049]
2025-12-24T02:11:30.326857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241366678093701:2049] [node 3] ICS09 handshake done sender: [1:7587241366678093683:2114] self: [1:7587241366678093682:0] peer: [3:7587241362634981587:0] socket: 222 qp: -1
2025-12-24T02:11:30.326863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241366678093701:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:30.326871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.331537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241366678093607:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.331563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.331569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241366678093607:2048] [node 2] ICP22 created new session: [1:7587241366678093703:2048]
2025-12-24T02:11:30.331573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241366678093703:2048] [node 2] ICS09 handshake done sender: [1:7587241366678093681:2113] self: [1:7587241366678093680:0] peer: [2:7587241363700070775:0] socket: 213 qp: -1
2025-12-24T02:11:30.331587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241366678093703:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:30.331592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.330880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241366717750880:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.330904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.330923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241366717750880:2049] [node 2] ICP22 created new session: [4:7587241366717750969:2049]
2025-12-24T02:11:30.330931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241366717750969:2049] [node 2] ICS09 handshake done sender: [4:7587241366717750953:2114] self: [4:7587241366717750960:0] peer: [2:7587241363700070759:0] socket: 234 qp: -1
2025-12-24T02:11:30.330938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241366717750969:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.330946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.334886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854863:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.334935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241364524854778:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.335136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187498:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.335145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187499:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.335229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241363701187494:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.335341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241363701187456:2093] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.335353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.335358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241363701187456:2093] [node 5] ICP22 created new session: [6:7587241363701187500:2093]
2025-12-24T02:11:30.335364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241363701187500:2093] [node 5] ICS09 handshake done sender: [6:7587241363701187494:2119] self: [6:7587241363701187497:0] peer: [5:7587241364524854853:0] socket: 250 qp: -1
2025-12-24T02:11:30.335370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241363701187500:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:30.335376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.336250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241364524854868:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.336272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241364524854779:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.335442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241362634981504:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241362634981574:2112] held: no
2025-12-24T02:11:30.335573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241362634981574:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.335646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241362634981504:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.335654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.335660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241362634981504:2049] [node 2] ICP22 created new session: [3:7587241362634981600:2049]
2025-12-24T02:11:30.335666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241362634981600:2049] [node 2] ICS09 handshake done sender: [3:7587241362634981574:2112] self: [3:7587241362634981585:0] peer: [2:7587241363700070751:0] socket: 214 qp: -1
2025-12-24T02:11:30.335672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241362634981600:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.335679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.335878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241362634981580:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.335936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241362634981583:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.336039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241362634981593:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.336047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.336052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241362634981593:2118] [node 4] ICP22 created new session: [3:7587241362634981602:2118]
2025-12-24T02:11:30.336057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241362634981602:2118] [node 4] ICS09 handshake done sender: [3:7587241362634981583:2117] self: [3:7587241362634981594:0] peer: [4:7587241366717750945:0] socket: 239 qp: -1
2025-12-24T02:11:30.336061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241362634981602:2118] [node 4] ICS10 traffic start
2025-12-24T02:11:30.336066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.339191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241362634981503:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.339207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.339216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241362634981503:2048] [node 1] ICP22 created new session: [3:7587241362634981604:2048]
2025-12-24T02:11:30.339222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241362634981604:2048] [node 1] ICS09 handshake done sender: [3:7587241362634981580:2115] self: [3:7587241362634981587:0] peer: [1:7587241366678093682:0] socket: 226 qp: -1
2025-12-24T02:11:30.339228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241362634981604:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.339233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.340126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241366678093700:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47692
2025-12-24T02:11:30.340396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241366678093686:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.340400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241366678093686:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.341941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241366678093686:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.344124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241363701187498:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.344154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241363701187404:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.342921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241366678093608:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16029399ns) {HandleHandshakeStatus:329(16028602ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(225ns) SwitchToState:179(377ns) ProcessPendingSessionEvents:495(15919714ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 15906201ns max# 15860342ns}}}
2025-12-24T02:11:30.343025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241366678093674:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.343038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.343044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241366678093674:2110] [node 5] ICP22 created new session: [1:7587241366678093706:2110]
2025-12-24T02:11:30.343049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241366678093706:2110] [node 5] ICS09 handshake done sender: [1:7587241366678093686:2115] self: [1:7587241366678093675:0] peer: [5:7587241364524854874:0] socket: 244 qp: -1
2025-12-24T02:11:30.343054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241366678093706:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:30.343060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.346159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241363701187503:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58725
2025-12-24T02:11:30.346307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241364524854779:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241364524854868:2121] held: no
2025-12-24T02:11:30.346609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241363701187498:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.343283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070764:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.343296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363700070764:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.344268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363700070764:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.344350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363700070773:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.344352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363700070773:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.344825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363700070758:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.344836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.344841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363700070758:2110] [node 5] ICP22 created new session: [2:7587241363700070789:2110]
2025-12-24T02:11:30.344846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363700070789:2110] [node 5] ICS09 handshake done sender: [2:7587241363700070764:2113] self: [2:7587241363700070763:0] peer: [5:7587241364524854875:0] socket: 260 qp: -1
2025-12-24T02:11:30.344851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363700070789:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:30.344858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.346453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363700070773:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.346667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241363701187404:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.346679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.346684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241363701187404:2049] [node 2] ICP22 created new session: [6:7587241363701187504:2049]
2025-12-24T02:11:30.346690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241363701187504:2049] [node 2] ICS09 handshake done sender: [6:7587241363701187498:2120] self: [6:7587241363701187502:0] peer: [2:7587241363700070765:0] socket: 262 qp: -1
2025-12-24T02:11:30.346706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241363701187504:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.346712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.347920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363700070755:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.347936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.347940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363700070755:2107] [node 6] ICP22 created new session: [2:7587241363700070792:2107]
2025-12-24T02:11:30.347945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363700070792:2107] [node 6] ICS09 handshake done sender: [2:7587241363700070773:2118] self: [2:7587241363700070765:0] peer: [6:7587241363701187502:0] socket: 224 qp: -1
2025-12-24T02:11:30.347949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363700070792:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:30.347954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.359990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241366678093674:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(16838040ns) {HandleHandshakeStatus:329(16837351ns) {DropIncomingHandshake:442(3695ns) DropOutgoingHandshake:457(113ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(16779348ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16753383ns max# 16676760ns}}}
2025-12-24T02:11:30.378814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241366678093608:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:30.382987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241363700070755:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(34951080ns) {HandleHandshakeStatus:329(34950274ns) {DropIncomingHandshake:442(3895ns) DropOutgoingHandshake:457(82ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(34873889ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 34825099ns max# 34743768ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.387047Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241364524854778:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241364524854863:2119] held: no
2025-12-24T02:11:30.387221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854876:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.397585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241364524854863:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.397716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241364524854778:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.397734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.397749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241364524854778:2048] [node 1] ICP22 created new session: [5:7587241364524854878:2048]
2025-12-24T02:11:30.397756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241364524854878:2048] [node 1] ICS09 handshake done sender: [5:7587241364524854863:2119] self: [5:7587241364524854874:0] peer: [1:7587241366678093675:0] socket: 247 qp: -1
2025-12-24T02:11:30.397767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241364524854878:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.397778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.398134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241364524854877:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26956
2025-12-24T02:11:30.402933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241364524854868:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.403131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241364524854779:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.403153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.403161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241364524854779:2049] [node 2] ICP22 created new session: [5:7587241364524854880:2049]
2025-12-24T02:11:30.403168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241364524854880:2049] [node 2] ICS09 handshake done sender: [5:7587241364524854868:2121] self: [5:7587241364524854875:0] peer: [2:7587241363700070763:0] socket: 261 qp: -1
2025-12-24T02:11:30.403175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241364524854880:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.403183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.507500Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:30.507616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241363700070786:2048] [node 1] ICS07 socket disconnect 221 reason# EndOfStream
2025-12-24T02:11:30.507623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241363700070786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.507660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.507678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241363700070786:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.515492Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241363700070690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:30.515511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241363700070786:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.516000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241363700070690:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.516012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241363700070786:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.516017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241363700070690:2048] [node 1] ICP30 unregister session Session# [2:7587241363700070786:2048] VirtualId# [2:7587241363700070775:0]
2025-12-24T02:11:30.516043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.516050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241363700070690:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.516054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.524061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:30.524139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241363701187489:2048] [node 1] ICS07 socket disconnect 201 reason# EndOfStream
2025-12-24T02:11:30.524146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241363701187489:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.524186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.524205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241363701187489:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.525012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187403:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.525021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241363701187489:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.525025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241363701187403:2048] [node 1] ICP30 unregister session Session# [6:7587241363701187489:2048] VirtualId# [6:7587241363701187483:0]
2025-12-24T02:11:30.525067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.525071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241363701187403:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.525073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.518903Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:30.518994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241362634981604:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-12-24T02:11:30.519003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241362634981503:2048] [node 1] ICP30 unregister session Session# [3:7587241362634981604:2048] VirtualId# [3:7587241362634981587:0]
2025-12-24T02:11:30.519010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241362634981604:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.519061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.519088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.525672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:30.525754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241364524854878:2048] [node 1] ICS07 socket disconnect 247 reason# EndOfStream
2025-12-24T02:11:30.525761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241364524854878:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.525808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.525830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241364524854878:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.526174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241364524854778:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.526186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241364524854878:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.526191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241364524854778:2048] [node 1] ICP30 unregister session Session# [5:7587241364524854878:2048] VirtualId# [5:7587241364524854874:0]
2025-12-24T02:11:30.526225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.526230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241364524854778:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.526234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.524407Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:30.524468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241366717750966:2048] [node 1] ICS07 socket disconnect 216 reason# EndOfStream
2025-12-24T02:11:30.524473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241366717750966:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.524498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:30.524516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241366717750966:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:30.524810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241366717750879:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.524816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241366717750966:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.524821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241366717750879:2048] [node 1] ICP30 unregister session Session# [4:7587241366717750966:2048] VirtualId# [4:7587241366717750951:0]
2025-12-24T02:11:30.524865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:30.524871Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241366717750879:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.524874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19642) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.536203Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:30.536297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241362634981600:2049] [node 2] ICS07 socket disconnect 214 reason# EndOfStream
2025-12-24T02:11:30.536304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241362634981600:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.536337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.536354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241362634981600:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.538777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241363701187403:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:30.538821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241366717750879:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:30.538899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241364524854778:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:30.543376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241362634981504:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:30.543393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241362634981600:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.543787Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241362634981504:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.543799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241362634981600:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.543805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241362634981504:2049] [node 2] ICP30 unregister session Session# [3:7587241362634981600:2049] VirtualId# [3:7587241362634981585:0]
2025-12-24T02:11:30.543829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.543836Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241362634981504:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.543841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.554988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:30.555098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241366717750969:2049] [node 2] ICS07 socket disconnect 234 reason# EndOfStream
2025-12-24T02:11:30.555103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241366717750969:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.555138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.555158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241366717750969:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.554964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:30.555076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241364524854880:2049] [node 2] ICS07 socket disconnect 261 reason# EndOfStream
2025-12-24T02:11:30.555084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241364524854880:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.555130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.555155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241364524854880:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.555514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241366717750880:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.555526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241366717750969:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.555531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241366717750880:2049] [node 2] ICP30 unregister session Session# [4:7587241366717750969:2049] VirtualId# [4:7587241366717750960:0]
2025-12-24T02:11:30.555590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.555596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241366717750880:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.555600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.555626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241364524854779:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.555640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241364524854880:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.555645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241364524854779:2049] [node 2] ICP30 unregister session Session# [5:7587241364524854880:2049] VirtualId# [5:7587241364524854875:0]
2025-12-24T02:11:30.555677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.555684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241364524854779:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.555686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.558885Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:30.558990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241363701187504:2049] [node 2] ICS07 socket disconnect 262 reason# EndOfStream
2025-12-24T02:11:30.559001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241363701187504:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.559048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:30.559070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241363701187504:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:30.559414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187404:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.559428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241363701187504:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.559433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241363701187404:2049] [node 2] ICP30 unregister session Session# [6:7587241363701187504:2049] VirtualId# [6:7587241363701187502:0]
2025-12-24T02:11:30.559474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:30.559480Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241363701187404:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.559483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16106) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.566889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241366717750880:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:30.567614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241364524854779:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:30.569437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:30.569538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241363701187495:2050] [node 3] ICS07 socket disconnect 207 reason# EndOfStream
2025-12-24T02:11:30.569549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241363701187495:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.569598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:30.569620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241363701187495:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:30.569994Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187405:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.570007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241363701187495:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.570013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241363701187405:2050] [node 3] ICP30 unregister session Session# [6:7587241363701187495:2050] VirtualId# [6:7587241363701187484:0]
2025-12-24T02:11:30.570043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:30.570049Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241363701187405:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.570053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.569801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:30.569885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241364524854869:2050] [node 3] ICS07 socket disconnect 208 reason# EndOfStream
2025-12-24T02:11:30.569893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241364524854869:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.569952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:30.569984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241364524854869:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:30.570373Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241364524854780:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.570386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241364524854869:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.570392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241364524854780:2050] [node 3] ICP30 unregister session Session# [5:7587241364524854869:2050] VirtualId# [5:7587241364524854858:0]
2025-12-24T02:11:30.570428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:30.570434Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241364524854780:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.570437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31440) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.574816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241363701187404:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:30.579115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241364524854780:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:30.579414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241363701187405:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:30.589494Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:30.589602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241363701187487:2089] [node 4] ICS07 socket disconnect 193 reason# EndOfStream
2025-12-24T02:11:30.589609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241363701187487:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.589658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:30.589679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241363701187487:2089] [node 4] ICS15 start handshake
2025-12-24T02:11:30.590079Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241363701187446:2089] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17534) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:30.590092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241363701187487:2089] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:30.590098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241363701187446:2089] [node 4] ICP30 unregister session Session# [6:7587241363701187487:2089] VirtualId# [6:7587241363701187449:0]
2025-12-24T02:11:30.590119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:30.590126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241363701187446:2089] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17534) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.590129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17534) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:30.602743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241363701187446:2089] [node 4] ICP33 wake up from error state
2025-12-24T02:11:30.602860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:30.602955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241363701187500:2093] [node 5] ICS01 socket: 250 reason# EndOfStream
2025-12-24T02:11:30.602964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241363701187456:2093] [node 5] ICP30 unregister session Session# [6:7587241363701187500:2093] VirtualId# [6:7587241363701187497:0]
2025-12-24T02:11:30.602971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241363701187500:2093] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:30.603019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:30.603045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:30.650295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363421293052:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.650310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363421293053:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:30.646979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241366268923914:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.646989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241366268923915:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.638966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241364269032270:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:30.638973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241364269032271:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:30.667095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241366338788687:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.667105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241366338788688:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.666728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241364260399537:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.666764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241364260399538:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.670796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241363621815035:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.670807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241363621815036:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:30.671370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241366338788686:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:30.670812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241363621815037:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.666770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241364260399539:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:30.759089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241366338788737:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:30.759759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241366338788744:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:30.760185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241364269032333:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:30.759036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363421293114:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:30.759157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363421293115:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.767120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241363421293125:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.771022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241364269032341:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21025
2025-12-24T02:11:30.772461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241364269032343:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:30.772410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399600:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.772573Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399598:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.774770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363421293128:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9005
2025-12-24T02:11:30.774967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399598:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.775156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241364260399602:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:30.775638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241364260399602:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.774714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241364269032332:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:30.776147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032341:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.777112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923978:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.777931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399599:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.782886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241366338788744:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23042482ns) {Configure:104(23041426ns) {ProcessConfigured:135(22793287ns) {StartInitialHandshake:157(22792602ns) {DropHandshakes:470(321ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(1647ns) {SwitchToState:179(876ns)}}}}}
2025-12-24T02:11:30.783019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399599:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.783427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241364260399538:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.786949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815096:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.786912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363421293131:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9005
2025-12-24T02:11:30.787755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923978:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.787840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293131:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.787985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399600:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.788352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241364260399537:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.790624Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241366338788743:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.790641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241366338788743:2102] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.790788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293118:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.790808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363421293118:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.790749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241363621815096:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.790969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241363621815098:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:30.791397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241363621815098:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.791111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241363421293115:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28758337ns) {Configure:104(28754789ns) {ProcessConfigured:135(28430104ns) {StartInitialHandshake:157(28429582ns) {DropHandshakes:470(698ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(327ns) PrepareNewSessionHandshake:477(669ns) {SwitchToState:179(158ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.794787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032336:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.794803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241364269032336:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.795401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241364269032352:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21025
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.796161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399606:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.796398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241364269032336:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.796600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241364260399600:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.796702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241364260399537:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.796720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.796727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241364260399537:2048] [node 1] ICP22 created new session: [4:7587241364260399607:2048]
2025-12-24T02:11:30.796734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241364260399607:2048] [node 1] ICS09 handshake done sender: [4:7587241364260399600:2110] self: [4:7587241364260399605:0] peer: [1:7587241364269032334:0] socket: 211 qp: -1
2025-12-24T02:11:30.796741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241364260399607:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.796764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.796999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241364269032333:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.797011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.797017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241364269032333:2108] [node 4] ICP22 created new session: [1:7587241364269032353:2108]
2025-12-24T02:11:30.797022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241364269032353:2108] [node 4] ICS09 handshake done sender: [1:7587241364269032336:2109] self: [1:7587241364269032334:0] peer: [4:7587241364260399605:0] socket: 204 qp: -1
2025-12-24T02:11:30.797026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241364269032353:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:30.797040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.795607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241366338788755:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.795619Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241366338788755:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.798651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241366338788755:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.798994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241366338788744:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.799006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.799013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241366338788744:2099] [node 5] ICP22 created new session: [6:7587241366338788758:2099]
2025-12-24T02:11:30.799019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241366338788758:2099] [node 5] ICS09 handshake done sender: [6:7587241366338788755:2111] self: [6:7587241366338788745:0] peer: [5:7587241363621815099:0] socket: 218 qp: -1
2025-12-24T02:11:30.799026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241366338788758:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:30.799035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.799718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788761:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
2025-12-24T02:11:30.799779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788762:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
2025-12-24T02:11:30.798917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293128:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.799000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241363421293052:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.800330Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241366268923914:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27673467ns) {Configure:104(27673163ns) {ProcessConfigured:135(27274404ns) {StartInitialHandshake:157(27273992ns) {DropHandshakes:470(341ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(61ns)}}}}}
2025-12-24T02:11:30.800375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241366268923914:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.802974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241366338788743:2102] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.802982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241366338788761:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.803001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241366338788762:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.803273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241366338788737:2097] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.803290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.802835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363421293118:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.803627Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293127:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.803632Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363421293127:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.803298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241366338788737:2097] [node 4] ICP22 created new session: [6:7587241366338788763:2097]
2025-12-24T02:11:30.803305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241366338788763:2097] [node 4] ICS09 handshake done sender: [6:7587241366338788743:2102] self: [6:7587241366338788741:0] peer: [4:7587241364260399603:0] socket: 198 qp: -1
2025-12-24T02:11:30.803311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241366338788763:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:30.803320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.803459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241366338788687:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.803634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399609:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.803845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241364260399598:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.807209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241366338788686:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.804204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363421293127:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.804228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363421293114:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.804238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.804244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363421293114:2107] [node 4] ICP22 created new session: [2:7587241363421293137:2107]
2025-12-24T02:11:30.804251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363421293137:2107] [node 4] ICS09 handshake done sender: [2:7587241363421293118:2109] self: [2:7587241363421293116:0] peer: [4:7587241364260399604:0] socket: 200 qp: -1
2025-12-24T02:11:30.804257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363421293137:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:30.804275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.804620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363421293125:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.804627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.804633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363421293125:2111] [node 6] ICP22 created new session: [2:7587241363421293140:2111]
2025-12-24T02:11:30.804639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363421293140:2111] [node 6] ICS09 handshake done sender: [2:7587241363421293127:2112] self: [2:7587241363421293126:0] peer: [6:7587241366338788765:0] socket: 215 qp: -1
2025-12-24T02:11:30.804644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363421293140:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:30.804650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.810750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032338:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.810769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241364269032338:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.810813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032344:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.810820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241364269032344:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.810944Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399610:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.810994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032346:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.811005Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241364269032346:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.811331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032352:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.811913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241364269032346:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.811981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241364269032343:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.811994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.812001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241364269032343:2112] [node 6] ICP22 created new session: [1:7587241364269032358:2112]
2025-12-24T02:11:30.812007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241364269032358:2112] [node 6] ICS09 handshake done sender: [1:7587241364269032346:2114] self: [1:7587241364269032345:0] peer: [6:7587241366338788766:0] socket: 225 qp: -1
2025-12-24T02:11:30.812014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241364269032358:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:30.812022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.811639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923982:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.814979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923982:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.815013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241366268923915:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.821570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293130:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.821582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363421293130:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.822074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363421293130:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.822544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788767:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.822872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363421293053:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.822897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.822903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363421293053:2049] [node 3] ICP22 created new session: [2:7587241363421293144:2049]
2025-12-24T02:11:30.822912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363421293144:2049] [node 3] ICS09 handshake done sender: [2:7587241363421293130:2114] self: [2:7587241363421293119:0] peer: [3:7587241366268923983:0] socket: 233 qp: -1
2025-12-24T02:11:30.822918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363421293144:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:30.822925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.823705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241363421293146:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9005
2025-12-24T02:11:30.824284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363421293128:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.824329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241363421293052:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.824338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.824344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363421293052:2048] [node 1] ICP22 created new session: [2:7587241363421293147:2048]
2025-12-24T02:11:30.824350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363421293147:2048] [node 1] ICS09 handshake done sender: [2:7587241363421293128:2113] self: [2:7587241363421293134:0] peer: [1:7587241364269032337:0] socket: 207 qp: -1
2025-12-24T02:11:30.824355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363421293147:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.824360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.823508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241364269032338:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.823792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241364269032270:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.823805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.823812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241364269032270:2048] [node 2] ICP22 created new session: [1:7587241364269032362:2048]
2025-12-24T02:11:30.823818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241364269032362:2048] [node 2] ICS09 handshake done sender: [1:7587241364269032338:2110] self: [1:7587241364269032337:0] peer: [2:7587241363421293134:0] socket: 205 qp: -1
2025-12-24T02:11:30.823824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241364269032362:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:30.823847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.830129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241366338788761:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.826948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241366268923987:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:30.828931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241364269032344:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.830179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241364269032271:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.830196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.830227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241364269032271:2049] [node 3] ICP22 created new session: [1:7587241364269032365:2049]
2025-12-24T02:11:30.830234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241364269032365:2049] [node 3] ICS09 handshake done sender: [1:7587241364269032344:2113] self: [1:7587241364269032335:0] peer: [3:7587241366268923981:0] socket: 214 qp: -1
2025-12-24T02:11:30.830240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241364269032365:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:30.830247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.831163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788768:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.835413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815101:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.835599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815102:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.835672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815103:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.835528Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241366268923915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241366268923982:2110] held: no
2025-12-24T02:11:30.836107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241366268923993:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:30.836178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923989:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.835882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241363621815102:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.836272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241363621815036:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.837429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815105:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.837619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241363621815103:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.838103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241363621815102:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.838249Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241363621815096:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.838114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241366338788687:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.838133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.838140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241366338788687:2049] [node 2] ICP22 created new session: [6:7587241366338788769:2049]
2025-12-24T02:11:30.838146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241366338788769:2049] [node 2] ICS09 handshake done sender: [6:7587241366338788761:2112] self: [6:7587241366338788765:0] peer: [2:7587241363421293126:0] socket: 217 qp: -1
2025-12-24T02:11:30.838152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241366338788769:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.838159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.838496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241366338788762:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.836455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241363421293133:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.836466Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241363421293133:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.837545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241363421293133:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.837698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241363421293115:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.837709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.837714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241363421293115:2108] [node 5] ICP22 created new session: [2:7587241363421293150:2108]
2025-12-24T02:11:30.837720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241363421293150:2108] [node 5] ICS09 handshake done sender: [2:7587241363421293133:2116] self: [2:7587241363421293117:0] peer: [5:7587241363621815104:0] socket: 234 qp: -1
2025-12-24T02:11:30.837725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241363421293150:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:30.837731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.838566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241363621815035:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.838578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241363621815036:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.838585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.838589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241363621815036:2049] [node 2] ICP22 created new session: [5:7587241363621815107:2049]
2025-12-24T02:11:30.838593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241363621815107:2049] [node 2] ICS09 handshake done sender: [5:7587241363621815102:2111] self: [5:7587241363621815104:0] peer: [2:7587241363421293117:0] socket: 208 qp: -1
2025-12-24T02:11:30.838598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241363621815107:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.838630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.839094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241366268923986:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:30.839361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241363621815098:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.839372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.839377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241363621815098:2109] [node 6] ICP22 created new session: [5:7587241363621815109:2109]
2025-12-24T02:11:30.839381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241363621815109:2109] [node 6] ICS09 handshake done sender: [5:7587241363621815096:2108] self: [5:7587241363621815099:0] peer: [6:7587241366338788745:0] socket: 220 qp: -1
2025-12-24T02:11:30.839386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241363621815109:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:30.839391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.843666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241366338788686:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.843684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.842899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241364260399602:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.842926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.842935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241364260399602:2111] [node 6] ICP22 created new session: [4:7587241364260399612:2111]
2025-12-24T02:11:30.842941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241364260399612:2111] [node 6] ICS09 handshake done sender: [4:7587241364260399598:2108] self: [4:7587241364260399603:0] peer: [6:7587241366338788741:0] socket: 201 qp: -1
2025-12-24T02:11:30.842949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241364260399612:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:30.842957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:30.843027Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241364260399602:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(28195832ns) {HandleHandshakeStatus:329(28195096ns) {DropIncomingHandshake:442(1565ns) DropOutgoingHandshake:457(56ns) SwitchToState:179(230ns) ProcessPendingSessionEvents:495(111ns)}}
2025-12-24T02:11:30.843291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241364260399614:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:30.843396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241364260399599:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.843695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241366338788686:2048] [node 1] ICP22 created new session: [6:7587241366338788771:2048]
2025-12-24T02:11:30.843701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241366338788771:2048] [node 1] ICS09 handshake done sender: [6:7587241366338788762:2113] self: [6:7587241366338788766:0] peer: [1:7587241364269032345:0] socket: 228 qp: -1
2025-12-24T02:11:30.843706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241366338788771:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.843712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.846883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241364260399538:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.846900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.846909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241364260399538:2049] [node 2] ICP22 created new session: [4:7587241364260399615:2049]
2025-12-24T02:11:30.846915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241364260399615:2049] [node 2] ICS09 handshake done sender: [4:7587241364260399599:2109] self: [4:7587241364260399604:0] peer: [2:7587241363421293116:0] socket: 202 qp: -1
2025-12-24T02:11:30.846923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241364260399615:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.846930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.850769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923990:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.850747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241364269032348:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.850763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241364269032348:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.850960Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241364269032271:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20657891ns) {HandleHandshakeStatus:329(20657105ns) {DropIncomingHandshake:442(3964ns) DropOutgoingHandshake:457(115ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(20551266ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20531617ns max# 20456734ns}}}
2025-12-24T02:11:30.854912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241366268923914:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241366268923978:2108] held: no
2025-12-24T02:11:30.855417Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241366268923982:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:30.855561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241366268923978:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.854959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241363621815112:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:30.855581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241363621815098:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(16182453ns) {HandleHandshakeStatus:329(16181583ns) {DropIncomingHandshake:442(170ns) DropOutgoingHandshake:457(77ns) SwitchToState:179(61ns) ProcessPendingSessionEvents:495(16129165ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 16123302ns max# 16103504ns}}}
2025-12-24T02:11:30.855867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815113:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.855688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241366268923915:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.855699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:30.855705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241366268923915:2049] [node 2] ICP22 created new session: [3:7587241366268924002:2049]
2025-12-24T02:11:30.855710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241366268924002:2049] [node 2] ICS09 handshake done sender: [3:7587241366268923982:2110] self: [3:7587241366268923983:0] peer: [2:7587241363421293119:0] socket: 237 qp: -1
2025-12-24T02:11:30.855716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241366268924002:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:30.855721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:30.863626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241364269032348:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.863792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241364269032332:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.863805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.863813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241364269032332:2107] [node 5] ICP22 created new session: [1:7587241364269032368:2107]
2025-12-24T02:11:30.863820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241364269032368:2107] [node 5] ICS09 handshake done sender: [1:7587241364269032348:2115] self: [1:7587241364269032347:0] peer: [5:7587241363621815106:0] socket: 238 qp: -1
2025-12-24T02:11:30.863829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241364269032368:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:30.863835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.866790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923998:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.866827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241366268923914:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.866839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.866846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241366268923914:2048] [node 1] ICP22 created new session: [3:7587241366268924004:2048]
2025-12-24T02:11:30.866852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241366268924004:2048] [node 1] ICS09 handshake done sender: [3:7587241366268923978:2108] self: [3:7587241366268923981:0] peer: [1:7587241364269032335:0] socket: 216 qp: -1
2025-12-24T02:11:30.866857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241366268924004:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.866864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.870752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268923997:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.874872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241363621815037:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40056281ns) {Configure:104(40055263ns) {ProcessConfigured:135(39667744ns) {StartInitialHandshake:157(39666936ns) {DropHandshakes:470(418ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(1630ns) {SwitchToState:179(1052ns)}}}}}
2025-12-24T02:11:30.875228Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241363621815112:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20040264ns) {Configure:104(20039853ns) {ProcessConfigured:135(19720057ns) {StartInitialHandshake:157(19719737ns) {DropHandshakes:470(366ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(108ns) PrepareNewSessionHandshake:477(1105ns) {SwitchToState:179(419ns)}}}}}
2025-12-24T02:11:30.875881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815119:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.876000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815120:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.875459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923997:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.878926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241363621815103:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:30.882729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788776:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.883503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.883557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399624:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.886813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241364269032332:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(22857444ns) {HandleHandshakeStatus:329(22856321ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(157ns) SwitchToState:179(530ns) ProcessPendingSessionEvents:495(22781204ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22757863ns max# 22676352ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.886921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241363621815035:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.886948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:30.886958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241363621815035:2048] [node 1] ICP22 created new session: [5:7587241363621815121:2048]
2025-12-24T02:11:30.886969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241363621815121:2048] [node 1] ICS09 handshake done sender: [5:7587241363621815103:2112] self: [5:7587241363621815106:0] peer: [1:7587241364269032347:0] socket: 243 qp: -1
2025-12-24T02:11:30.886978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241363621815121:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:30.886988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:30.887377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399623:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.887428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241364260399539:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.887532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268924000:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.887542Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241366268924000:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.890780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241366338788776:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.890823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241366338788688:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.894809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241363621815113:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.894847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241363621815112:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.897768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241366268924008:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44601
2025-12-24T02:11:30.898860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268924008:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.898779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399620:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.898794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241364260399620:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.899051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923998:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.899151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241363621815120:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.899175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241363621815037:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:30.899294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241364260399624:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.902983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923992:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.902992Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241366268923992:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.903300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241366268924000:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.910738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241366268923995:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:30.910755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241366268923995:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:30.910879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241366268923986:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.910901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.910909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241366268923986:2111] [node 4] ICP22 created new session: [3:7587241366268924012:2111]
2025-12-24T02:11:30.910917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241366268924012:2111] [node 4] ICS09 handshake done sender: [3:7587241366268924000:2120] self: [3:7587241366268923999:0] peer: [4:7587241364260399625:0] socket: 256 qp: -1
2025-12-24T02:11:30.910925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241366268924012:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:30.910937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.920808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241364260399620:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.920490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241366268923992:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:30.925450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241366268923995:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:30.936559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241364260399539:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241364260399623:2118] held: no
2025-12-24T02:11:30.934800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241366268923993:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.934837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.934851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241366268923993:2113] [node 5] ICP22 created new session: [3:7587241366268924016:2113]
2025-12-24T02:11:30.934862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241366268924016:2113] [node 5] ICS09 handshake done sender: [3:7587241366268923995:2117] self: [3:7587241366268923994:0] peer: [5:7587241363621815124:0] socket: 255 qp: -1
2025-12-24T02:11:30.934874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241366268924016:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:30.934886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.935191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241366268923987:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.935203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:30.935210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241366268923987:2112] [node 6] ICP22 created new session: [3:7587241366268924018:2112]
2025-12-24T02:11:30.935218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241366268924018:2112] [node 6] ICS09 handshake done sender: [3:7587241366268923992:2116] self: [3:7587241366268923991:0] peer: [6:7587241366338788777:0] socket: 251 qp: -1
2025-12-24T02:11:30.935224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241366268924018:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:30.935230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.938872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815125:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.942876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241364260399614:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:30.942903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:30.942914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241364260399614:2115] [node 5] ICP22 created new session: [4:7587241364260399629:2115]
2025-12-24T02:11:30.942924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241364260399629:2115] [node 5] ICS09 handshake done sender: [4:7587241364260399620:2117] self: [4:7587241364260399619:0] peer: [5:7587241363621815123:0] socket: 247 qp: -1
2025-12-24T02:11:30.942932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241364260399629:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:30.942939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:30.946137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241366338788688:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241366338788776:2117] held: no
2025-12-24T02:11:30.946747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241363621815126:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18504
2025-12-24T02:11:30.946920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241363621815037:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14766) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241363621815120:2119] held: no
2025-12-24T02:11:30.950852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241364260399628:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20260
2025-12-24T02:11:30.951902Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241364260399614:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(29061137ns) {HandleHandshakeStatus:329(29060233ns) {DropIncomingHandshake:442(6218ns) DropOutgoingHandshake:457(466ns) SwitchToState:179(368ns) ProcessPendingSessionEvents:495(8803865ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8785943ns max# 8703491ns}}}
2025-12-24T02:11:30.955100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241364260399623:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.955197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241364260399539:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.955218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.955227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241364260399539:2050] [node 3] ICP22 created new session: [4:7587241364260399631:2050]
2025-12-24T02:11:30.955234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241364260399631:2050] [node 3] ICS09 handshake done sender: [4:7587241364260399623:2118] self: [4:7587241364260399625:0] peer: [3:7587241366268923999:0] socket: 257 qp: -1
2025-12-24T02:11:30.955242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241364260399631:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.955250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:30.987010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241363621815113:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:30.987259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241366338788778:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14598
2025-12-24T02:11:30.988135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241363621815120:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.993913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241363621815112:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9295) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241363621815113:2115] held: no
2025-12-24T02:11:30.994030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241363621815037:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.994047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.994055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241363621815037:2050] [node 3] ICP22 created new session: [5:7587241363621815127:2050]
2025-12-24T02:11:30.994062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241363621815127:2050] [node 3] ICS09 handshake done sender: [5:7587241363621815120:2119] self: [5:7587241363621815124:0] peer: [3:7587241366268923994:0] socket: 260 qp: -1
2025-12-24T02:11:30.994072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241363621815127:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.994083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:30.994615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241366338788776:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:30.994800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241363621815112:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.994825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:30.994832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241363621815112:2114] [node 4] ICP22 created new session: [5:7587241363621815129:2114]
2025-12-24T02:11:30.994838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241363621815129:2114] [node 4] ICS09 handshake done sender: [5:7587241363621815113:2115] self: [5:7587241363621815123:0] peer: [4:7587241364260399619:0] socket: 219 qp: -1
2025-12-24T02:11:30.994844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241363621815129:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:30.994851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:30.994817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241366338788688:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:30.994834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:30.994840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241366338788688:2050] [node 3] ICP22 created new session: [6:7587241366338788779:2050]
2025-12-24T02:11:30.994847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241366338788779:2050] [node 3] ICS09 handshake done sender: [6:7587241366338788776:2117] self: [6:7587241366338788777:0] peer: [3:7587241366268923991:0] socket: 252 qp: -1
2025-12-24T02:11:30.994854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241366338788779:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:30.994865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.012366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:31.012470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241363421293147:2048] [node 1] ICS01 socket: 207 reason# EndOfStream
2025-12-24T02:11:31.012480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241363421293052:2048] [node 1] ICP30 unregister session Session# [2:7587241363421293147:2048] VirtualId# [2:7587241363421293134:0]
2025-12-24T02:11:31.012488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241363421293147:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.012522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.012544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.012974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:31.013043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241366268924004:2048] [node 1] ICS01 socket: 216 reason# EndOfStream
2025-12-24T02:11:31.013048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241366268923914:2048] [node 1] ICP30 unregister session Session# [3:7587241366268924004:2048] VirtualId# [3:7587241366268923981:0]
2025-12-24T02:11:31.013053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241366268924004:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.013076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.013094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.022764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:31.022866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241366338788771:2048] [node 1] ICS07 socket disconnect 228 reason# EndOfStream
2025-12-24T02:11:31.022873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241366338788771:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.022927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.022950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241366338788771:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.022757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:31.022866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241363621815121:2048] [node 1] ICS07 socket disconnect 243 reason# EndOfStream
2025-12-24T02:11:31.022872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241363621815121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.022919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.022943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241363621815121:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.022981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:31.023047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241364260399607:2048] [node 1] ICS07 socket disconnect 211 reason# EndOfStream
2025-12-24T02:11:31.023052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241364260399607:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.023077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.023099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241364260399607:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.023630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241366338788686:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.023644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241366338788771:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.023649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241366338788686:2048] [node 1] ICP30 unregister session Session# [6:7587241366338788771:2048] VirtualId# [6:7587241366338788766:0]
2025-12-24T02:11:31.023695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.023701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241366338788686:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.023705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.026899Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241364260399537:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.026913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241364260399607:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.026918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241364260399537:2048] [node 1] ICP30 unregister session Session# [4:7587241364260399607:2048] VirtualId# [4:7587241364260399605:0]
2025-12-24T02:11:31.026968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.026975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241364260399537:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.026977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.027246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241363621815035:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.027261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241363621815121:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.027266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241363621815035:2048] [node 1] ICP30 unregister session Session# [5:7587241363621815121:2048] VirtualId# [5:7587241363621815106:0]
2025-12-24T02:11:31.027293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.027299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241363621815035:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.027302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8530) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.030724Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:31.030803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241366268924002:2049] [node 2] ICS07 socket disconnect 237 reason# EndOfStream
2025-12-24T02:11:31.030810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241366268924002:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.030841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.030857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241366268924002:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.033466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241366268923915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:31.033478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241366268924002:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.033726Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241366268923915:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.033735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241366268924002:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.033740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241366268923915:2049] [node 2] ICP30 unregister session Session# [3:7587241366268924002:2049] VirtualId# [3:7587241366268923983:0]
2025-12-24T02:11:31.033761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.033766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241366268923915:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.033769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.034742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241363621815035:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.034771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241366338788686:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.038777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241364260399537:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.038805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:31.038885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241364260399615:2049] [node 2] ICS07 socket disconnect 202 reason# EndOfStream
2025-12-24T02:11:31.038891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241364260399615:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.038930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.038950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241364260399615:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.038803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:31.038890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241363621815107:2049] [node 2] ICS07 socket disconnect 208 reason# EndOfStream
2025-12-24T02:11:31.038896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241363621815107:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.038946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.038969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241363621815107:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.039332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241364260399538:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.039155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:31.039342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241364260399615:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.039348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241364260399538:2049] [node 2] ICP30 unregister session Session# [4:7587241364260399615:2049] VirtualId# [4:7587241364260399604:0]
2025-12-24T02:11:31.039225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241366338788769:2049] [node 2] ICS07 socket disconnect 217 reason# EndOfStream
2025-12-24T02:11:31.039231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241366338788769:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.039394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.039259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.039400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241364260399538:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.039279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241366338788769:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.039402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.039695Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241366338788687:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.039704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241366338788769:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.039708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241366338788687:2049] [node 2] ICP30 unregister session Session# [6:7587241366338788769:2049] VirtualId# [6:7587241366338788765:0]
2025-12-24T02:11:31.039751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.039756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241366338788687:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.039757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.039977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241363621815036:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.039987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241363621815107:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.039992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241363621815036:2049] [node 2] ICP30 unregister session Session# [5:7587241363621815107:2049] VirtualId# [5:7587241363621815104:0]
2025-12-24T02:11:31.040030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.040035Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241363621815036:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.040037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11555) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.046746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241366338788687:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.050820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241363621815036:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.053794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241364260399538:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.062809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:31.062923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241366338788779:2050] [node 3] ICS01 socket: 252 reason# EndOfStream
2025-12-24T02:11:31.062936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241366338788688:2050] [node 3] ICP30 unregister session Session# [6:7587241366338788779:2050] VirtualId# [6:7587241366338788777:0]
2025-12-24T02:11:31.062944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241366338788779:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.063001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.063025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.063063Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:31.063132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241363621815127:2050] [node 3] ICS01 socket: 260 reason# EndOfStream
2025-12-24T02:11:31.063137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241363621815037:2050] [node 3] ICP30 unregister session Session# [5:7587241363621815127:2050] VirtualId# [5:7587241363621815124:0]
2025-12-24T02:11:31.063142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241363621815127:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.063169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.063200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.094851Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:31.094962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241366338788763:2097] [node 4] ICS07 socket disconnect 198 reason# EndOfStream
2025-12-24T02:11:31.094969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241366338788763:2097] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.095015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:31.095044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241366338788763:2097] [node 4] ICS15 start handshake
2025-12-24T02:11:31.095368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241366338788737:2097] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9295) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.095380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241366338788763:2097] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.095385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241366338788737:2097] [node 4] ICP30 unregister session Session# [6:7587241366338788763:2097] VirtualId# [6:7587241366338788741:0]
2025-12-24T02:11:31.095407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:31.095414Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241366338788737:2097] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.095418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9295) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.105299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241366338788737:2097] [node 4] ICP33 wake up from error state
2025-12-24T02:11:31.110821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:31.110929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241366338788758:2099] [node 5] ICS01 socket: 218 reason# EndOfStream
2025-12-24T02:11:31.110938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241366338788744:2099] [node 5] ICP30 unregister session Session# [6:7587241366338788758:2099] VirtualId# [6:7587241366338788745:0]
2025-12-24T02:11:31.110945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241366338788758:2099] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.110976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:31.111005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:31.162890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241369551950572:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.162898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241369551950573:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.154884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241367346634815:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:31.154899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241367346634816:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:31.170857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241367394650883:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.170872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241367394650884:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.170878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241367394650885:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.162802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369357993888:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.162816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369357993889:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:31.182829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371096236826:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.182846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371096236827:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.182852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371096236828:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.185292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241370789020710:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.185304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241370789020711:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.185309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241370789020712:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.243151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241367346634876:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.243000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241370789020770:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:31.244597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369357993949:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.246838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020777:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
2025-12-24T02:11:31.244959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241369551950638:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59942
2025-12-24T02:11:31.247125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241367346634881:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.250780Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369357993953:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22039
2025-12-24T02:11:31.250813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369357993954:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22039
2025-12-24T02:11:31.250896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236888:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.250922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236887:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.257323Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993954:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.255756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650944:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.256314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241369551950641:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:31.256417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241369551950642:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:31.258772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241370789020777:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.259174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241370789020711:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.257753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241369357993888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.261669Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993951:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.261675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369357993951:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.262404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650944:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.261531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241367346634888:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27196
2025-12-24T02:11:31.262739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241367346634883:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.262745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241367346634883:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.264010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241367346634878:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.264019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241367346634878:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.264464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241367346634885:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.264467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241367346634885:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.262837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241367394650883:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.262941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371096236888:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.263331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371096236887:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.263338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371096236826:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.263469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371096236891:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:31.263711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371096236891:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.266769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241367346634888:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.267189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241367346634816:2049] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.268619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950638:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.268671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950635:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.268675Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241369551950635:2108] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.269650Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241369551950648:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59942
2025-12-24T02:11:31.269824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241369551950651:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:31.270247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241369551950651:2115] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.266786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369357993951:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.270662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241367346634878:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.270660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369357993949:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.270674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.270682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369357993949:2107] [node 6] ICP22 created new session: [2:7587241369357993962:2107]
2025-12-24T02:11:31.270591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241370789020772:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.270605Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241370789020772:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.270482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236893:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.270856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241369551950635:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.270690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369357993962:2107] [node 6] ICS09 handshake done sender: [2:7587241369357993951:2108] self: [2:7587241369357993950:0] peer: [6:7587241370789020779:0] socket: 211 qp: -1
2025-12-24T02:11:31.270708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369357993962:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:31.270732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.271770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369357993956:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:31.272199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241370789020775:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.272212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241370789020775:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.272457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241370789020772:2107] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.276777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369357993965:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22039
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.277246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650947:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.277397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236894:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.274747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241369551950572:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.274772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.274779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241369551950572:2048] [node 1] ICP22 created new session: [3:7587241369551950653:2048]
2025-12-24T02:11:31.274786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241369551950653:2048] [node 1] ICS09 handshake done sender: [3:7587241369551950635:2108] self: [3:7587241369551950634:0] peer: [1:7587241367346634890:0] socket: 206 qp: -1
2025-12-24T02:11:31.274794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241369551950653:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.274804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.275862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241369551950655:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59942
2025-12-24T02:11:31.276261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241367346634876:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.276280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.276287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241367346634876:2107] [node 4] ICP22 created new session: [1:7587241367346634892:2107]
2025-12-24T02:11:31.276293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241367346634892:2107] [node 4] ICS09 handshake done sender: [1:7587241367346634878:2108] self: [1:7587241367346634877:0] peer: [4:7587241367394650945:0] socket: 204 qp: -1
2025-12-24T02:11:31.276298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241367346634892:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:31.276319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.278675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371096236893:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.275956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020782:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
2025-12-24T02:11:31.278592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241367346634883:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.278496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241369551950638:2109] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.278932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241367346634815:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.278945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241367346634885:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.278950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.278963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241367346634815:2048] [node 2] ICP22 created new session: [1:7587241367346634897:2048]
2025-12-24T02:11:31.278970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241367346634897:2048] [node 2] ICS09 handshake done sender: [1:7587241367346634883:2110] self: [1:7587241367346634882:0] peer: [2:7587241369357993958:0] socket: 216 qp: -1
2025-12-24T02:11:31.278977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241367346634897:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:31.278990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.279079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241367346634896:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27196
2025-12-24T02:11:31.279460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241367346634881:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.279470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.279476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241367346634881:2109] [node 5] ICP22 created new session: [1:7587241367346634899:2109]
2025-12-24T02:11:31.279482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241367346634899:2109] [node 5] ICS09 handshake done sender: [1:7587241367346634885:2111] self: [1:7587241367346634884:0] peer: [5:7587241371096236889:0] socket: 217 qp: -1
2025-12-24T02:11:31.279487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241367346634899:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:31.279493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.279506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241367346634901:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:31.279531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241367346634888:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.282603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371096236888:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.282742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371096236826:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.282751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.282756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371096236826:2048] [node 1] ICP22 created new session: [5:7587241371096236896:2048]
2025-12-24T02:11:31.282762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371096236896:2048] [node 1] ICS09 handshake done sender: [5:7587241371096236888:2109] self: [5:7587241371096236889:0] peer: [1:7587241367346634884:0] socket: 222 qp: -1
2025-12-24T02:11:31.282769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371096236896:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.282788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.283480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241369551950651:2115] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.283497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.283505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241369551950651:2115] [node 6] ICP22 created new session: [3:7587241369551950657:2115]
2025-12-24T02:11:31.283512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241369551950657:2115] [node 6] ICS09 handshake done sender: [3:7587241369551950638:2109] self: [3:7587241369551950652:0] peer: [6:7587241370789020771:0] socket: 209 qp: -1
2025-12-24T02:11:31.283519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241369551950657:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:31.283536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.283757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950645:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.283761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241369551950645:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.287163Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241369551950645:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.287372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241369551950642:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.287390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.287398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241369551950642:2111] [node 5] ICP22 created new session: [3:7587241369551950661:2111]
2025-12-24T02:11:31.287404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241369551950661:2111] [node 5] ICS09 handshake done sender: [3:7587241369551950645:2112] self: [3:7587241369551950644:0] peer: [5:7587241371096236895:0] socket: 224 qp: -1
2025-12-24T02:11:31.287411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241369551950661:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:31.287420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.287739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241370789020777:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.289096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369357993954:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.290855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241370789020711:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.290877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.290883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241370789020711:2049] [node 2] ICP22 created new session: [6:7587241370789020784:2049]
2025-12-24T02:11:31.290889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241370789020784:2049] [node 2] ICS09 handshake done sender: [6:7587241370789020777:2109] self: [6:7587241370789020779:0] peer: [2:7587241369357993950:0] socket: 212 qp: -1
2025-12-24T02:11:31.290895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241370789020784:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.290905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.291333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241370789020775:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.289166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241369357993888:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.289183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.289190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369357993888:2048] [node 1] ICP22 created new session: [2:7587241369357993970:2048]
2025-12-24T02:11:31.289197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369357993970:2048] [node 1] ICS09 handshake done sender: [2:7587241369357993954:2110] self: [2:7587241369357993958:0] peer: [1:7587241367346634882:0] socket: 219 qp: -1
2025-12-24T02:11:31.289204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369357993970:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.289222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.295053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241367346634816:2049] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.295074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.295083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241367346634816:2049] [node 3] ICP22 created new session: [1:7587241367346634902:2049]
2025-12-24T02:11:31.295089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241367346634902:2049] [node 3] ICS09 handshake done sender: [1:7587241367346634888:2112] self: [1:7587241367346634890:0] peer: [3:7587241369551950634:0] socket: 214 qp: -1
2025-12-24T02:11:31.295095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241367346634902:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:31.295112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.294733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241370789020712:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.294763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.294773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241370789020712:2050] [node 3] ICP22 created new session: [6:7587241370789020786:2050]
2025-12-24T02:11:31.294783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241370789020786:2050] [node 3] ICS09 handshake done sender: [6:7587241370789020772:2107] self: [6:7587241370789020771:0] peer: [3:7587241369551950652:0] socket: 207 qp: -1
2025-12-24T02:11:31.294792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241370789020786:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.294812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.296834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993953:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.296876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993960:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.296880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369357993960:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.297858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369357993960:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.299006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369357993889:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.299027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.299033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369357993889:2049] [node 3] ICP22 created new session: [2:7587241369357993974:2049]
2025-12-24T02:11:31.299038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369357993974:2049] [node 3] ICS09 handshake done sender: [2:7587241369357993960:2112] self: [2:7587241369357993957:0] peer: [3:7587241369551950663:0] socket: 230 qp: -1
2025-12-24T02:11:31.299044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369357993974:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:31.299049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.296478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950648:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.296524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241369551950660:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59942
2025-12-24T02:11:31.296551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241369551950573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.297766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241369551950664:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59942
2025-12-24T02:11:31.298063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241369551950648:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.298111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241369551950573:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.298123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.298129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241369551950573:2049] [node 2] ICP22 created new session: [3:7587241369551950665:2049]
2025-12-24T02:11:31.298135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241369551950665:2049] [node 2] ICS09 handshake done sender: [3:7587241369551950648:2114] self: [3:7587241369551950663:0] peer: [2:7587241369357993957:0] socket: 231 qp: -1
2025-12-24T02:11:31.298142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241369551950665:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.298149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.317795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241367346634881:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(38226131ns) {HandleHandshakeStatus:329(38224925ns) {DropIncomingHandshake:442(126ns) DropOutgoingHandshake:457(153ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(38158662ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 38141354ns max# 38090300ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.323526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650954:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.323602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241367394650944:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.323687Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241369551950572:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(48906123ns) {HandleHandshakeStatus:329(48905775ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(172ns) SwitchToState:179(228ns) ProcessPendingSessionEvents:495(40830215ns) {ForwardSessionEventToSession:612(40817880ns) ForwardSessionEventToSession:612(1288ns) ForwardSessionEventToSession:612(930ns) ForwardSessionEventToSession:612(1095ns) ForwardSessionEventToSession:612(650ns) ForwardSessionEventToSession:612(803ns) ForwardSessionEventToSession:612(653ns) ForwardSessionEventToSession:612(1217ns) ForwardSessionEventToSession:612(887ns) ForwardSessionEventToSession:612(611ns)}}}
2025-12-24T02:11:31.326642Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241370789020711:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(35764277ns) {HandleHandshakeStatus:329(35763651ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(56ns) SwitchToState:179(68ns) ProcessPendingSessionEvents:495(889ns)}}
2025-12-24T02:11:31.326966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371096236828:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.327017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241371096236828:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48221170ns) {Configure:104(48220844ns) {ProcessConfigured:135(47866908ns) {IncomingHandshake:261(47854799ns) {DropIncomingHandshake:442(62ns) PrepareNewSessionHandshake:477(455ns) {SwitchToState:179(100ns)} GenerateSessionVirtualId:988(458ns) IssueIncomingHandshakeReply:193(47848969ns) {SwitchToState:179(570ns)}}}}}
2025-12-24T02:11:31.324035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241367394650884:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41085781ns) {Configure:104(41085281ns) {ProcessConfigured:135(40760147ns) {StartInitialHandshake:157(40759216ns) {DropHandshakes:470(531ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(993ns) {SwitchToState:179(471ns)}}}}}
2025-12-24T02:11:31.325259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650953:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.326744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241367394650950:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:31.326643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369357993967:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:31.326720Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241369357993949:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(55993525ns) {HandleHandshakeStatus:329(55992645ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(360ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(51865723ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 51858714ns max# 51840751ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.330779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650953:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.330814Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650954:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.330899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241367394650883:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.330912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.330918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241367394650883:2048] [node 1] ICP22 created new session: [4:7587241367394650958:2048]
2025-12-24T02:11:31.330924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241367394650958:2048] [node 1] ICS09 handshake done sender: [4:7587241367394650944:2108] self: [4:7587241367394650945:0] peer: [1:7587241367346634877:0] socket: 223 qp: -1
2025-12-24T02:11:31.330930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241367394650958:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.330938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.331516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241367394650885:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.331531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241367394650884:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.331010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241370789020770:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.331028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.331037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241370789020770:2102] [node 5] ICP22 created new session: [6:7587241370789020788:2102]
2025-12-24T02:11:31.331044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241370789020788:2102] [node 5] ICS09 handshake done sender: [6:7587241370789020775:2108] self: [6:7587241370789020774:0] peer: [5:7587241371096236892:0] socket: 208 qp: -1
2025-12-24T02:11:31.331052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241370789020788:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:31.331070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.330920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236898:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.331991Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241369551950642:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(44549125ns) {HandleHandshakeStatus:329(44548063ns) {DropIncomingHandshake:442(131ns) DropOutgoingHandshake:457(413ns) SwitchToState:179(372ns) ProcessPendingSessionEvents:495(44454533ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 44441449ns max# 44402416ns}}}
2025-12-24T02:11:31.334839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950660:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.334881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241369551950647:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.334895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241369551950647:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.335623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993968:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.335635Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369357993968:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.336372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369357993968:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.336463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369357993956:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.336479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.336486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369357993956:2111] [node 4] ICP22 created new session: [2:7587241369357993978:2111]
2025-12-24T02:11:31.336493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369357993978:2111] [node 4] ICS09 handshake done sender: [2:7587241369357993968:2115] self: [2:7587241369357993964:0] peer: [4:7587241367394650961:0] socket: 245 qp: -1
2025-12-24T02:11:31.336500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369357993978:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:31.336508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.338834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241369551950647:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.339033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241369551950641:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.339056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
No RDMA context found for GID: 2025-12-24T02:11:31.339062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241369551950641:2110] [node 4] ICP22 created new session: [3:7587241369551950669:2110]
2025-12-24T02:11:31.339069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241369551950669:2110] [node 4] ICS09 handshake done sender: [3:7587241369551950647:2113] self: [3:7587241369551950643:0] peer: [4:7587241367394650960:0] socket: 240 qp: -1
2025-12-24T02:11:31.339074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241369551950669:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:31.339086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.338839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020790:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
2025-12-24T02:11:31.339017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241370789020712:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(64247019ns) {HandleHandshakeStatus:329(64246279ns) {DropIncomingHandshake:442(1462ns) DropOutgoingHandshake:457(282ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(12257822ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 12252186ns max# 12234423ns}}}
2025-12-24T02:11:31.339387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241370789020790:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.339844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241370789020710:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.338929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371096236893:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.341962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371096236828:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.341979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.341986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371096236828:2050] [node 3] ICP22 created new session: [5:7587241371096236900:2050]
2025-12-24T02:11:31.341991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371096236900:2050] [node 3] ICS09 handshake done sender: [5:7587241371096236893:2111] self: [5:7587241371096236895:0] peer: [3:7587241369551950644:0] socket: 225 qp: -1
2025-12-24T02:11:31.341996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371096236900:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.342002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
::ffff:127.1.0.0
2025-12-24T02:11:31.338856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650962:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.342273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650963:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.342387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241367394650954:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.342422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241367394650884:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.342430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.342435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241367394650884:2049] [node 2] ICP22 created new session: [4:7587241367394650964:2049]
2025-12-24T02:11:31.342440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241367394650964:2049] [node 2] ICS09 handshake done sender: [4:7587241367394650954:2113] self: [4:7587241367394650961:0] peer: [2:7587241369357993964:0] socket: 205 qp: -1
2025-12-24T02:11:31.342445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241367394650964:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.342480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.342788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241369357993967:2114] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16009254ns) {Configure:104(16008331ns) {ProcessConfigured:135(15649232ns) {StartInitialHandshake:157(15648734ns) {DropHandshakes:470(1122ns) {DropIncomingHandshake:442(313ns) DropOutgoingHandshake:457(293ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(684ns) {SwitchToState:179(483ns)}}}}}
2025-12-24T02:11:31.342778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241367346634905:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.342790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241367346634905:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.346802Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369357993982:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22039
2025-12-24T02:11:31.346985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241367394650950:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20170843ns) {Configure:104(20170158ns) {ProcessConfigured:135(19819423ns) {StartInitialHandshake:157(19819041ns) {DropHandshakes:470(523ns) {DropIncomingHandshake:442(91ns) DropOutgoingHandshake:457(113ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(828ns) {SwitchToState:179(324ns)}}}}}
2025-12-24T02:11:31.347007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241367394650953:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.346873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236902:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.347172Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236899:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.350813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371096236887:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.352287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020792:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
2025-12-24T02:11:31.354836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241367346634905:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.354936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241367346634901:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.354949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.354957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241367346634901:2113] [node 6] ICP22 created new session: [1:7587241367346634908:2113]
2025-12-24T02:11:31.354974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241367346634908:2113] [node 6] ICS09 handshake done sender: [1:7587241367346634905:2115] self: [1:7587241367346634904:0] peer: [6:7587241370789020791:0] socket: 248 qp: -1
2025-12-24T02:11:31.354981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241367346634908:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:31.354993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.356379Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241370789020790:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.356472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241370789020710:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.356485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.356492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241370789020710:2048] [node 1] ICP22 created new session: [6:7587241370789020793:2048]
2025-12-24T02:11:31.356499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241370789020793:2048] [node 1] ICS09 handshake done sender: [6:7587241370789020790:2113] self: [6:7587241370789020791:0] peer: [1:7587241367346634904:0] socket: 252 qp: -1
2025-12-24T02:11:31.356505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241370789020793:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.356514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.362983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236903:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.363353Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371096236902:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.363824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371096236827:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.364383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369357993980:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.364395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369357993980:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.364962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241369357993956:2111] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:31.365224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369357993980:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.365307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369357993967:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.365322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.365329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369357993967:2114] [node 5] ICP22 created new session: [2:7587241369357993984:2114]
2025-12-24T02:11:31.365336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369357993984:2114] [node 5] ICS09 handshake done sender: [2:7587241369357993980:2116] self: [2:7587241369357993976:0] peer: [5:7587241371096236904:0] socket: 244 qp: -1
2025-12-24T02:11:31.365342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369357993984:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:31.365352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.365751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236905:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.365923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371096236902:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.366029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371096236827:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.366042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.366047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371096236827:2049] [node 2] ICP22 created new session: [5:7587241371096236906:2049]
2025-12-24T02:11:31.366053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371096236906:2049] [node 2] ICS09 handshake done sender: [5:7587241371096236902:2115] self: [5:7587241371096236904:0] peer: [2:7587241369357993976:0] socket: 258 qp: -1
2025-12-24T02:11:31.366059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371096236906:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.366065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.370902Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241367394650885:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241367394650953:2112] held: no
2025-12-24T02:11:31.371092Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241367394650885:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23881212ns) {HandleHandshakeStatus:395(23880653ns) {LogHandshakeFail:472(3333ns) {UpdateErrorStateLog:545(864ns)} DropOutgoingHandshake:457(698ns) ProcessPendingSessionEvents:495(78300ns) {unordered DropSessionEvent:512 num# 70 sum# 72213ns max# 8365ns}}}
2025-12-24T02:11:31.375661Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241367346634901:2113] [node 6] ICP35 event processing took too much time PendingConnection:279(20636247ns) {HandleHandshakeStatus:329(20634991ns) {DropIncomingHandshake:442(269ns) DropOutgoingHandshake:457(327ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(20543190ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 20527140ns max# 20480382ns}}}
2025-12-24T02:11:31.376738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241367394650885:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.376763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.376797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241367394650885:2050] [node 3] ICP22 created new session: [4:7587241367394650968:2050]
2025-12-24T02:11:31.376803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241367394650968:2050] [node 3] ICS09 handshake done sender: [4:7587241367394650953:2112] self: [4:7587241367394650960:0] peer: [3:7587241369551950643:0] socket: 241 qp: -1
2025-12-24T02:11:31.376810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241367394650968:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.376818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.379289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371096236891:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.379313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.379322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371096236891:2110] [node 6] ICP22 created new session: [5:7587241371096236908:2110]
2025-12-24T02:11:31.379331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371096236908:2110] [node 6] ICS09 handshake done sender: [5:7587241371096236887:2108] self: [5:7587241371096236892:0] peer: [6:7587241370789020774:0] socket: 220 qp: -1
2025-12-24T02:11:31.379338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371096236908:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:31.379364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.383477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371096236903:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.383734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371096236911:2118] [node 4] ICP01 ready to work
2025-12-24T02:11:31.384130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371096236911:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.387629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241371096236891:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(32638683ns) {HandleHandshakeStatus:329(32638107ns) {DropIncomingHandshake:442(710ns) DropOutgoingHandshake:457(67ns) SwitchToState:179(221ns) ProcessPendingSessionEvents:495(578ns)}}
2025-12-24T02:11:31.388209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241367394650971:2118] [node 6] ICP01 ready to work
2025-12-24T02:11:31.390808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650966:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.390825Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241367394650966:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.404106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241367394650966:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.404223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241367394650950:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.404239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.404252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241367394650950:2109] [node 5] ICP22 created new session: [4:7587241367394650974:2109]
2025-12-24T02:11:31.404259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241367394650974:2109] [node 5] ICS09 handshake done sender: [4:7587241367394650966:2117] self: [4:7587241367394650957:0] peer: [5:7587241371096236912:0] socket: 242 qp: -1
2025-12-24T02:11:31.404267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241367394650974:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:31.404275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.407725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241367394650971:2118] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19434644ns) {Configure:104(19433536ns) {ProcessConfigured:135(19068303ns) {StartInitialHandshake:157(19067165ns) {DropHandshakes:470(1149ns) {DropIncomingHandshake:442(309ns) DropOutgoingHandshake:457(327ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(1196ns) {SwitchToState:179(506ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.414997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241370789020796:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:31.415283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020797:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.423333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371096236913:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50679
2025-12-24T02:11:31.426877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371096236903:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.426953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371096236911:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.426969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.426979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371096236911:2118] [node 4] ICP22 created new session: [5:7587241371096236914:2118]
2025-12-24T02:11:31.426987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371096236914:2118] [node 4] ICS09 handshake done sender: [5:7587241371096236903:2116] self: [5:7587241371096236912:0] peer: [4:7587241367394650957:0] socket: 262 qp: -1
2025-12-24T02:11:31.426995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371096236914:2118] [node 4] ICS10 traffic start
2025-12-24T02:11:31.427008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.429911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241370789020797:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.429965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241370789020796:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.442892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650976:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.442933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241367394650976:2119] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.444496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241367394650979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5663
2025-12-24T02:11:31.444634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241370789020802:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27504
2025-12-24T02:11:31.450984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241367394650976:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.451153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241367394650971:2118] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.451168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241367394650979:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.451173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.451181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241367394650971:2118] [node 6] ICP22 created new session: [4:7587241367394650980:2118]
2025-12-24T02:11:31.451189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241367394650980:2118] [node 6] ICS09 handshake done sender: [4:7587241367394650976:2119] self: [4:7587241367394650972:0] peer: [6:7587241370789020801:0] socket: 265 qp: -1
2025-12-24T02:11:31.451195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241367394650980:2118] [node 6] ICS10 traffic start
2025-12-24T02:11:31.451206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.451377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241367394650971:2118] [node 6] ICP18 handshake (actor [4:7587241367394650979:2120]) is too old
2025-12-24T02:11:31.451945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241370789020797:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.452082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241370789020796:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.452099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.452105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241370789020796:2115] [node 4] ICP22 created new session: [6:7587241370789020803:2115]
2025-12-24T02:11:31.452111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241370789020803:2115] [node 4] ICS09 handshake done sender: [6:7587241370789020797:2116] self: [6:7587241370789020801:0] peer: [4:7587241367394650972:0] socket: 266 qp: -1
2025-12-24T02:11:31.452118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241370789020803:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:31.452131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.467401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:31.467496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241369357993970:2048] [node 1] ICS01 socket: 219 reason# EndOfStream
2025-12-24T02:11:31.467504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241369357993888:2048] [node 1] ICP30 unregister session Session# [2:7587241369357993970:2048] VirtualId# [2:7587241369357993958:0]
2025-12-24T02:11:31.467511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241369357993970:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.467548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.467571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.468240Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:31.468325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241369551950653:2048] [node 1] ICS07 socket disconnect 206 reason# EndOfStream
2025-12-24T02:11:31.468334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241369551950653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.468383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.468403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241369551950653:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.474959Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241369551950572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.474980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241369551950653:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.474988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241369551950572:2048] [node 1] ICP30 unregister session Session# [3:7587241369551950653:2048] VirtualId# [3:7587241369551950634:0]
2025-12-24T02:11:31.475018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.475024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241369551950572:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.475027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.482622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:31.482730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241371096236896:2048] [node 1] ICS07 socket disconnect 222 reason# EndOfStream
2025-12-24T02:11:31.482737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371096236896:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.479197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:31.482791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.479288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241367394650958:2048] [node 1] ICS01 socket: 223 reason# EndOfStream
2025-12-24T02:11:31.479296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241367394650883:2048] [node 1] ICP30 unregister session Session# [4:7587241367394650958:2048] VirtualId# [4:7587241367394650945:0]
2025-12-24T02:11:31.479303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241367394650958:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.479361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.479393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.482852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:31.482921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241370789020793:2048] [node 1] ICS07 socket disconnect 252 reason# EndOfStream
2025-12-24T02:11:31.482926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241370789020793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.483105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.483129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241370789020793:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.483536Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241370789020710:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.483549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241370789020793:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.483556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241370789020710:2048] [node 1] ICP30 unregister session Session# [6:7587241370789020793:2048] VirtualId# [6:7587241370789020791:0]
2025-12-24T02:11:31.483610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.483615Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241370789020710:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.483618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.482818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241371096236896:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.486763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241369551950572:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.486932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241371096236826:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.486949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371096236896:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.486955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371096236826:2048] [node 1] ICP30 unregister session Session# [5:7587241371096236896:2048] VirtualId# [5:7587241371096236889:0]
2025-12-24T02:11:31.486999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.487008Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241371096236826:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.487012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15466) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.487922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:31.487994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241369551950665:2049] [node 2] ICS07 socket disconnect 231 reason# EndOfStream
2025-12-24T02:11:31.488000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241369551950665:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.488032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.488049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241369551950665:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.494784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241370789020710:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.502780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241371096236826:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.506686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:31.506803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241367394650964:2049] [node 2] ICS07 socket disconnect 205 reason# EndOfStream
2025-12-24T02:11:31.506809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241367394650964:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.506881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:31.506960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241370789020784:2049] [node 2] ICS07 socket disconnect 212 reason# EndOfStream
2025-12-24T02:11:31.506966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241370789020784:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.507017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.507038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241370789020784:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.506865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.506899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241367394650964:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.507191Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:31.507256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241371096236906:2049] [node 2] ICS07 socket disconnect 258 reason# EndOfStream
2025-12-24T02:11:31.507261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371096236906:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.507293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.507315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241371096236906:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.507540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241367394650884:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.507552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241367394650964:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.507557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241367394650884:2049] [node 2] ICP30 unregister session Session# [4:7587241367394650964:2049] VirtualId# [4:7587241367394650961:0]
2025-12-24T02:11:31.507571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.507582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241367394650884:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.507586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.507766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241370789020711:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.507776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241370789020784:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.507780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241370789020711:2049] [node 2] ICP30 unregister session Session# [6:7587241370789020784:2049] VirtualId# [6:7587241370789020779:0]
2025-12-24T02:11:31.507819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.507824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241370789020711:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.507826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.510909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241371096236827:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.510923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371096236906:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.510927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371096236827:2049] [node 2] ICP30 unregister session Session# [5:7587241371096236906:2049] VirtualId# [5:7587241371096236904:0]
2025-12-24T02:11:31.510968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.510973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241371096236827:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.510976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5974) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.518862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241371096236827:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.518800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241367394650884:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.522830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241370789020711:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.610784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:31.610901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241370789020786:2050] [node 3] ICS07 socket disconnect 207 reason# EndOfStream
2025-12-24T02:11:31.610908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241370789020786:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.610980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.611012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241370789020786:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:31.610943Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:31.611035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241371096236900:2050] [node 3] ICS07 socket disconnect 225 reason# EndOfStream
2025-12-24T02:11:31.611041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371096236900:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.611099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.611124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241371096236900:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:31.611509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241371096236828:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.611520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371096236900:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.611525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371096236828:2050] [node 3] ICP30 unregister session Session# [5:7587241371096236900:2050] VirtualId# [5:7587241371096236895:0]
2025-12-24T02:11:31.611569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.611574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241371096236828:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.611577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.611475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241370789020712:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.611493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241370789020786:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.611502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241370789020712:2050] [node 3] ICP30 unregister session Session# [6:7587241370789020786:2050] VirtualId# [6:7587241370789020771:0]
2025-12-24T02:11:31.611544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.611551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241370789020712:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.611553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9962) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.622761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241371096236828:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:31.622815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241370789020712:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:31.638806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:31.638908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241370789020803:2115] [node 4] ICS01 socket: 266 reason# EndOfStream
2025-12-24T02:11:31.638921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241370789020796:2115] [node 4] ICP30 unregister session Session# [6:7587241370789020803:2115] VirtualId# [6:7587241370789020801:0]
2025-12-24T02:11:31.638928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241370789020803:2115] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.638982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:31.639015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:31.674585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241370791415766:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:31.674592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241370791415767:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:31.677194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369972354541:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.677205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369972354542:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:31.681772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241368234805753:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.681828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241368234805754:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.681856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241368234805755:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.685342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241367845346651:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.685352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241367845346653:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.685481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241367845346652:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.688033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371061818994:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.688177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371061818996:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:31.688243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371061818995:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.678897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241370705248717:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:31.678902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241370705248718:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:31.731854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371061819056:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.732782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346712:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
2025-12-24T02:11:31.733334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241370791415833:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:31.734062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241370791415838:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:31.734606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241370791415836:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.735019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248777:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.736420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819060:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.738114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346713:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.738842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248778:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
2025-12-24T02:11:31.737002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369972354607:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:31.737131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369972354608:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:31.738567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241369972354616:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:31.738103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805814:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.744375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805815:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.750193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354620:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.743024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241370705248780:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.751040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.751574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241370705248791:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:31.759996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241370705248794:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.770087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241368234805824:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:31.770739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.770799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241370791415845:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50753
2025-12-24T02:11:31.770852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241370791415846:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50753
2025-12-24T02:11:31.773157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241368234805827:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.782920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248798:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.783609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805831:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.782929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241367845346719:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:31.783991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241367845346725:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:31.784673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371061819063:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.785329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805832:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.790945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241368234805824:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20764971ns) {Configure:104(20764450ns) {ProcessConfigured:135(20517748ns) {StartInitialHandshake:157(20517390ns) {DropHandshakes:470(634ns) {DropIncomingHandshake:442(110ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(1338ns) {SwitchToState:179(681ns)}}}}}
2025-12-24T02:11:31.810837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241368234805827:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37577790ns) {Configure:104(37576614ns) {ProcessConfigured:135(37248837ns) {StartInitialHandshake:157(37248118ns) {DropHandshakes:470(470ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(120ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1000ns) {SwitchToState:179(552ns)}}}}}
2025-12-24T02:11:31.811097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241371061818995:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23808452ns) {Configure:104(23807841ns) {ProcessConfigured:135(23316316ns) {StartInitialHandshake:157(23315680ns) {DropHandshakes:470(236ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(163ns) PrepareNewSessionHandshake:477(828ns) {SwitchToState:179(188ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.818817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805815:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.818869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241368234805755:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.822779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241370791415847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50753
2025-12-24T02:11:31.822825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805832:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.826812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241367845346653:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48875047ns) {Configure:104(48874440ns) {ProcessConfigured:135(48626139ns) {StartInitialHandshake:157(48625610ns) {DropHandshakes:470(478ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(95ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:477(1208ns) {SwitchToState:179(292ns)}}}}}
2025-12-24T02:11:31.830769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248786:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.830795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241370705248786:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.831523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241370705248786:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.831683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241370705248780:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.831698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.831706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241370705248780:2109] [node 4] ICP22 created new session: [3:7587241370705248800:2109]
2025-12-24T02:11:31.831712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241370705248800:2109] [node 4] ICS09 handshake done sender: [3:7587241370705248786:2112] self: [3:7587241370705248785:0] peer: [4:7587241368234805835:0] socket: 229 qp: -1
2025-12-24T02:11:31.831720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241370705248800:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:31.831741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.833313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.833573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241368234805815:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.833647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241368234805755:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.833670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.833684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241368234805755:2050] [node 3] ICP22 created new session: [4:7587241368234805838:2050]
2025-12-24T02:11:31.833690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241368234805838:2050] [node 3] ICS09 handshake done sender: [4:7587241368234805815:2109] self: [4:7587241368234805835:0] peer: [3:7587241370705248785:0] socket: 230 qp: -1
2025-12-24T02:11:31.833697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241368234805838:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.833719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.835623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819073:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.834959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248778:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.835011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241370705248718:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.839254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354603:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.839280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369972354603:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.838894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241370705248780:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:31.839190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248804:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
2025-12-24T02:11:31.839260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248777:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.839290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241370705248717:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.840151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354620:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.840209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241369972354541:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.839812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241371061818994:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23770520ns) {Configure:104(23770002ns) {ProcessConfigured:135(23319849ns) {StartInitialHandshake:157(23319636ns) {DropHandshakes:470(540ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(436ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(175ns)}}}}}
2025-12-24T02:11:31.843527Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369972354603:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.845291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346731:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.842929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415830:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.842950Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241370791415830:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.845348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415827:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.845353Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241370791415827:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.845553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415846:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.846258Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241370791415827:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.846344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241370791415766:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.846356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.846368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241370791415766:2048] [node 2] ICP22 created new session: [1:7587241370791415850:2048]
2025-12-24T02:11:31.846377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241370791415850:2048] [node 2] ICS09 handshake done sender: [1:7587241370791415827:2107] self: [1:7587241370791415826:0] peer: [2:7587241369972354623:0] socket: 212 qp: -1
2025-12-24T02:11:31.846384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241370791415850:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:31.846402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.843644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369972354542:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.843662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.843670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369972354542:2049] [node 3] ICP22 created new session: [2:7587241369972354625:2049]
2025-12-24T02:11:31.843676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369972354625:2049] [node 3] ICS09 handshake done sender: [2:7587241369972354603:2107] self: [2:7587241369972354602:0] peer: [3:7587241370705248803:0] socket: 219 qp: -1
2025-12-24T02:11:31.843683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369972354625:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:31.843700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.849526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248806:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
2025-12-24T02:11:31.849683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241370705248778:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.849592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346732:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
2025-12-24T02:11:31.846880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241370791415830:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.849792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346731:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.849837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346732:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.849884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241367845346719:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.855014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346735:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.858874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805841:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.851255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241370791415767:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.851276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.851289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241370791415767:2049] [node 3] ICP22 created new session: [1:7587241370791415854:2049]
2025-12-24T02:11:31.851296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241370791415854:2049] [node 3] ICS09 handshake done sender: [1:7587241370791415830:2108] self: [1:7587241370791415829:0] peer: [3:7587241370705248805:0] socket: 213 qp: -1
2025-12-24T02:11:31.851303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241370791415854:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:31.851313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.855133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415847:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.862788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805833:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.862815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241368234805833:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.862829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371061819060:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.862991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371061818994:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.863436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346712:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.866441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415840:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.866457Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241370791415840:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.867178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346737:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
2025-12-24T02:11:31.866931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241370705248718:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.866958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.866967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241370705248718:2049] [node 2] ICP22 created new session: [3:7587241370705248807:2049]
2025-12-24T02:11:31.866975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241370705248807:2049] [node 2] ICS09 handshake done sender: [3:7587241370705248778:2108] self: [3:7587241370705248803:0] peer: [2:7587241369972354602:0] socket: 226 qp: -1
2025-12-24T02:11:31.866984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241370705248807:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.866994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.867542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248795:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.867546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241370705248795:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.868429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241370705248795:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.868859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241368234805833:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.869000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241368234805824:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.869014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.869023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241368234805824:2112] [node 5] ICP22 created new session: [4:7587241368234805843:2112]
2025-12-24T02:11:31.869030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241368234805843:2112] [node 5] ICS09 handshake done sender: [4:7587241368234805833:2117] self: [4:7587241368234805825:0] peer: [5:7587241367845346733:0] socket: 255 qp: -1
2025-12-24T02:11:31.869037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241368234805843:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:31.869052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.874977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354628:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.875162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241370705248791:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.875181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.875190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241370705248791:2113] [node 5] ICP22 created new session: [3:7587241370705248810:2113]
2025-12-24T02:11:31.875200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241370705248810:2113] [node 5] ICS09 handshake done sender: [3:7587241370705248795:2115] self: [3:7587241370705248792:0] peer: [5:7587241367845346734:0] socket: 253 qp: -1
2025-12-24T02:11:31.875209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241370705248810:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:31.875218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.874905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819076:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.875641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248804:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.875777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241367845346653:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.875814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241367845346653:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25913710ns) {IncomingHandshake:261(25913238ns) {DropIncomingHandshake:442(152ns) PrepareNewSessionHandshake:477(551ns) {SwitchToState:179(113ns)} GenerateSessionVirtualId:988(496ns) IssueIncomingHandshakeReply:193(25906429ns) {SwitchToState:179(153ns)}}}
2025-12-24T02:11:31.875840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241367845346732:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.879136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241367845346719:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.879159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.879167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241367845346719:2110] [node 4] ICP22 created new session: [5:7587241367845346738:2110]
2025-12-24T02:11:31.879174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241367845346738:2110] [node 4] ICS09 handshake done sender: [5:7587241367845346732:2118] self: [5:7587241367845346733:0] peer: [4:7587241368234805825:0] socket: 239 qp: -1
2025-12-24T02:11:31.879182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241367845346738:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:31.879219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.882050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369972354620:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.882383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241369972354541:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.882407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.882416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369972354541:2048] [node 1] ICP22 created new session: [2:7587241369972354629:2048]
2025-12-24T02:11:31.882422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369972354629:2048] [node 1] ICS09 handshake done sender: [2:7587241369972354620:2115] self: [2:7587241369972354623:0] peer: [1:7587241370791415826:0] socket: 215 qp: -1
2025-12-24T02:11:31.882431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369972354629:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.882439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.884023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346735:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.884060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241367845346651:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.884198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415844:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.884208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241370791415844:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.884780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346713:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.884823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241367845346652:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.885339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241370791415844:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.885463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241370791415840:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.885540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415845:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.885563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241370791415836:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.885569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241370791415838:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.885578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.885579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.885585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241370791415836:2110] [node 5] ICP22 created new session: [1:7587241370791415860:2110]
2025-12-24T02:11:31.885588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241370791415838:2111] [node 6] ICP22 created new session: [1:7587241370791415861:2111]
2025-12-24T02:11:31.885591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241370791415860:2110] [node 5] ICS09 handshake done sender: [1:7587241370791415844:2114] self: [1:7587241370791415843:0] peer: [5:7587241367845346740:0] socket: 266 qp: -1
2025-12-24T02:11:31.885596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241370791415861:2111] [node 6] ICS09 handshake done sender: [1:7587241370791415840:2112] self: [1:7587241370791415839:0] peer: [6:7587241371061819074:0] socket: 214 qp: -1
2025-12-24T02:11:31.885598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241370791415860:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:31.885603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241370791415861:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:31.885605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.885612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.885876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241370791415767:2049] [node 3] ICP18 handshake (actor [1:7587241370791415845:2115]) is too old
2025-12-24T02:11:31.885332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805831:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.887189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241368234805824:2112] [node 5] ICP35 event processing took too much time PendingConnection:279(18074616ns) {HandleHandshakeStatus:329(18073785ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(190ns) SwitchToState:179(189ns) ProcessPendingSessionEvents:495(17982130ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17954897ns max# 17880474ns}}}
2025-12-24T02:11:31.887264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805841:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.887287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241368234805753:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.887408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241368234805824:2112] [node 5] ICP18 handshake (actor [4:7587241368234805831:2115]) is too old
2025-12-24T02:11:31.887521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241370791415842:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.887531Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241370791415842:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.888195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241370791415842:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.891064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805847:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.891512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241368234805841:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.891624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241368234805753:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.891638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.891649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241368234805753:2048] [node 1] ICP22 created new session: [4:7587241368234805848:2048]
2025-12-24T02:11:31.891656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241368234805848:2048] [node 1] ICS09 handshake done sender: [4:7587241368234805841:2120] self: [4:7587241368234805846:0] peer: [1:7587241370791415835:0] socket: 263 qp: -1
2025-12-24T02:11:31.891665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241368234805848:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.891685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.894889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346743:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
2025-12-24T02:11:31.894971Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354612:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.894988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369972354612:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.894864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241370705248812:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57711
2025-12-24T02:11:31.895048Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346742:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
2025-12-24T02:11:31.894935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819075:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.895171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241370705248777:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.895020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805814:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.895144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241368234805754:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.895235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354622:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.895294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371061819075:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.895341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371061818995:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.895567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241367845346731:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.895698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241367845346735:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.895812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241367845346651:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.895833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.895839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241367845346651:2048] [node 1] ICP22 created new session: [5:7587241367845346744:2048]
2025-12-24T02:11:31.895845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241367845346744:2048] [node 1] ICS09 handshake done sender: [5:7587241367845346735:2119] self: [5:7587241367845346740:0] peer: [1:7587241370791415843:0] socket: 267 qp: -1
2025-12-24T02:11:31.895852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241367845346744:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.895874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.899216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354618:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.899232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369972354618:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.899344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241370791415833:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.899383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.899393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241370791415833:2109] [node 4] ICP22 created new session: [1:7587241370791415865:2109]
2025-12-24T02:11:31.899406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241370791415865:2109] [node 4] ICS09 handshake done sender: [1:7587241370791415842:2113] self: [1:7587241370791415835:0] peer: [4:7587241368234805846:0] socket: 262 qp: -1
2025-12-24T02:11:31.899414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241370791415865:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:31.899444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.899904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371061819076:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.899954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371061819063:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.900379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819077:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.901264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354610:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.901286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241369972354610:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.902740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241370705248717:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.902773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.902785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241370705248717:2048] [node 1] ICP22 created new session: [3:7587241370705248815:2048]
2025-12-24T02:11:31.902794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241370705248815:2048] [node 1] ICS09 handshake done sender: [3:7587241370705248777:2107] self: [3:7587241370705248805:0] peer: [1:7587241370791415829:0] socket: 217 qp: -1
2025-12-24T02:11:31.902811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241370705248815:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.902834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:31.904322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241370705248717:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:31.904369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241367845346653:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.904390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371061819073:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.904392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.904399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241367845346653:2050] [node 3] ICP22 created new session: [5:7587241367845346747:2050]
2025-12-24T02:11:31.904406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241367845346747:2050] [node 3] ICS09 handshake done sender: [5:7587241367845346731:2117] self: [5:7587241367845346734:0] peer: [3:7587241370705248792:0] socket: 256 qp: -1
2025-12-24T02:11:31.904411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241367845346747:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.904418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.904457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371061819056:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.904951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369972354618:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.905106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369972354616:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.905136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.905151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369972354616:2112] [node 6] ICP22 created new session: [2:7587241369972354633:2112]
2025-12-24T02:11:31.905158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369972354633:2112] [node 6] ICS09 handshake done sender: [2:7587241369972354618:2113] self: [2:7587241369972354617:0] peer: [6:7587241371061819078:0] socket: 258 qp: -1
2025-12-24T02:11:31.905164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369972354633:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:31.905199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.905561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369972354612:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.905653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369972354608:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.905668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.905675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369972354608:2109] [node 5] ICP22 created new session: [2:7587241369972354636:2109]
2025-12-24T02:11:31.905681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369972354636:2109] [node 5] ICS09 handshake done sender: [2:7587241369972354612:2111] self: [2:7587241369972354611:0] peer: [5:7587241367845346741:0] socket: 222 qp: -1
2025-12-24T02:11:31.905687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369972354636:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:31.905694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.905739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371061819077:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.905791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371061818996:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:31.905932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241368234805834:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.905951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241368234805834:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.907928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241370705248797:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.907947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241370705248797:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.907930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241369972354610:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:31.908002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819082:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.908225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371061819060:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:31.908334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371061818994:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.908362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:31.908368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371061818994:2048] [node 1] ICP22 created new session: [6:7587241371061819083:2048]
2025-12-24T02:11:31.908373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371061819083:2048] [node 1] ICS09 handshake done sender: [6:7587241371061819060:2110] self: [6:7587241371061819074:0] peer: [1:7587241370791415839:0] socket: 216 qp: -1
2025-12-24T02:11:31.908379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371061819083:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:31.908396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.908940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354639:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.909048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241369972354542:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:31.909079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241368234805834:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.909185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241368234805827:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.909211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.909222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241368234805827:2113] [node 6] ICP22 created new session: [4:7587241368234805852:2113]
2025-12-24T02:11:31.909184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241367845346727:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.909210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241367845346727:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:31.909293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241369972354607:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.909337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.909349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241369972354607:2108] [node 4] ICP22 created new session: [2:7587241369972354640:2108]
2025-12-24T02:11:31.909356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241369972354640:2108] [node 4] ICS09 handshake done sender: [2:7587241369972354610:2110] self: [2:7587241369972354609:0] peer: [4:7587241368234805850:0] socket: 221 qp: -1
2025-12-24T02:11:31.909366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241369972354640:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:31.909388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.909236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241368234805852:2113] [node 6] ICS09 handshake done sender: [4:7587241368234805834:2118] self: [4:7587241368234805828:0] peer: [6:7587241371061819079:0] socket: 271 qp: -1
2025-12-24T02:11:31.909245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241368234805852:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:31.909253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.909942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241370791415833:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(20469664ns) {HandleHandshakeStatus:329(20468988ns) {DropIncomingHandshake:442(10870ns) DropOutgoingHandshake:457(255ns) SwitchToState:179(105ns) ProcessPendingSessionEvents:495(2185046ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 2161991ns max# 2087865ns}}}
2025-12-24T02:11:31.909972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241370705248797:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.910070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241370705248794:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.910091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:31.910108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241370705248794:2114] [node 6] ICP22 created new session: [3:7587241370705248818:2114]
2025-12-24T02:11:31.910120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241370705248818:2114] [node 6] ICS09 handshake done sender: [3:7587241370705248797:2116] self: [3:7587241370705248796:0] peer: [6:7587241371061819081:0] socket: 274 qp: -1
2025-12-24T02:11:31.910128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241370705248818:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:31.910143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.910332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371061819063:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62360) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241371061819076:2118] held: no
2025-12-24T02:11:31.910501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241368234805754:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241368234805814:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.911162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241367845346727:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:31.911308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241367845346725:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:31.911322Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241367845346749:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27207
No RDMA context found for GID: 2025-12-24T02:11:31.911285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241368234805854:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39155
2025-12-24T02:11:31.911339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.02025-12-24T02:11:31.911351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241367845346725:2111] [node 6] ICP22 created new session: [5:7587241367845346751:2111]
2025-12-24T02:11:31.911436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241367845346751:2111] [node 6] ICS09 handshake done sender: [5:7587241367845346727:2114] self: [5:7587241367845346726:0] peer: [6:7587241371061819080:0] socket: 245 qp: -1
2025-12-24T02:11:31.911446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241367845346751:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:31.911458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:31.911650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241367845346713:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.911708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354639:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.911739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241369972354608:2109] [node 5] ICP18 handshake (actor [2:7587241369972354639:2119]) is too old
2025-12-24T02:11:31.911881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241368234805814:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.911961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241368234805754:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.911977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.911989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241368234805754:2049] [node 2] ICP22 created new session: [4:7587241368234805855:2049]
2025-12-24T02:11:31.911999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241368234805855:2049] [node 2] ICS09 handshake done sender: [4:7587241368234805814:2108] self: [4:7587241368234805850:0] peer: [2:7587241369972354609:0] socket: 223 qp: -1
2025-12-24T02:11:31.912006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241368234805855:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.912014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.913141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241367845346652:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.913179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.913186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241367845346652:2049] [node 2] ICP22 created new session: [5:7587241367845346753:2049]
2025-12-24T02:11:31.913191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241367845346753:2049] [node 2] ICS09 handshake done sender: [5:7587241367845346713:2109] self: [5:7587241367845346741:0] peer: [2:7587241369972354611:0] socket: 224 qp: -1
2025-12-24T02:11:31.913197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241367845346753:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:31.913204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.913233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819086:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.913212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241369972354642:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40711
2025-12-24T02:11:31.913973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371061818996:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241371061819077:2119] held: no
2025-12-24T02:11:31.914803Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371061819056:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18282) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241371061819073:2116] held: no
2025-12-24T02:11:31.914867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371061819075:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:31.914845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241369972354642:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:31.914895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241369972354616:2112] [node 6] ICP18 handshake (actor [2:7587241369972354642:2120]) is too old
2025-12-24T02:11:31.915496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371061818995:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.915514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:31.915520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371061818995:2049] [node 2] ICP22 created new session: [6:7587241371061819090:2049]
2025-12-24T02:11:31.915526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371061819090:2049] [node 2] ICS09 handshake done sender: [6:7587241371061819075:2117] self: [6:7587241371061819078:0] peer: [2:7587241369972354617:0] socket: 264 qp: -1
2025-12-24T02:11:31.915532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371061819090:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.915539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:31.916369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819087:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.922853Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819089:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.923106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371061819076:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:31.924446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371061819088:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43502
2025-12-24T02:11:31.924582Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371061819073:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:31.926927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371061819063:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.926950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:31.926959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371061819063:2111] [node 4] ICP22 created new session: [6:7587241371061819092:2111]
2025-12-24T02:11:31.926966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371061819092:2111] [node 4] ICS09 handshake done sender: [6:7587241371061819076:2118] self: [6:7587241371061819079:0] peer: [4:7587241368234805828:0] socket: 273 qp: -1
2025-12-24T02:11:31.926975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371061819092:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:31.926985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:31.927151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371061819056:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.927161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:31.927168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371061819056:2108] [node 5] ICP22 created new session: [6:7587241371061819094:2108]
2025-12-24T02:11:31.927174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371061819094:2108] [node 5] ICS09 handshake done sender: [6:7587241371061819073:2116] self: [6:7587241371061819080:0] peer: [5:7587241367845346726:0] socket: 248 qp: -1
2025-12-24T02:11:31.927181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371061819094:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:31.927189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:31.927499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371061819077:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:31.927676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371061818996:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:31.927692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:31.927699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371061818996:2050] [node 3] ICP22 created new session: [6:7587241371061819096:2050]
2025-12-24T02:11:31.927704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371061819096:2050] [node 3] ICS09 handshake done sender: [6:7587241371061819077:2119] self: [6:7587241371061819081:0] peer: [3:7587241370705248796:0] socket: 275 qp: -1
2025-12-24T02:11:31.927710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371061819096:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:31.927720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:31.937438Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:31.937549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241369972354629:2048] [node 1] ICS01 socket: 215 reason# EndOfStream
2025-12-24T02:11:31.937565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241369972354541:2048] [node 1] ICP30 unregister session Session# [2:7587241369972354629:2048] VirtualId# [2:7587241369972354623:0]
2025-12-24T02:11:31.937573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241369972354629:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.937613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.937649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.938165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:31.938276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241370705248815:2048] [node 1] ICS01 socket: 217 reason# EndOfStream
2025-12-24T02:11:31.938290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241370705248717:2048] [node 1] ICP30 unregister session Session# [3:7587241370705248815:2048] VirtualId# [3:7587241370705248805:0]
2025-12-24T02:11:31.938296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241370705248815:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.938354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.938387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.938447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:31.938539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241368234805848:2048] [node 1] ICS01 socket: 263 reason# EndOfStream
2025-12-24T02:11:31.938554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241368234805753:2048] [node 1] ICP30 unregister session Session# [4:7587241368234805848:2048] VirtualId# [4:7587241368234805846:0]
2025-12-24T02:11:31.938558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241368234805848:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.938598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.938631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.938827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:31.938907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241371061819083:2048] [node 1] ICS07 socket disconnect 216 reason# EndOfStream
2025-12-24T02:11:31.938912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371061819083:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.938959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.938983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241371061819083:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:31.939376Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371061818994:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16838) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.939386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371061819083:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.939391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371061818994:2048] [node 1] ICP30 unregister session Session# [6:7587241371061819083:2048] VirtualId# [6:7587241371061819074:0]
2025-12-24T02:11:31.939445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.939452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241371061818994:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.939455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16838) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.943054Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:31.943158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241367845346744:2048] [node 1] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:11:31.943169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241367845346651:2048] [node 1] ICP30 unregister session Session# [5:7587241367845346744:2048] VirtualId# [5:7587241367845346740:0]
2025-12-24T02:11:31.943176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241367845346744:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.943226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:31.943262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:31.945357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:31.945433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241370705248807:2049] [node 2] ICS01 socket: 226 reason# EndOfStream
2025-12-24T02:11:31.945440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241370705248718:2049] [node 2] ICP30 unregister session Session# [3:7587241370705248807:2049] VirtualId# [3:7587241370705248803:0]
2025-12-24T02:11:31.945446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241370705248807:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.945634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.945666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.946304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:31.946393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241368234805855:2049] [node 2] ICS01 socket: 223 reason# EndOfStream
2025-12-24T02:11:31.946406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241368234805754:2049] [node 2] ICP30 unregister session Session# [4:7587241368234805855:2049] VirtualId# [4:7587241368234805850:0]
2025-12-24T02:11:31.946412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241368234805855:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.946460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.946492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.946820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:31.946912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241367845346753:2049] [node 2] ICS07 socket disconnect 224 reason# EndOfStream
2025-12-24T02:11:31.946922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241367845346753:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.946978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.947008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241367845346753:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.947360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241367845346652:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.947373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241367845346753:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.947378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241367845346652:2049] [node 2] ICP30 unregister session Session# [5:7587241367845346753:2049] VirtualId# [5:7587241367845346741:0]
2025-12-24T02:11:31.947419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.947425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241367845346652:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.947427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.949212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:31.949317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241371061819090:2049] [node 2] ICS07 socket disconnect 264 reason# EndOfStream
2025-12-24T02:11:31.949322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371061819090:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.949379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:31.949409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241371061819090:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:31.949481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241371061818994:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:31.949855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371061818995:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.949866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371061819090:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.949872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371061818995:2049] [node 2] ICP30 unregister session Session# [6:7587241371061819090:2049] VirtualId# [6:7587241371061819078:0]
2025-12-24T02:11:31.949918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:31.949924Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241371061818995:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.949926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1951) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.958751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241367845346652:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.959433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241371061818995:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:31.959892Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:31.959975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371061819096:2050] [node 3] ICS01 socket: 275 reason# EndOfStream
2025-12-24T02:11:31.959983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371061818996:2050] [node 3] ICP30 unregister session Session# [6:7587241371061819096:2050] VirtualId# [6:7587241371061819081:0]
2025-12-24T02:11:31.959989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371061819096:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.960044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.960073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.962638Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:31.962759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241367845346747:2050] [node 3] ICS07 socket disconnect 256 reason# EndOfStream
2025-12-24T02:11:31.962767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241367845346747:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.962835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:31.962869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241367845346747:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:31.963274Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241367845346653:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:31.963292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241367845346747:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:31.963297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241367845346653:2050] [node 3] ICP30 unregister session Session# [5:7587241367845346747:2050] VirtualId# [5:7587241367845346734:0]
2025-12-24T02:11:31.963351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:31.963359Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241367845346653:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.963364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:31.972117Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:31.972218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371061819092:2111] [node 4] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:31.972234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371061819063:2111] [node 4] ICP30 unregister session Session# [6:7587241371061819092:2111] VirtualId# [6:7587241371061819079:0]
2025-12-24T02:11:31.972242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371061819092:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.972302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:31.972338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:31.989000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:31.989092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371061819094:2108] [node 5] ICS01 socket: 248 reason# EndOfStream
2025-12-24T02:11:31.989100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371061819056:2108] [node 5] ICP30 unregister session Session# [6:7587241371061819094:2108] VirtualId# [6:7587241371061819080:0]
2025-12-24T02:11:31.989108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371061819094:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:31.989168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:31.989219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:32.013695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241371431763701:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:32.013711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241371431763702:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.015318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371963476466:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.015337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371963476467:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.018344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241371630327872:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.018357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241371630327873:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.018361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241371630327874:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.016824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374453240967:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.016887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374453240968:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.021472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241374633381099:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.021488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241374633381100:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.021492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241374633381101:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.024510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371253447177:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.024521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371253447178:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.024525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371253447179:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.049399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371253447238:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:32.049987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241371431763765:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:32.050134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241371431763767:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.051684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241371431763773:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.055443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327933:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.054952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241033:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.059019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381160:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.071179Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241371963476466:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20860525ns) {Configure:104(20859240ns) {ProcessConfigured:135(20509090ns) {StartInitialHandshake:157(20508588ns) {DropHandshakes:470(13789ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(13357ns)} GenerateSessionVirtualId:988(178ns) PrepareNewSessionHandshake:477(1583ns) {SwitchToState:179(915ns)}}}}}
2025-12-24T02:11:32.071474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371963476531:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:32.071617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371963476532:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:32.071745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371963476533:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.072964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476534:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.072994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476535:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.083039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241371630327872:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20117906ns) {Configure:104(20116976ns) {ProcessConfigured:135(19704249ns) {StartInitialHandshake:157(19703603ns) {DropHandshakes:470(870ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(133ns)} GenerateSessionVirtualId:988(266ns) PrepareNewSessionHandshake:477(1478ns) {SwitchToState:179(663ns)}}}}}
2025-12-24T02:11:32.087001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241374633381099:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21881598ns) {Configure:104(21880803ns) {ProcessConfigured:135(21488884ns) {StartInitialHandshake:157(21488283ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1803ns) {SwitchToState:179(793ns)}}}}}
2025-12-24T02:11:32.087348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241374633381165:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:32.087480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241374633381166:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:32.092903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241371630327940:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:32.093068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241371630327941:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.115160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241371630327874:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19927096ns) {Configure:104(19923287ns) {ProcessConfigured:135(19584024ns) {StartInitialHandshake:157(19582899ns) {DropHandshakes:470(1047ns) {DropIncomingHandshake:442(160ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1605ns) {SwitchToState:179(585ns)}}}}}
2025-12-24T02:11:32.116045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327942:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.122681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374453241037:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:32.123975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241371253447244:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:32.122892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374453241038:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:32.123011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374453241039:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.151713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241040:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.151735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241041:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.151745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241042:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.173648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381160:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.187331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241041:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.219867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476547:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.219932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.220154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241040:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.220195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241374453240968:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.222975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476547:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.223004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476535:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.223045Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476537:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.223053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371963476537:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.223988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371963476537:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.224209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371963476467:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.224244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.224251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371963476467:2049] [node 3] ICP22 created new session: [2:7587241371963476552:2049]
2025-12-24T02:11:32.224261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371963476552:2049] [node 3] ICS09 handshake done sender: [2:7587241371963476537:2113] self: [2:7587241371963476536:0] peer: [3:7587241374453241051:0] socket: 231 qp: -1
2025-12-24T02:11:32.224268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371963476552:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.224280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.226470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381179:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.223900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241052:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.224273Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374453241040:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.224366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241374453240968:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.224375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.224381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374453240968:2049] [node 2] ICP22 created new session: [3:7587241374453241053:2049]
2025-12-24T02:11:32.224386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374453241053:2049] [node 2] ICS09 handshake done sender: [3:7587241374453241040:2113] self: [3:7587241374453241051:0] peer: [2:7587241371963476536:0] socket: 246 qp: -1
2025-12-24T02:11:32.224391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374453241053:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.224396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.226599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.226664Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.230924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327952:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.230982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327953:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.231034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241371630327874:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.231106Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476548:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.226799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241042:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.231424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381180:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.234983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327942:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.235026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241371630327873:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.238905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241371431763780:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21260
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.238970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241371431763781:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21260
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.239018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241371431763782:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21260
2025-12-24T02:11:32.239080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241371431763783:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21260
2025-12-24T02:11:32.239243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241371431763784:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21260
2025-12-24T02:11:32.243205Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763784:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.243235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763782:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.243301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763783:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.243519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763771:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.243525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241371431763771:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.244232Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241371431763771:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.244431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241371431763765:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.244449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.244457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241371431763765:2108] [node 4] ICP22 created new session: [1:7587241371431763789:2108]
2025-12-24T02:11:32.244464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241371431763789:2108] [node 4] ICS09 handshake done sender: [1:7587241371431763771:2112] self: [1:7587241371431763770:0] peer: [4:7587241371630327957:0] socket: 221 qp: -1
2025-12-24T02:11:32.244471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241371431763789:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:32.244479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.243362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327933:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.243400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241371630327872:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.244345Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327958:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.244750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241371630327933:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.244813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241371630327872:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.244824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.244829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241371630327872:2048] [node 1] ICP22 created new session: [4:7587241371630327959:2048]
2025-12-24T02:11:32.244834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241371630327959:2048] [node 1] ICS09 handshake done sender: [4:7587241371630327933:2108] self: [4:7587241371630327957:0] peer: [1:7587241371431763770:0] socket: 226 qp: -1
2025-12-24T02:11:32.244841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241371630327959:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.244847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.248129Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381181:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.248168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381182:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.262872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381182:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.262929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241374633381165:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.264145Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381184:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.264438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241374633381182:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.265450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447255:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.263134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327946:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.263149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241371630327946:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.264234Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241371630327946:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.264559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241374633381165:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.264576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.264582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241374633381165:2109] [node 4] ICP22 created new session: [5:7587241374633381185:2109]
2025-12-24T02:11:32.264588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241374633381185:2109] [node 4] ICS09 handshake done sender: [5:7587241374633381182:2119] self: [5:7587241374633381183:0] peer: [4:7587241371630327945:0] socket: 265 qp: -1
2025-12-24T02:11:32.264595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241374633381185:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:32.264604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.271428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241371630327940:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.271448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.271456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241371630327940:2109] [node 5] ICP22 created new session: [4:7587241371630327963:2109]
2025-12-24T02:11:32.271462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241371630327963:2109] [node 5] ICS09 handshake done sender: [4:7587241371630327946:2115] self: [4:7587241371630327945:0] peer: [5:7587241374633381183:0] socket: 240 qp: -1
2025-12-24T02:11:32.271470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241371630327963:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:32.271479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.272349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447256:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.279072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241044:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.279087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241374453241044:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.279693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241033:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.279830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241374453240967:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.280020Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374453241044:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.280124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241374453241037:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.280139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.280144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374453241037:2110] [node 4] ICP22 created new session: [3:7587241374453241059:2110]
2025-12-24T02:11:32.280150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374453241059:2110] [node 4] ICS09 handshake done sender: [3:7587241374453241044:2116] self: [3:7587241374453241043:0] peer: [4:7587241371630327955:0] socket: 254 qp: -1
2025-12-24T02:11:32.280156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374453241059:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:32.280165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.281095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447257:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.282228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371253447256:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.282256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371253447244:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.281934Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241371630327874:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241371630327953:2118] held: no
2025-12-24T02:11:32.282152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327966:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.282552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241371630327948:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.282557Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241371630327948:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.283152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241371630327953:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.283346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241371630327874:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.283361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.283367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241371630327874:2050] [node 3] ICP22 created new session: [4:7587241371630327968:2050]
2025-12-24T02:11:32.283373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241371630327968:2050] [node 3] ICS09 handshake done sender: [4:7587241371630327953:2118] self: [4:7587241371630327955:0] peer: [3:7587241374453241043:0] socket: 260 qp: -1
2025-12-24T02:11:32.283379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241371630327968:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.283387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.283862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447259:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.283910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241371630327948:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.284132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371253447255:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.284144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371253447256:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.284118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241371630327941:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.284133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.284138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241371630327941:2110] [node 6] ICP22 created new session: [4:7587241371630327970:2110]
2025-12-24T02:11:32.284142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241371630327970:2110] [node 6] ICS09 handshake done sender: [4:7587241371630327948:2116] self: [4:7587241371630327947:0] peer: [6:7587241371253447258:0] socket: 270 qp: -1
2025-12-24T02:11:32.284147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241371630327970:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:32.284153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.284295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371253447244:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.284308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.284313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371253447244:2110] [node 4] ICP22 created new session: [6:7587241371253447260:2110]
2025-12-24T02:11:32.284317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371253447260:2110] [node 4] ICS09 handshake done sender: [6:7587241371253447256:2116] self: [6:7587241371253447258:0] peer: [4:7587241371630327947:0] socket: 271 qp: -1
2025-12-24T02:11:32.284322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371253447260:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:32.284329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.285493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371253447238:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.285754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381174:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.285764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241374633381174:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.286593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241374633381174:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.286652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763780:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.286689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763769:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.286705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241371431763769:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.287804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447264:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.288007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241371431763769:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.288263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241374633381166:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.288284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.288289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241374633381166:2110] [node 6] ICP22 created new session: [5:7587241374633381189:2110]
2025-12-24T02:11:32.288294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241374633381189:2110] [node 6] ICS09 handshake done sender: [5:7587241374633381174:2115] self: [5:7587241374633381173:0] peer: [6:7587241371253447262:0] socket: 268 qp: -1
2025-12-24T02:11:32.288298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241374633381189:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:32.288304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.288259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476539:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.288271Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371963476539:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.288719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476534:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.288832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241371963476466:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.288892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.289175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241371431763702:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.289191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.289196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241371431763702:2049] [node 3] ICP22 created new session: [1:7587241371431763794:2049]
2025-12-24T02:11:32.289200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241371431763794:2049] [node 3] ICS09 handshake done sender: [1:7587241371431763769:2111] self: [1:7587241371431763768:0] peer: [3:7587241374453241058:0] socket: 220 qp: -1
2025-12-24T02:11:32.289205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241371431763794:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.289211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.289326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381181:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.289346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241374633381099:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.289715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371963476539:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.289871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476556:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.289881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371963476531:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.289893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.289897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371963476531:2107] [node 4] ICP22 created new session: [2:7587241371963476559:2107]
2025-12-24T02:11:32.289902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371963476559:2107] [node 4] ICS09 handshake done sender: [2:7587241371963476539:2114] self: [2:7587241371963476538:0] peer: [4:7587241371630327956:0] socket: 232 qp: -1
2025-12-24T02:11:32.289906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371963476559:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:32.289912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.289824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763781:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.289857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763762:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.289859Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241371431763762:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.290015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381180:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.290037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241374633381100:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.290873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476541:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.290884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371963476541:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.291646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381193:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.291749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371963476541:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.291931Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241374633381180:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.291917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371963476532:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.291938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.291945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371963476532:2108] [node 5] ICP22 created new session: [2:7587241371963476564:2108]
2025-12-24T02:11:32.291951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371963476564:2108] [node 5] ICS09 handshake done sender: [2:7587241371963476541:2115] self: [2:7587241371963476540:0] peer: [5:7587241374633381192:0] socket: 229 qp: -1
2025-12-24T02:11:32.291956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371963476564:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:32.291965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.292973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371253447238:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18510) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241371253447255:2115] held: no
2025-12-24T02:11:32.293114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371253447255:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.293484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371253447238:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.293497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.293502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371253447238:2107] [node 5] ICP22 created new session: [6:7587241371253447266:2107]
2025-12-24T02:11:32.293507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371253447266:2107] [node 5] ICS09 handshake done sender: [6:7587241371253447255:2115] self: [6:7587241371253447262:0] peer: [5:7587241374633381173:0] socket: 269 qp: -1
2025-12-24T02:11:32.293512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371253447266:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:32.293518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.293837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447265:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.292018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241374633381100:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.292030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.292035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241374633381100:2049] [node 2] ICP22 created new session: [5:7587241374633381194:2049]
2025-12-24T02:11:32.292039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241374633381194:2049] [node 2] ICS09 handshake done sender: [5:7587241374633381180:2117] self: [5:7587241374633381192:0] peer: [2:7587241371963476540:0] socket: 261 qp: -1
2025-12-24T02:11:32.292046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241374633381194:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.292052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.294944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241374633381179:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.294976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241374633381101:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.295665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381197:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.294415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763775:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.294426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241371431763775:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.297100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241371431763762:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.297120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447269:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.297179Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374453240967:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241374453241033:2109] held: no
2025-12-24T02:11:32.297259Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241371431763775:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.297502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241374633381181:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.297567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371253447265:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.297595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371253447178:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.297944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241374633381099:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.297958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.297971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241374633381099:2048] [node 1] ICP22 created new session: [5:7587241374633381198:2048]
2025-12-24T02:11:32.297976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241374633381198:2048] [node 1] ICS09 handshake done sender: [5:7587241374633381181:2118] self: [5:7587241374633381191:0] peer: [1:7587241371431763774:0] socket: 264 qp: -1
2025-12-24T02:11:32.297981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241374633381198:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.297988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.298094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371963476543:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.298104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371963476543:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.298429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241371431763701:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.298447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.298451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241371431763701:2048] [node 2] ICP22 created new session: [1:7587241371431763800:2048]
2025-12-24T02:11:32.298457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241371431763800:2048] [node 2] ICS09 handshake done sender: [1:7587241371431763762:2107] self: [1:7587241371431763761:0] peer: [2:7587241371963476558:0] socket: 219 qp: -1
2025-12-24T02:11:32.298461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241371431763800:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:32.298468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.298567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371963476543:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.298721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371963476533:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.298742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.298749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371963476533:2109] [node 6] ICP22 created new session: [2:7587241371963476568:2109]
2025-12-24T02:11:32.298754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371963476568:2109] [node 6] ICS09 handshake done sender: [2:7587241371963476543:2116] self: [2:7587241371963476542:0] peer: [6:7587241371253447270:0] socket: 245 qp: -1
2025-12-24T02:11:32.298760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371963476568:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:32.298766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.299325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241371431763767:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.299351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.299359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241371431763767:2109] [node 5] ICP22 created new session: [1:7587241371431763802:2109]
2025-12-24T02:11:32.299364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241371431763802:2109] [node 5] ICS09 handshake done sender: [1:7587241371431763775:2113] self: [1:7587241371431763774:0] peer: [5:7587241374633381191:0] socket: 251 qp: -1
2025-12-24T02:11:32.299369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241371431763802:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:32.299378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.300335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241374633381200:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20040
2025-12-24T02:11:32.300986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241371963476466:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241371963476534:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.301157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371963476566:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15415
2025-12-24T02:11:32.301469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371963476534:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.301560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241371963476466:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.301574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.301580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371963476466:2048] [node 1] ICP22 created new session: [2:7587241371963476570:2048]
2025-12-24T02:11:32.301584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371963476570:2048] [node 1] ICS09 handshake done sender: [2:7587241371963476534:2111] self: [2:7587241371963476558:0] peer: [1:7587241371431763761:0] socket: 228 qp: -1
2025-12-24T02:11:32.301591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371963476570:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.301598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.302744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447271:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.302831Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241371630327873:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241371630327942:2113] held: no
2025-12-24T02:11:32.297523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241062:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.299281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374453241063:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37944
2025-12-24T02:11:32.299395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241046:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.299399Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241374453241046:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.299865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374453241033:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.300048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241374453240967:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.300057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.300062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374453240967:2048] [node 1] ICP22 created new session: [3:7587241374453241065:2048]
2025-12-24T02:11:32.300068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374453241065:2048] [node 1] ICS09 handshake done sender: [3:7587241374453241033:2109] self: [3:7587241374453241058:0] peer: [1:7587241371431763768:0] socket: 225 qp: -1
2025-12-24T02:11:32.300073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374453241065:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.300080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.300443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374453241046:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.303072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241371630327972:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33144
2025-12-24T02:11:32.303136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371253447265:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.303251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371253447178:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.303269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.303276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371253447178:2049] [node 2] ICP22 created new session: [6:7587241371253447272:2049]
2025-12-24T02:11:32.303282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371253447272:2049] [node 2] ICS09 handshake done sender: [6:7587241371253447265:2120] self: [6:7587241371253447270:0] peer: [2:7587241371963476542:0] socket: 244 qp: -1
2025-12-24T02:11:32.303289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371253447272:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.303296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.303907Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371253447257:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.304662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241374633381179:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.304781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241371253447178:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:32.304811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371253447179:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.304795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241374633381101:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241374633381179:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.305680Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447275:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.305873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371253447257:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.304908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241374633381101:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.304922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.304926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241374633381101:2050] [node 3] ICP22 created new session: [5:7587241374633381201:2050]
2025-12-24T02:11:32.304933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241374633381201:2050] [node 3] ICS09 handshake done sender: [5:7587241374633381179:2116] self: [5:7587241374633381196:0] peer: [3:7587241374453241045:0] socket: 259 qp: -1
2025-12-24T02:11:32.304938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241374633381201:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.304945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.306152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371253447179:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.306167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.306172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371253447179:2050] [node 3] ICP22 created new session: [6:7587241371253447276:2050]
2025-12-24T02:11:32.306177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371253447276:2050] [node 3] ICS09 handshake done sender: [6:7587241371253447257:2117] self: [6:7587241371253447274:0] peer: [3:7587241374453241047:0] socket: 273 qp: -1
2025-12-24T02:11:32.306184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371253447276:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.306196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.303736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241371630327942:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.303079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241374453241038:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.303100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.303111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374453241038:2111] [node 5] ICP22 created new session: [3:7587241374453241067:2111]
2025-12-24T02:11:32.303118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374453241067:2111] [node 5] ICS09 handshake done sender: [3:7587241374453241046:2117] self: [3:7587241374453241045:0] peer: [5:7587241374633381196:0] socket: 224 qp: -1
2025-12-24T02:11:32.303124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374453241067:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:32.303131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.307298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241371253447269:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.303918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241063:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.304902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374453241048:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.304907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241374453241048:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.305668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374453241048:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.306126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241374453241039:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.306143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.306148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374453241039:2112] [node 6] ICP22 created new session: [3:7587241374453241071:2112]
2025-12-24T02:11:32.306153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374453241071:2112] [node 6] ICS09 handshake done sender: [3:7587241374453241048:2118] self: [3:7587241374453241047:0] peer: [6:7587241371253447274:0] socket: 248 qp: -1
2025-12-24T02:11:32.306158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374453241071:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:32.306165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.307341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241371253447177:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.307423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241371431763777:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.307435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241371431763777:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.307915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241371431763777:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.307972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241371253447279:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14156
2025-12-24T02:11:32.308046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241371431763773:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.308063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.308069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241371431763773:2110] [node 6] ICP22 created new session: [1:7587241371431763805:2110]
2025-12-24T02:11:32.308073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241371431763805:2110] [node 6] ICS09 handshake done sender: [1:7587241371431763777:2114] self: [1:7587241371431763776:0] peer: [6:7587241371253447278:0] socket: 284 qp: -1
2025-12-24T02:11:32.308078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241371431763805:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:32.308085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.308209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241371253447269:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.308282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241371253447177:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.308295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.308301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241371253447177:2048] [node 1] ICP22 created new session: [6:7587241371253447280:2048]
2025-12-24T02:11:32.308307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241371253447280:2048] [node 1] ICS09 handshake done sender: [6:7587241371253447269:2121] self: [6:7587241371253447278:0] peer: [1:7587241371431763776:0] socket: 286 qp: -1
2025-12-24T02:11:32.308316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241371253447280:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.308323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.307065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241371630327873:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.307086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.307093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241371630327873:2049] [node 2] ICP22 created new session: [4:7587241371630327973:2049]
2025-12-24T02:11:32.307099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241371630327973:2049] [node 2] ICS09 handshake done sender: [4:7587241371630327942:2113] self: [4:7587241371630327956:0] peer: [2:7587241371963476538:0] socket: 238 qp: -1
2025-12-24T02:11:32.307105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241371630327973:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.307113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.316579Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:32.316704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241371630327959:2048] [node 1] ICS07 socket disconnect 226 reason# EndOfStream
2025-12-24T02:11:32.316721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241371630327959:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.316794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.316822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241371630327959:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:32.316805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:32.316944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241374633381198:2048] [node 1] ICS01 socket: 264 reason# EndOfStream
2025-12-24T02:11:32.316958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241374633381099:2048] [node 1] ICP30 unregister session Session# [5:7587241374633381198:2048] VirtualId# [5:7587241374633381191:0]
2025-12-24T02:11:32.316965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241374633381198:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.317006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.317031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.316889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:32.316965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241371253447280:2048] [node 1] ICS07 socket disconnect 286 reason# EndOfStream
2025-12-24T02:11:32.316971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371253447280:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.317013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.317035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241371253447280:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:32.317436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241371630327872:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.317450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241371630327959:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.317455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241371630327872:2048] [node 1] ICP30 unregister session Session# [4:7587241371630327959:2048] VirtualId# [4:7587241371630327957:0]
2025-12-24T02:11:32.317504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.317515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241371630327872:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.317518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.317512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371253447177:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.317526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371253447280:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.317531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371253447177:2048] [node 1] ICP30 unregister session Session# [6:7587241371253447280:2048] VirtualId# [6:7587241371253447278:0]
2025-12-24T02:11:32.317574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.317584Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241371253447177:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.317586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3155) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.318496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:32.318576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241374453241065:2048] [node 1] ICS01 socket: 225 reason# EndOfStream
2025-12-24T02:11:32.318588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241374453240967:2048] [node 1] ICP30 unregister session Session# [3:7587241374453241065:2048] VirtualId# [3:7587241374453241058:0]
2025-12-24T02:11:32.318593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241374453241065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.318655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.318687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.323687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:32.323825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241374453241053:2049] [node 2] ICS07 socket disconnect 246 reason# EndOfStream
2025-12-24T02:11:32.323831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241374453241053:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.323887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.323908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241374453241053:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:32.327036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241371253447177:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:32.327054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241371630327872:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:32.334826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374453240968:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:32.334849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241374453241053:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:32.335186Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374453240968:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.335205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241374453241053:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.335211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241374453240968:2049] [node 2] ICP30 unregister session Session# [3:7587241374453241053:2049] VirtualId# [3:7587241374453241051:0]
2025-12-24T02:11:32.335235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.335249Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241374453240968:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.335253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.336300Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:32.336397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:32.336456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241371630327973:2049] [node 2] ICS01 socket: 238 reason# EndOfStream
2025-12-24T02:11:32.336471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241371630327873:2049] [node 2] ICP30 unregister session Session# [4:7587241371630327973:2049] VirtualId# [4:7587241371630327956:0]
2025-12-24T02:11:32.336478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241371630327973:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.336492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241374633381194:2049] [node 2] ICS07 socket disconnect 261 reason# EndOfStream
2025-12-24T02:11:32.336503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241374633381194:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.336542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.336567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.336604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241374633381194:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:32.336590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.336571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:32.336661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241371253447272:2049] [node 2] ICS07 socket disconnect 244 reason# EndOfStream
2025-12-24T02:11:32.336667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371253447272:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.336712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.336749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241371253447272:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:32.336911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241374633381100:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.336926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241374633381194:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.336931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241374633381100:2049] [node 2] ICP30 unregister session Session# [5:7587241374633381194:2049] VirtualId# [5:7587241374633381192:0]
2025-12-24T02:11:32.336986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.337000Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241374633381100:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.337002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.337012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371253447178:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.337025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371253447272:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.337029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371253447178:2049] [node 2] ICP30 unregister session Session# [6:7587241371253447272:2049] VirtualId# [6:7587241371253447270:0]
2025-12-24T02:11:32.337071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.337080Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241371253447178:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.337083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14140) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.346506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241374633381100:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:32.346500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241371253447178:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:32.351759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:32.351864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241374633381201:2050] [node 3] ICS01 socket: 259 reason# EndOfStream
2025-12-24T02:11:32.351878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241374633381101:2050] [node 3] ICP30 unregister session Session# [5:7587241374633381201:2050] VirtualId# [5:7587241374633381196:0]
2025-12-24T02:11:32.351891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241374633381201:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.351946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:32.351984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:32.353533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:32.353660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241371253447276:2050] [node 3] ICS07 socket disconnect 273 reason# EndOfStream
2025-12-24T02:11:32.353670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371253447276:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.353717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:32.353786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241371253447276:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:32.354099Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241371253447179:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.354115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371253447276:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.354121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371253447179:2050] [node 3] ICP30 unregister session Session# [6:7587241371253447276:2050] VirtualId# [6:7587241371253447274:0]
2025-12-24T02:11:32.354164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:32.354169Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241371253447179:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.354170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14484) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.360622Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:32.360722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371253447260:2110] [node 4] ICS01 socket: 271 reason# EndOfStream
2025-12-24T02:11:32.360733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371253447244:2110] [node 4] ICP30 unregister session Session# [6:7587241371253447260:2110] VirtualId# [6:7587241371253447258:0]
2025-12-24T02:11:32.360743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371253447260:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.360794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:32.360906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:32.363909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241371253447179:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:32.372587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:32.372714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241371253447266:2107] [node 5] ICS01 socket: 269 reason# EndOfStream
2025-12-24T02:11:32.372730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241371253447238:2107] [node 5] ICP30 unregister session Session# [6:7587241371253447266:2107] VirtualId# [6:7587241371253447262:0]
2025-12-24T02:11:32.372738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241371253447266:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.372790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:32.372824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:32.397724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371752027783:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.397751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371752027784:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.399636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372995714101:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:32.399646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372995714102:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.410918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241372347418097:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.410931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241372347418098:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.422780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241372721555687:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.422801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241372721555688:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.422812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241372721555689:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.430775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241373792345074:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.430794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241373792345075:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.430800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241373792345076:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.438832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241373929768613:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.438845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241373929768614:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.438850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241373929768615:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.452980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418121:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.455075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241372347418130:2076] [node 6] ICP01 ready to work
2025-12-24T02:11:32.455107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027838:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
2025-12-24T02:11:32.455124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371752027839:2101] [node 4] ICP01 ready to work
2025-12-24T02:11:32.456396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241372347418138:2082] [node 4] ICP01 ready to work
2025-12-24T02:11:32.456479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241372347418139:2084] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.459043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768635:2069] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.460961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372995714168:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:32.461498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372995714172:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.462439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418156:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
2025-12-24T02:11:32.462476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241372721555707:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.462594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241372721555708:2062] [node 6] ICP01 ready to work
2025-12-24T02:11:32.458983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241373792345092:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:32.459086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241373792345093:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:32.463426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555704:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.469727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372995714177:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:32.470599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418121:2071] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.470655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241372347418097:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.471036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555705:2065] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.479300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372995714180:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46861
2025-12-24T02:11:32.479335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372995714181:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46861
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.479362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241373929768654:2073] [node 5] ICP01 ready to work
2025-12-24T02:11:32.479534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241373929768655:2080] [node 4] ICP01 ready to work
2025-12-24T02:11:32.479595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768653:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.481978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372995714182:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46861
2025-12-24T02:11:32.486719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714163:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.486750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372995714163:2107] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.491034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027838:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.491220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371752027856:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:32.492882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241372721555708:2062] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21737011ns) {Configure:104(21736415ns) {ProcessConfigured:135(21369348ns) {StartInitialHandshake:157(21369026ns) {DropHandshakes:470(777ns) {DropIncomingHandshake:442(115ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(260ns) PrepareNewSessionHandshake:477(1462ns) {SwitchToState:179(1062ns)}}}}}
2025-12-24T02:11:32.491491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241371752027857:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:32.492528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241371752027783:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.493980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555706:2066] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.492567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027854:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
2025-12-24T02:11:32.492581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027855:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
2025-12-24T02:11:32.494489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714161:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.494501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372995714161:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.495029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555706:2066] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.506408Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241373929768614:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23272403ns) {Configure:104(23271389ns) {ProcessConfigured:135(22003496ns) {StartInitialHandshake:157(22002756ns) {DropHandshakes:470(666ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(1538ns) {SwitchToState:179(794ns)}}}}}
2025-12-24T02:11:32.506816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768690:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.506829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768691:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.510060Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241372721555707:2059] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47386205ns) {Configure:104(47385481ns) {ProcessConfigured:135(46935166ns) {StartInitialHandshake:157(46934237ns) {DropHandshakes:470(661ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:477(1640ns) {SwitchToState:179(1002ns)}}}}}
2025-12-24T02:11:32.510117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241372721555687:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.511010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372995714161:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.510878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418179:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
2025-12-24T02:11:32.510912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418180:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
2025-12-24T02:11:32.511113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372995714163:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.510905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027866:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.518299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345151:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.518338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345152:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.514712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714171:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.514725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372995714171:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.522818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345151:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.522834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241373929768653:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.522886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241373929768654:2073] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.522807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372995714102:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.522830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.522845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372995714102:2049] [node 3] ICP22 created new session: [1:7587241372995714187:2049]
2025-12-24T02:11:32.522854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372995714187:2049] [node 3] ICS09 handshake done sender: [1:7587241372995714163:2107] self: [1:7587241372995714162:0] peer: [3:7587241372347418171:0] socket: 223 qp: -1
2025-12-24T02:11:32.522863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372995714187:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.522873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.523164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372995714185:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46861
2025-12-24T02:11:32.524274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372995714101:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.524285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.524292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372995714101:2048] [node 2] ICP22 created new session: [1:7587241372995714189:2048]
2025-12-24T02:11:32.524299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372995714189:2048] [node 2] ICS09 handshake done sender: [1:7587241372995714161:2106] self: [1:7587241372995714160:0] peer: [2:7587241371752027864:0] socket: 222 qp: -1
2025-12-24T02:11:32.524306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372995714189:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:32.526930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768696:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.524315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.526890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371752027838:2102] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.530787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241373792345076:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55935131ns) {Configure:104(55933928ns) {ProcessConfigured:135(55561207ns) {StartInitialHandshake:157(55560488ns) {DropHandshakes:470(325ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(1726ns) {SwitchToState:179(671ns)}}}}}
2025-12-24T02:11:32.531148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714179:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.531159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372995714179:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.531490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345152:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.531521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241373792345074:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.530972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241373929768696:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.531012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241373929768613:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.534876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241371752027783:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.534918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.534925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371752027783:2048] [node 1] ICP22 created new session: [2:7587241371752027867:2048]
2025-12-24T02:11:32.534931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371752027867:2048] [node 1] ICS09 handshake done sender: [2:7587241371752027838:2102] self: [2:7587241371752027864:0] peer: [1:7587241372995714160:0] socket: 226 qp: -1
2025-12-24T02:11:32.534937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371752027867:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.534947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.535773Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372995714171:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.538776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241373929768690:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.538819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241373929768614:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.542917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345099:2067] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.542936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241373792345099:2067] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.546909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768699:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.550771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241373792345099:2067] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.551312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241373929768653:2085] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.551454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241373929768654:2073] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.551479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.551487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241373929768654:2073] [node 5] ICP22 created new session: [6:7587241373929768700:2073]
2025-12-24T02:11:32.551496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241373929768700:2073] [node 5] ICS09 handshake done sender: [6:7587241373929768653:2085] self: [6:7587241373929768695:0] peer: [5:7587241373792345098:0] socket: 247 qp: -1
2025-12-24T02:11:32.551505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241373929768700:2073] [node 5] ICS10 traffic start
2025-12-24T02:11:32.551519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.552232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555769:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.555449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418156:2095] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.555475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241372347418121:2071] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.555699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241372347418097:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.555718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.555726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241372347418097:2048] [node 1] ICP22 created new session: [3:7587241372347418181:2048]
2025-12-24T02:11:32.555731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241372347418181:2048] [node 1] ICS09 handshake done sender: [3:7587241372347418121:2071] self: [3:7587241372347418171:0] peer: [1:7587241372995714162:0] socket: 224 qp: -1
2025-12-24T02:11:32.555739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241372347418181:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.555757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.555579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027855:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.558963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714174:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.558975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372995714174:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.559355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714181:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.559599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372995714168:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.559622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.559630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372995714168:2109] [node 4] ICP22 created new session: [1:7587241372995714193:2109]
2025-12-24T02:11:32.559635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372995714193:2109] [node 4] ICS09 handshake done sender: [1:7587241372995714171:2111] self: [1:7587241372995714170:0] peer: [4:7587241372721555767:0] socket: 243 qp: -1
2025-12-24T02:11:32.559642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372995714193:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:32.559650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.560066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241373792345093:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.560085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.560090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241373792345093:2064] [node 6] ICP22 created new session: [5:7587241373792345157:2064]
2025-12-24T02:11:32.560096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241373792345157:2064] [node 6] ICS09 handshake done sender: [5:7587241373792345099:2067] self: [5:7587241373792345098:0] peer: [6:7587241373929768695:0] socket: 242 qp: -1
2025-12-24T02:11:32.560100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241373792345157:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:32.560106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.563124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241373929768691:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.566678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555705:2065] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.566738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555704:2064] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.568398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241373929768655:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.568516Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.569372Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241373929768635:2069] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.569401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768705:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.569422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241373929768615:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.566870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027859:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.566839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241372721555706:2066] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.566888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371752027859:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.569230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418183:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
2025-12-24T02:11:32.567640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714185:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.567664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241372995714101:2048] [node 2] ICP18 handshake (actor [1:7587241372995714181:2116]) is too old
2025-12-24T02:11:32.569107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372995714182:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.569122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241372995714168:2109] [node 4] ICP18 handshake (actor [1:7587241372995714185:2118]) is too old
2025-12-24T02:11:32.569633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372995714197:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46861
2025-12-24T02:11:32.569738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372995714179:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.569842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372995714174:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.569903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372995714177:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.569917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.569922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372995714177:2113] [node 6] ICP22 created new session: [1:7587241372995714199:2113]
2025-12-24T02:11:32.569927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372995714199:2113] [node 6] ICS09 handshake done sender: [1:7587241372995714179:2114] self: [1:7587241372995714178:0] peer: [6:7587241373929768697:0] socket: 270 qp: -1
2025-12-24T02:11:32.569933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372995714199:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:32.569940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.570171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372995714172:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.570177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.570181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372995714172:2110] [node 5] ICP22 created new session: [1:7587241372995714201:2110]
2025-12-24T02:11:32.570185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372995714201:2110] [node 5] ICS09 handshake done sender: [1:7587241372995714174:2112] self: [1:7587241372995714173:0] peer: [5:7587241373792345155:0] socket: 268 qp: -1
2025-12-24T02:11:32.570190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372995714201:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:32.570194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.571414Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345160:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.573448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241373792345152:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.570916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555753:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.570926Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241372721555753:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.574818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345161:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.574837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241373929768613:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3511) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241373929768696:2119] held: no
2025-12-24T02:11:32.575024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241373792345074:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.575041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.575048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241373792345074:2048] [node 1] ICP22 created new session: [5:7587241373792345162:2048]
2025-12-24T02:11:32.575054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241373792345162:2048] [node 1] ICS09 handshake done sender: [5:7587241373792345152:2115] self: [5:7587241373792345155:0] peer: [1:7587241372995714173:0] socket: 269 qp: -1
2025-12-24T02:11:32.575059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241373792345162:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.575067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.573632Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371752027859:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.573768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027854:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.573819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371752027856:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.573834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.573841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371752027856:2109] [node 6] ICP22 created new session: [2:7587241371752027872:2109]
2025-12-24T02:11:32.573847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371752027872:2109] [node 6] ICS09 handshake done sender: [2:7587241371752027859:2115] self: [2:7587241371752027858:0] peer: [6:7587241373929768698:0] socket: 254 qp: -1
2025-12-24T02:11:32.573854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371752027872:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:32.573862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.574959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241372721555688:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.574987Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241372721555687:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3511) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241372721555706:2066] held: no
2025-12-24T02:11:32.575081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241372721555687:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.575091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.575099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241372721555687:2048] [node 1] ICP22 created new session: [4:7587241372721555774:2048]
2025-12-24T02:11:32.575106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241372721555774:2048] [node 1] ICS09 handshake done sender: [4:7587241372721555706:2066] self: [4:7587241372721555767:0] peer: [1:7587241372995714170:0] socket: 245 qp: -1
2025-12-24T02:11:32.575112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241372721555774:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.575119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.575941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027841:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.575952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371752027841:2103] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.576412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371752027841:2103] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.576471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371752027839:2101] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.576480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.576485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371752027839:2101] [node 4] ICP22 created new session: [2:7587241371752027877:2101]
2025-12-24T02:11:32.576490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371752027877:2101] [node 4] ICS09 handshake done sender: [2:7587241371752027841:2103] self: [2:7587241371752027840:0] peer: [4:7587241372721555772:0] socket: 227 qp: -1
2025-12-24T02:11:32.576495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371752027877:2101] [node 4] ICS10 traffic start
2025-12-24T02:11:32.576503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.579084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418141:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.579100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241372347418141:2086] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.580136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555770:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.580336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555776:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.579580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418179:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.579625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418183:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.582646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768707:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.579750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241372347418098:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.581894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241372347418141:2086] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.582012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241372347418130:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.582029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.582035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241372347418130:2076] [node 6] ICP22 created new session: [3:7587241372347418189:2076]
2025-12-24T02:11:32.582042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241372347418189:2076] [node 6] ICS09 handshake done sender: [3:7587241372347418141:2086] self: [3:7587241372347418140:0] peer: [6:7587241373929768706:0] socket: 229 qp: -1
2025-12-24T02:11:32.582047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241372347418189:2076] [node 6] ICS10 traffic start
2025-12-24T02:11:32.582054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.586552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241373929768696:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.586432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241372721555753:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.583326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241372995714102:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:32.583352Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241372995714172:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:32.587367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345161:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.587400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241373792345075:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.587590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241373929768690:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.587770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241373929768615:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7989) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241373929768635:2069] held: no
2025-12-24T02:11:32.587875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241373929768708:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30054
2025-12-24T02:11:32.587988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241373929768614:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.588002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.588007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241373929768614:2049] [node 2] ICP22 created new session: [6:7587241373929768710:2049]
2025-12-24T02:11:32.588013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241373929768710:2049] [node 2] ICS09 handshake done sender: [6:7587241373929768690:2114] self: [6:7587241373929768698:0] peer: [2:7587241371752027858:0] socket: 261 qp: -1
2025-12-24T02:11:32.588018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241373929768710:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.588024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.588668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241373929768691:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.588712Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241373929768635:2069] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.588776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241373929768655:2080] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.588788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.588792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241373929768655:2080] [node 4] ICP22 created new session: [6:7587241373929768712:2080]
2025-12-24T02:11:32.588797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241373929768712:2080] [node 4] ICS09 handshake done sender: [6:7587241373929768691:2115] self: [6:7587241373929768703:0] peer: [4:7587241372721555720:0] socket: 262 qp: -1
2025-12-24T02:11:32.588801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241373929768712:2080] [node 4] ICS10 traffic start
2025-12-24T02:11:32.588806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.589000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241373929768615:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.589005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.589009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241373929768615:2050] [node 3] ICP22 created new session: [6:7587241373929768714:2050]
2025-12-24T02:11:32.589012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241373929768714:2050] [node 3] ICS09 handshake done sender: [6:7587241373929768635:2069] self: [6:7587241373929768706:0] peer: [3:7587241372347418140:0] socket: 236 qp: -1
2025-12-24T02:11:32.589015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241373929768714:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.589020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.586737Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027861:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.586749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371752027861:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.590751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241373929768613:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.590768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.590776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241373929768613:2048] [node 1] ICP22 created new session: [6:7587241373929768716:2048]
2025-12-24T02:11:32.590782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241373929768716:2048] [node 1] ICS09 handshake done sender: [6:7587241373929768696:2119] self: [6:7587241373929768697:0] peer: [1:7587241372995714178:0] socket: 271 qp: -1
2025-12-24T02:11:32.590787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241373929768716:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.590793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.590769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418188:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.592728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418188:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.590958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555770:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.591917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345166:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.590976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241372721555708:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.590988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.590995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241372721555708:2062] [node 6] ICP22 created new session: [4:7587241372721555778:2062]
2025-12-24T02:11:32.591000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241372721555778:2062] [node 6] ICS09 handshake done sender: [4:7587241372721555753:2109] self: [4:7587241372721555720:0] peer: [6:7587241373929768703:0] socket: 256 qp: -1
2025-12-24T02:11:32.591006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241372721555778:2062] [node 6] ICS10 traffic start
2025-12-24T02:11:32.591013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.591506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241372721555708:2062] [node 6] ICP18 handshake (actor [4:7587241372721555770:2119]) is too old
2025-12-24T02:11:32.592192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241372721555704:2064] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.594824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241371752027863:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.594834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241371752027863:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.595510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027879:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
2025-12-24T02:11:32.595525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241371752027880:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38727
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.598884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241372721555688:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.598908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.598914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241372721555688:2049] [node 2] ICP22 created new session: [4:7587241372721555782:2049]
2025-12-24T02:11:32.598920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241372721555782:2049] [node 2] ICS09 handshake done sender: [4:7587241372721555704:2064] self: [4:7587241372721555772:0] peer: [2:7587241371752027840:0] socket: 232 qp: -1
2025-12-24T02:11:32.598926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241372721555782:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.598933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.599707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345167:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.599279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555781:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.599105Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371752027861:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.599342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371752027784:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.599355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.599367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371752027784:2049] [node 3] ICP22 created new session: [2:7587241371752027882:2049]
2025-12-24T02:11:32.599372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371752027882:2049] [node 3] ICS09 handshake done sender: [2:7587241371752027861:2116] self: [2:7587241371752027860:0] peer: [3:7587241372347418187:0] socket: 231 qp: -1
2025-12-24T02:11:32.599377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371752027882:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.599383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.602955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555781:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.602985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241372721555689:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.603270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241372347418194:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.606802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345166:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.606836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345168:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.606838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241373792345076:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.606830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241371752027863:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.607151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241373792345161:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.610718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418151:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.610733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241372347418151:2090] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.614837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241373792345075:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.614862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.614873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241373792345075:2049] [node 2] ICP22 created new session: [5:7587241373792345170:2049]
2025-12-24T02:11:32.614879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241373792345170:2049] [node 2] ICS09 handshake done sender: [5:7587241373792345161:2118] self: [5:7587241373792345165:0] peer: [2:7587241371752027862:0] socket: 267 qp: -1
2025-12-24T02:11:32.614887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241373792345170:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.614896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.618966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241373792345167:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.619010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241373792345092:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.618779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241372347418098:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18327) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241372347418183:2119] held: no
2025-12-24T02:11:32.619142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241372721555766:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.619153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241372721555766:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.620018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345173:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.620147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241372721555766:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.620317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241373792345167:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.620369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241373792345092:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.620382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.620387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241373792345092:2059] [node 4] ICP22 created new session: [5:7587241373792345174:2059]
2025-12-24T02:11:32.620392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241373792345174:2059] [node 4] ICS09 handshake done sender: [5:7587241373792345167:2120] self: [5:7587241373792345172:0] peer: [4:7587241372721555712:0] socket: 290 qp: -1
2025-12-24T02:11:32.620398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241373792345174:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:32.620405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.621387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241372721555707:2059] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.621404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.621409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241372721555707:2059] [node 5] ICP22 created new session: [4:7587241372721555786:2059]
2025-12-24T02:11:32.621414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241372721555786:2059] [node 5] ICS09 handshake done sender: [4:7587241372721555766:2116] self: [4:7587241372721555712:0] peer: [5:7587241373792345172:0] socket: 238 qp: -1
2025-12-24T02:11:32.621418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241372721555786:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:32.621425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.618978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241371752027857:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.619015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.619023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241371752027857:2112] [node 5] ICP22 created new session: [2:7587241371752027885:2112]
2025-12-24T02:11:32.619030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241371752027885:2112] [node 5] ICS09 handshake done sender: [2:7587241371752027863:2117] self: [2:7587241371752027862:0] peer: [5:7587241373792345165:0] socket: 259 qp: -1
2025-12-24T02:11:32.622744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241372347418146:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.619036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241371752027885:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:32.619046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.622758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241372347418146:2088] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.622868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241372347418183:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.623472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241372347418151:2090] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.626804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241372721555788:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23855
2025-12-24T02:11:32.627040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241372721555781:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.627322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241372347418098:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.627125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241372721555689:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.627138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.627146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241372721555689:2050] [node 3] ICP22 created new session: [4:7587241372721555789:2050]
2025-12-24T02:11:32.627337Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.627152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241372721555789:2050] [node 3] ICS09 handshake done sender: [4:7587241372721555781:2121] self: [4:7587241372721555784:0] peer: [3:7587241372347418142:0] socket: 281 qp: -1
2025-12-24T02:11:32.627345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241372347418098:2049] [node 2] ICP22 created new session: [3:7587241372347418196:2049]
2025-12-24T02:11:32.627159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241372721555789:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.627165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.627350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241372347418196:2049] [node 2] ICS09 handshake done sender: [3:7587241372347418183:2119] self: [3:7587241372347418187:0] peer: [2:7587241371752027860:0] socket: 273 qp: -1
2025-12-24T02:11:32.627356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241372347418196:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.627365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.628127Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241372347418146:2088] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.630889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241373792345176:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7798
2025-12-24T02:11:32.631071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241373792345166:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.631161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241373792345076:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.631175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.631181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241373792345076:2050] [node 3] ICP22 created new session: [5:7587241373792345177:2050]
2025-12-24T02:11:32.631187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241373792345177:2050] [node 3] ICS09 handshake done sender: [5:7587241373792345166:2119] self: [5:7587241373792345169:0] peer: [3:7587241372347418145:0] socket: 246 qp: -1
2025-12-24T02:11:32.631192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241373792345177:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.631200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.634931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241372347418139:2084] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.634955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.634965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241372347418139:2084] [node 5] ICP22 created new session: [3:7587241372347418199:2084]
2025-12-24T02:11:32.634971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241372347418199:2084] [node 5] ICS09 handshake done sender: [3:7587241372347418146:2088] self: [3:7587241372347418145:0] peer: [5:7587241373792345169:0] socket: 233 qp: -1
2025-12-24T02:11:32.634976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241372347418199:2084] [node 5] ICS10 traffic start
2025-12-24T02:11:32.634982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.638798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241372347418138:2082] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.638814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.638821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241372347418138:2082] [node 4] ICP22 created new session: [3:7587241372347418201:2082]
2025-12-24T02:11:32.638827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241372347418201:2082] [node 4] ICS09 handshake done sender: [3:7587241372347418151:2090] self: [3:7587241372347418142:0] peer: [4:7587241372721555784:0] socket: 282 qp: -1
2025-12-24T02:11:32.638833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241372347418201:2082] [node 4] ICS10 traffic start
2025-12-24T02:11:32.638840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.651790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:32.651881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241371752027867:2048] [node 1] ICS01 socket: 226 reason# EndOfStream
2025-12-24T02:11:32.651889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241371752027783:2048] [node 1] ICP30 unregister session Session# [2:7587241371752027867:2048] VirtualId# [2:7587241371752027864:0]
2025-12-24T02:11:32.651895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241371752027867:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.651931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.651950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.653119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:32.653221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241372347418181:2048] [node 1] ICS01 socket: 224 reason# EndOfStream
2025-12-24T02:11:32.653234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241372347418097:2048] [node 1] ICP30 unregister session Session# [3:7587241372347418181:2048] VirtualId# [3:7587241372347418171:0]
2025-12-24T02:11:32.653242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241372347418181:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.653302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.653339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.657818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:32.657917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241372721555774:2048] [node 1] ICS01 socket: 245 reason# EndOfStream
2025-12-24T02:11:32.657925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241372721555687:2048] [node 1] ICP30 unregister session Session# [4:7587241372721555774:2048] VirtualId# [4:7587241372721555767:0]
2025-12-24T02:11:32.657934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241372721555774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.658008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.658040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.658775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:32.658854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241373929768716:2048] [node 1] ICS01 socket: 271 reason# EndOfStream
2025-12-24T02:11:32.658861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241373929768613:2048] [node 1] ICP30 unregister session Session# [6:7587241373929768716:2048] VirtualId# [6:7587241373929768697:0]
2025-12-24T02:11:32.658868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241373929768716:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.658918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.658943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.658982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:32.659091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241373792345162:2048] [node 1] ICS07 socket disconnect 269 reason# EndOfStream
2025-12-24T02:11:32.659099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241373792345162:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.659155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:32.659183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241373792345162:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:32.659722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241373792345074:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3511) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.659736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241373792345162:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.659741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241373792345074:2048] [node 1] ICP30 unregister session Session# [5:7587241373792345162:2048] VirtualId# [5:7587241373792345155:0]
2025-12-24T02:11:32.659788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:32.659795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241373792345074:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.659797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3511) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.670770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:32.670865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241372347418196:2049] [node 2] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:32.670875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241372347418098:2049] [node 2] ICP30 unregister session Session# [3:7587241372347418196:2049] VirtualId# [3:7587241372347418187:0]
2025-12-24T02:11:32.670884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241372347418196:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.670920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.670943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.670953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241373792345074:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:32.681698Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:32.682113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:32.682180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241373792345170:2049] [node 2] ICS01 socket: 267 reason# EndOfStream
2025-12-24T02:11:32.682187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241373792345075:2049] [node 2] ICP30 unregister session Session# [5:7587241373792345170:2049] VirtualId# [5:7587241373792345165:0]
2025-12-24T02:11:32.682194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241373792345170:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.682234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.682264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.682658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:32.682784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241373929768710:2049] [node 2] ICS01 socket: 261 reason# EndOfStream
2025-12-24T02:11:32.682793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241373929768614:2049] [node 2] ICP30 unregister session Session# [6:7587241373929768710:2049] VirtualId# [6:7587241373929768698:0]
2025-12-24T02:11:32.682801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241373929768710:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.682863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.682890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.681803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241372721555782:2049] [node 2] ICS07 socket disconnect 232 reason# EndOfStream
2025-12-24T02:11:32.681817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241372721555782:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.681870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:32.681894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241372721555782:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:32.682630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241372721555688:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18327) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:32.682638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241372721555782:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:32.682643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241372721555688:2049] [node 2] ICP30 unregister session Session# [4:7587241372721555782:2049] VirtualId# [4:7587241372721555772:0]
2025-12-24T02:11:32.682685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:32.682706Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241372721555688:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.682710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18327) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:32.694764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241372721555688:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:32.714762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:32.714871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241373929768714:2050] [node 3] ICS01 socket: 236 reason# EndOfStream
2025-12-24T02:11:32.714884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241373929768615:2050] [node 3] ICP30 unregister session Session# [6:7587241373929768714:2050] VirtualId# [6:7587241373929768706:0]
2025-12-24T02:11:32.714892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241373929768714:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.714953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:32.714979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:32.718919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:32.719028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241373792345177:2050] [node 3] ICS01 socket: 246 reason# EndOfStream
2025-12-24T02:11:32.719039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241373792345076:2050] [node 3] ICP30 unregister session Session# [5:7587241373792345177:2050] VirtualId# [5:7587241373792345169:0]
2025-12-24T02:11:32.719045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241373792345177:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.719099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:32.719130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:32.741468Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:32.741573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241373929768712:2080] [node 4] ICS01 socket: 262 reason# EndOfStream
2025-12-24T02:11:32.741584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241373929768655:2080] [node 4] ICP30 unregister session Session# [6:7587241373929768712:2080] VirtualId# [6:7587241373929768703:0]
2025-12-24T02:11:32.741591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241373929768712:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:32.741633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:32.741657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:32.810907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372268761388:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:32.810923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372268761389:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.810869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241374271382557:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.810886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241374271382558:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:32.814826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374195519109:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.814841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374195519110:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.814281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241374956036936:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.814292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241374956036937:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.814297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241374956036938:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.830878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371580539459:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:32.830901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371580539460:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.834812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371580539458:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.842787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241375333294503:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:32.846722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241375333294501:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:32.846836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241375333294502:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.886981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761448:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.886711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241375333294524:2067] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.889729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956036997:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:32.893884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761448:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.894037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372268761456:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:32.894327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241372268761456:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.895019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241375333294520:2065] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.895032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241375333294520:2065] [node 1] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.899877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241374271382623:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:32.900253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382621:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.900265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382622:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.902905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241374956036999:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.904463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761458:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.906685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761458:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.905197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241375333294547:2076] [node 5] ICP01 ready to work
2025-12-24T02:11:32.906800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241374271382619:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:32.906863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382621:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.906952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241374271382633:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:32.909552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241375333294520:2065] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.906772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374195519169:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:32.910821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956036997:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.911008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374195519169:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.911055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374195519174:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:32.911066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374195519175:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:32.911188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241375333294501:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.911203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.911208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241375333294501:2048] [node 1] ICP22 created new session: [6:7587241375333294578:2048]
2025-12-24T02:11:32.911217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241375333294578:2048] [node 1] ICS09 handshake done sender: [6:7587241375333294520:2065] self: [6:7587241375333294519:0] peer: [1:7587241372268761457:0] socket: 228 qp: -1
2025-12-24T02:11:32.911224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241375333294578:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.911233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.911433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241375333294576:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
2025-12-24T02:11:32.911477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241375333294577:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
2025-12-24T02:11:32.913031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241375333294576:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.913054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241375333294577:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.914055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241375333294524:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.911904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241374271382557:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.914160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374195519177:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:32.914229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374195519175:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.914878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374195519181:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:32.915299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241374195519181:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.918631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374195519174:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.918653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241374195519110:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.919401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372268761461:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:32.920551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241374195519186:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.922397Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241375333294502:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28870789ns) {Configure:104(28869263ns) {ProcessConfigured:135(28537670ns) {StartInitialHandshake:157(28537113ns) {DropHandshakes:470(395ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(828ns) {SwitchToState:179(420ns)}}}}}
2025-12-24T02:11:32.922504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241375333294502:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.923494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382625:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.923504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241374271382625:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.922943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956037001:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.922955Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241374956037001:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.923279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037005:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:32.923292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037006:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:32.924343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241374956037001:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.924425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241374956036999:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.924444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.924451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241374956036999:2109] [node 6] ICP22 created new session: [4:7587241374956037008:2109]
2025-12-24T02:11:32.924458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241374956037008:2109] [node 6] ICS09 handshake done sender: [4:7587241374956037001:2110] self: [4:7587241374956037000:0] peer: [6:7587241375333294581:0] socket: 243 qp: -1
2025-12-24T02:11:32.924465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241374956037008:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:32.924473Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.922818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761465:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.926825Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761466:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.926782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241374271382625:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:32.927103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956037006:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.927055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372268761448:2107] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.927247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241372268761456:2110] [node 6] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:32.927259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
::ffff:127.1.0.0
2025-12-24T02:11:32.927265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372268761456:2110] [node 6] ICP22 created new session: [1:7587241372268761467:2110]
2025-12-24T02:11:32.927271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372268761467:2110] [node 6] ICS09 handshake done sender: [1:7587241372268761448:2107] self: [1:7587241372268761457:0] peer: [6:7587241375333294519:0] socket: 229 qp: -1
2025-12-24T02:11:32.927276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372268761467:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:32.927282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:32.927264Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241375333294524:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64633) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241375333294577:2112] held: no
2025-12-24T02:11:32.927371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241375333294587:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
2025-12-24T02:11:32.927506Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241375333294588:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
2025-12-24T02:11:32.927700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241375333294577:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.927905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241375333294576:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.930814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241374271382623:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.930829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:32.930835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241374271382623:2108] [node 6] ICP22 created new session: [2:7587241374271382640:2108]
2025-12-24T02:11:32.930840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241374271382640:2108] [node 6] ICS09 handshake done sender: [2:7587241374271382625:2111] self: [2:7587241374271382624:0] peer: [6:7587241375333294586:0] socket: 240 qp: -1
2025-12-24T02:11:32.930845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241374271382640:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:32.930862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.932258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382643:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.932487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956037005:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.932875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241374956036938:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.930853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241375333294524:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.930868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.930874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241375333294524:2067] [node 4] ICP22 created new session: [6:7587241375333294590:2067]
2025-12-24T02:11:32.930879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241375333294590:2067] [node 4] ICS09 handshake done sender: [6:7587241375333294577:2112] self: [6:7587241375333294581:0] peer: [4:7587241374956037000:0] socket: 254 qp: -1
2025-12-24T02:11:32.930883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241375333294590:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:32.930888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.931071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241375333294502:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.931079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.931083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241375333294502:2049] [node 2] ICP22 created new session: [6:7587241375333294592:2049]
2025-12-24T02:11:32.931088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241375333294592:2049] [node 2] ICS09 handshake done sender: [6:7587241375333294576:2111] self: [6:7587241375333294586:0] peer: [2:7587241374271382624:0] socket: 247 qp: -1
2025-12-24T02:11:32.931092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241375333294592:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.931106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.934720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241375333294564:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.934732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241375333294564:2101] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.934720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761450:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.934736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372268761450:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.934771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382630:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.934780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241374271382630:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.938934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382622:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.939431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371580539526:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:32.939526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241371580539527:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:32.940531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371580539522:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:32.942746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374195519179:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.942758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241374195519179:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.942909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241371580539460:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(28071065ns) {Configure:104(28070009ns) {ProcessConfigured:135(27770047ns) {StartInitialHandshake:157(27769737ns) {DropHandshakes:470(468ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(255ns) PrepareNewSessionHandshake:477(682ns) {SwitchToState:179(471ns)}}}}}
2025-12-24T02:11:32.942960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371580539522:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.945201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371580539538:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:32.945250Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371580539539:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:32.945673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761466:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.945927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241372268761471:2116] [node 5] ICP01 ready to work
2025-12-24T02:11:32.946335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241372268761471:2116] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.946477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372268761450:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.946556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372268761388:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.946573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.946579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372268761388:2048] [node 2] ICP22 created new session: [1:7587241372268761473:2048]
2025-12-24T02:11:32.946584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372268761473:2048] [node 2] ICS09 handshake done sender: [1:7587241372268761450:2108] self: [1:7587241372268761449:0] peer: [2:7587241374271382637:0] socket: 231 qp: -1
2025-12-24T02:11:32.946590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372268761473:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:32.946607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.950817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241375333294564:2101] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.951008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371580539538:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.951037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371580539460:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.951246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241374271382557:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241374271382621:2109] held: no
2025-12-24T02:11:32.951293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382645:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.950906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241375333294503:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.950917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.950923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241375333294503:2050] [node 3] ICP22 created new session: [6:7587241375333294595:2050]
2025-12-24T02:11:32.950930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241375333294595:2050] [node 3] ICS09 handshake done sender: [6:7587241375333294564:2101] self: [6:7587241375333294563:0] peer: [3:7587241374195519182:0] socket: 251 qp: -1
2025-12-24T02:11:32.950936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241375333294595:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.950943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.950899Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241374195519109:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34216371ns) {Configure:104(34215595ns) {ProcessConfigured:135(33807477ns) {StartInitialHandshake:157(33807213ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(556ns) {SwitchToState:179(317ns)}}}}}
2025-12-24T02:11:32.950940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241374195519109:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:32.954771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371580539539:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.954849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371580539459:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.955325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371580539520:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.955336Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241371580539520:2108] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.955383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382646:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.958726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241374195519188:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.958736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241374195519188:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.959026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374195519179:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.958748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761453:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.958764Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372268761453:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.963013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382645:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.963066Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382636:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.963069Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241374271382636:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.963445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382646:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.963879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241374956036936:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(45097790ns) {Configure:104(45093972ns) {ProcessConfigured:135(44762881ns) {StartInitialHandshake:157(44762079ns) {DropHandshakes:470(523ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(511ns)}}}}}
2025-12-24T02:11:32.963947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241374956036936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.966870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761463:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.966888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241372268761463:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:32.966886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241371580539459:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241371580539539:2117] held: no
2025-12-24T02:11:32.967093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241375333294503:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16066726ns) {HandleHandshakeStatus:329(16066112ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(147ns) SwitchToState:179(62ns) ProcessPendingSessionEvents:495(15993319ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15972347ns max# 15891112ns}}}
2025-12-24T02:11:32.965292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241374271382636:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:32.965381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241374271382633:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.965400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.965407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241374271382633:2114] [node 5] ICP22 created new session: [2:7587241374271382652:2114]
2025-12-24T02:11:32.965413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241374271382652:2114] [node 5] ICS09 handshake done sender: [2:7587241374271382636:2115] self: [2:7587241374271382635:0] peer: [5:7587241371580539541:0] socket: 269 qp: -1
2025-12-24T02:11:32.965420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241374271382652:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:32.965431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.965972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241374271382630:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.966052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241374271382558:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.966067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.966099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241374271382558:2049] [node 3] ICP22 created new session: [2:7587241374271382655:2049]
2025-12-24T02:11:32.966105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241374271382655:2049] [node 3] ICS09 handshake done sender: [2:7587241374271382630:2113] self: [2:7587241374271382618:0] peer: [3:7587241374195519184:0] socket: 245 qp: -1
2025-12-24T02:11:32.966111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241374271382655:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.966116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.967698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241374956037018:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:32.971191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241374271382647:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13096
2025-12-24T02:11:32.970817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372268761463:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:32.974715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371580539542:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:32.975019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371580539539:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:32.975084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371580539459:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.975102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:32.975111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371580539459:2049] [node 2] ICP22 created new session: [5:7587241371580539543:2049]
2025-12-24T02:11:32.975118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371580539543:2049] [node 2] ICS09 handshake done sender: [5:7587241371580539539:2117] self: [5:7587241371580539541:0] peer: [2:7587241374271382635:0] socket: 274 qp: -1
2025-12-24T02:11:32.975127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371580539543:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:32.975137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:32.979691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241374271382621:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.979839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371580539520:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:32.979947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241371580539458:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.979964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.979990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371580539458:2048] [node 1] ICP22 created new session: [5:7587241371580539546:2048]
2025-12-24T02:11:32.979997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371580539546:2048] [node 1] ICS09 handshake done sender: [5:7587241371580539520:2108] self: [5:7587241371580539519:0] peer: [1:7587241372268761472:0] socket: 255 qp: -1
2025-12-24T02:11:32.980002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371580539546:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.980009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.982006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241374271382557:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.982027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:32.982035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241374271382557:2048] [node 1] ICP22 created new session: [2:7587241374271382657:2048]
2025-12-24T02:11:32.982041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241374271382657:2048] [node 1] ICS09 handshake done sender: [2:7587241374271382621:2109] self: [2:7587241374271382637:0] peer: [1:7587241372268761449:0] socket: 236 qp: -1
2025-12-24T02:11:32.982048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241374271382657:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:32.982056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:32.983249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372268761461:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.983268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.983274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372268761461:2112] [node 4] ICP22 created new session: [1:7587241372268761477:2112]
2025-12-24T02:11:32.983280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372268761477:2112] [node 4] ICS09 handshake done sender: [1:7587241372268761463:2113] self: [1:7587241372268761462:0] peer: [4:7587241374956037015:0] socket: 260 qp: -1
2025-12-24T02:11:32.983286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372268761477:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:32.983292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.983703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241374195519177:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.983720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:32.983725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374195519177:2111] [node 4] ICP22 created new session: [3:7587241374195519193:2111]
2025-12-24T02:11:32.983730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374195519193:2111] [node 4] ICS09 handshake done sender: [3:7587241374195519179:2112] self: [3:7587241374195519178:0] peer: [4:7587241374956037010:0] socket: 259 qp: -1
2025-12-24T02:11:32.983735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374195519193:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:32.983752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:32.984503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374195519110:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241374195519174:2109] held: no
2025-12-24T02:11:32.985292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372268761453:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.986883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374195519188:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.986969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241372268761389:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.986990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.987000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372268761389:2049] [node 3] ICP22 created new session: [1:7587241372268761482:2049]
2025-12-24T02:11:32.987005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372268761482:2049] [node 3] ICS09 handshake done sender: [1:7587241372268761453:2109] self: [1:7587241372268761452:0] peer: [3:7587241374195519191:0] socket: 232 qp: -1
2025-12-24T02:11:32.987011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372268761482:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:32.987020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.987300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761479:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.987336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761480:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
2025-12-24T02:11:32.987413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241372268761481:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14169
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.989455Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241371580539549:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:32.986992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241374195519186:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:32.987005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.987011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374195519186:2114] [node 5] ICP22 created new session: [3:7587241374195519197:2114]
2025-12-24T02:11:32.987016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241374195519197:2114] [node 5] ICS09 handshake done sender: [3:7587241374195519188:2115] self: [3:7587241374195519187:0] peer: [5:7587241371580539540:0] socket: 261 qp: -1
2025-12-24T02:11:32.987020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241374195519197:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:32.987027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.989665Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371580539538:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:32.989721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371580539460:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.989736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:32.989741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371580539460:2050] [node 3] ICP22 created new session: [5:7587241371580539550:2050]
2025-12-24T02:11:32.989746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241371580539550:2050] [node 3] ICS09 handshake done sender: [5:7587241371580539538:2116] self: [5:7587241371580539540:0] peer: [3:7587241374195519187:0] socket: 264 qp: -1
2025-12-24T02:11:32.989751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241371580539550:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:32.989756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:32.988959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037020:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:32.989001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037021:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.994961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761479:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.995013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241372268761466:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.995203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:32.999026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241372268761471:2116] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:32.999044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:32.999051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241372268761471:2116] [node 5] ICP22 created new session: [1:7587241372268761484:2116]
2025-12-24T02:11:32.999056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241372268761484:2116] [node 5] ICS09 handshake done sender: [1:7587241372268761466:2115] self: [1:7587241372268761472:0] peer: [5:7587241371580539519:0] socket: 262 qp: -1
2025-12-24T02:11:32.999062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241372268761484:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:32.999070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:32.999385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241372268761481:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:32.999406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241372268761461:2112] [node 4] ICP18 handshake (actor [1:7587241372268761479:2117]) is too old
2025-12-24T02:11:32.999639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241372268761389:2049] [node 3] ICP18 handshake (actor [1:7587241372268761481:2119]) is too old
2025-12-24T02:11:33.002754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241374195519199:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:33.006009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241374956037023:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:33.006338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956037020:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.006375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241374956037022:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.006228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374195519109:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241374195519169:2107] held: no
2025-12-24T02:11:33.006408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374195519175:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.006504Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241374956037018:2115] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(38743413ns) {Configure:104(38742556ns) {ProcessConfigured:135(38367890ns) {StartInitialHandshake:157(38366677ns) {DropHandshakes:470(438ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(1065ns) {SwitchToState:179(370ns)}}}}}
2025-12-24T02:11:33.006526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241374956036936:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241374956037006:2112] held: no
2025-12-24T02:11:33.006628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241374956036937:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.006673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241374956037005:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.006996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241374956037006:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.010775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241374271382638:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.010791Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241374271382638:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.010981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241374195519181:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.011002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.011009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374195519181:2113] [node 6] ICP22 created new session: [3:7587241378490486498:2113]
2025-12-24T02:11:33.011018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241378490486498:2113] [node 6] ICS09 handshake done sender: [3:7587241374195519175:2110] self: [3:7587241374195519182:0] peer: [6:7587241375333294563:0] socket: 253 qp: -1
2025-12-24T02:11:33.011026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241378490486498:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:33.011033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.011406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241378490486496:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:33.012180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241374956036936:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.012197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.012226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241374956036936:2048] [node 1] ICP22 created new session: [4:7587241379251004324:2048]
2025-12-24T02:11:33.012232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379251004324:2048] [node 1] ICS09 handshake done sender: [4:7587241374956037006:2112] self: [4:7587241374956037015:0] peer: [1:7587241372268761462:0] socket: 265 qp: -1
2025-12-24T02:11:33.012237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379251004324:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.012245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.012363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241374956036938:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.012381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.012385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241374956036938:2050] [node 3] ICP22 created new session: [4:7587241379251004326:2050]
2025-12-24T02:11:33.012389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379251004326:2050] [node 3] ICS09 handshake done sender: [4:7587241374956037005:2111] self: [4:7587241374956037010:0] peer: [3:7587241374195519178:0] socket: 263 qp: -1
2025-12-24T02:11:33.012394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379251004326:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.012400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.015297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374195519174:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.018811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241374271382638:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.014919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241375875506848:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:33.017100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241379628261893:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.019014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241378490486497:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58413
2025-12-24T02:11:33.018952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241374271382619:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.018972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.018984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241374271382619:2107] [node 4] ICP22 created new session: [2:7587241378566349956:2107]
2025-12-24T02:11:33.018991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378566349956:2107] [node 4] ICS09 handshake done sender: [2:7587241374271382638:2116] self: [2:7587241374271382634:0] peer: [4:7587241379251004321:0] socket: 270 qp: -1
2025-12-24T02:11:33.018997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378566349956:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:33.019005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.019984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241379628261893:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.020062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241375333294547:2076] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.019478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241374195519110:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.019491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.019512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374195519110:2049] [node 2] ICP22 created new session: [3:7587241378490486500:2049]
2025-12-24T02:11:33.019518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241378490486500:2049] [node 2] ICS09 handshake done sender: [3:7587241374195519174:2109] self: [3:7587241374195519184:0] peer: [2:7587241374271382618:0] socket: 248 qp: -1
2025-12-24T02:11:33.019523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241378490486500:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.019528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.022921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241375875506848:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.022959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241371580539527:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.023232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241371580539533:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.023240Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241371580539533:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.023172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241374195519169:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.034720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241374195519109:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.034742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.034752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241374195519109:2048] [node 1] ICP22 created new session: [3:7587241378490486502:2048]
2025-12-24T02:11:33.034762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241378490486502:2048] [node 1] ICS09 handshake done sender: [3:7587241374195519169:2107] self: [3:7587241374195519191:0] peer: [1:7587241372268761452:0] socket: 237 qp: -1
2025-12-24T02:11:33.034771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241378490486502:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.034784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.035003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379251004320:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.035018Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241379251004320:2120] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.039495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241379628261895:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27160
2025-12-24T02:11:33.042648Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241374956036937:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241374956037022:2118] held: no
2025-12-24T02:11:33.042884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241371580539533:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.043075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379251004320:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.046749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241371580539526:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.046785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.046804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371580539526:2110] [node 6] ICP22 created new session: [5:7587241375875506851:2110]
2025-12-24T02:11:33.046812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241375875506851:2110] [node 6] ICS09 handshake done sender: [5:7587241371580539533:2114] self: [5:7587241371580539532:0] peer: [6:7587241379628261894:0] socket: 241 qp: -1
2025-12-24T02:11:33.046821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241375875506851:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:33.046831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.046876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241379628261893:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.050719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241375333294547:2076] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.050738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.050785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241375333294547:2076] [node 5] ICP22 created new session: [6:7587241379628261896:2076]
2025-12-24T02:11:33.050791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241379628261896:2076] [node 5] ICS09 handshake done sender: [6:7587241379628261893:2119] self: [6:7587241379628261894:0] peer: [5:7587241371580539532:0] socket: 289 qp: -1
2025-12-24T02:11:33.050799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241379628261896:2076] [node 5] ICS10 traffic start
2025-12-24T02:11:33.050808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.046922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241374956037018:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.046949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.046957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241374956037018:2115] [node 5] ICP22 created new session: [4:7587241379251004330:2115]
2025-12-24T02:11:33.046964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379251004330:2115] [node 5] ICS09 handshake done sender: [4:7587241379251004320:2120] self: [4:7587241374956037019:0] peer: [5:7587241375875506849:0] socket: 283 qp: -1
2025-12-24T02:11:33.046970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379251004330:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:33.046978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.051435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241375875506853:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21035
2025-12-24T02:11:33.051580Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241375875506848:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.051787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241371580539527:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.051797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.051803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241371580539527:2111] [node 4] ICP22 created new session: [5:7587241375875506854:2111]
2025-12-24T02:11:33.051809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241375875506854:2111] [node 4] ICS09 handshake done sender: [5:7587241375875506848:2120] self: [5:7587241375875506849:0] peer: [4:7587241374956037019:0] socket: 291 qp: -1
2025-12-24T02:11:33.051815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241375875506854:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:33.051822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.051360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379251004329:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31228
2025-12-24T02:11:33.055098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241374956037022:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.055256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241374956036937:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.055270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.055277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241374956036937:2049] [node 2] ICP22 created new session: [4:7587241379251004332:2049]
2025-12-24T02:11:33.055283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379251004332:2049] [node 2] ICS09 handshake done sender: [4:7587241374956037022:2118] self: [4:7587241379251004321:0] peer: [2:7587241374271382634:0] socket: 293 qp: -1
2025-12-24T02:11:33.055290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379251004332:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.055297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.071045Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241375333294547:2076] [node 5] ICP35 event processing took too much time PendingConnection:279(20193870ns) {HandleHandshakeStatus:329(20192873ns) {DropIncomingHandshake:442(124ns) DropOutgoingHandshake:457(2238ns) SwitchToState:179(315ns) ProcessPendingSessionEvents:495(20067846ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20041131ns max# 19955447ns}}}
2025-12-24T02:11:33.074848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241374956037018:2115] [node 5] ICP35 event processing took too much time PendingConnection:279(27863802ns) {HandleHandshakeStatus:329(27862629ns) {DropIncomingHandshake:442(4932ns) DropOutgoingHandshake:457(108ns) SwitchToState:179(101ns) ProcessPendingSessionEvents:495(27718063ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 27712672ns max# 27691791ns}}}
2025-12-24T02:11:33.223255Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:33.223358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241374271382657:2048] [node 1] ICS07 socket disconnect 236 reason# EndOfStream
2025-12-24T02:11:33.223366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241374271382657:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.223404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.223421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241374271382657:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:33.234862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241374271382557:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:33.234881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241374271382657:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:33.234814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:33.234916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241378490486502:2048] [node 1] ICS01 socket: 237 reason# EndOfStream
2025-12-24T02:11:33.234929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241374195519109:2048] [node 1] ICP30 unregister session Session# [3:7587241378490486502:2048] VirtualId# [3:7587241374195519191:0]
2025-12-24T02:11:33.234939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241378490486502:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.234992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.235024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.235728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241374271382557:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:33.235742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241374271382657:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:33.235747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241374271382557:2048] [node 1] ICP30 unregister session Session# [2:7587241374271382657:2048] VirtualId# [2:7587241374271382637:0]
2025-12-24T02:11:33.235764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.235771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241374271382557:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.235774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22839) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.237497Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:33.237587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241375333294578:2048] [node 1] ICS01 socket: 228 reason# EndOfStream
2025-12-24T02:11:33.237596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241375333294501:2048] [node 1] ICP30 unregister session Session# [6:7587241375333294578:2048] VirtualId# [6:7587241375333294519:0]
2025-12-24T02:11:33.237604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241375333294578:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.237652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.237676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.238996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:33.239078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241379251004324:2048] [node 1] ICS01 socket: 265 reason# EndOfStream
2025-12-24T02:11:33.239084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241374956036936:2048] [node 1] ICP30 unregister session Session# [4:7587241379251004324:2048] VirtualId# [4:7587241374956037015:0]
2025-12-24T02:11:33.239091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241379251004324:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.239129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.239159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.239209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:33.239279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371580539546:2048] [node 1] ICS01 socket: 255 reason# EndOfStream
2025-12-24T02:11:33.239286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371580539458:2048] [node 1] ICP30 unregister session Session# [5:7587241371580539546:2048] VirtualId# [5:7587241371580539519:0]
2025-12-24T02:11:33.239292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371580539546:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.239315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.239333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.259393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:33.259496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241378490486500:2049] [node 2] ICS07 socket disconnect 248 reason# EndOfStream
2025-12-24T02:11:33.259502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241378490486500:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.259537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.259554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241378490486500:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:33.263045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241374195519110:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:33.263063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241378490486500:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:33.266462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:33.266558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371580539543:2049] [node 2] ICS01 socket: 274 reason# EndOfStream
2025-12-24T02:11:33.266569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371580539459:2049] [node 2] ICP30 unregister session Session# [5:7587241371580539543:2049] VirtualId# [5:7587241371580539541:0]
2025-12-24T02:11:33.266577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371580539543:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.266624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.266649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.266601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:33.266688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241379251004332:2049] [node 2] ICS07 socket disconnect 293 reason# EndOfStream
2025-12-24T02:11:33.266705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241379251004332:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.266758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.266782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241379251004332:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:33.267361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241374956036937:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:33.267374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241379251004332:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:33.267379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241374956036937:2049] [node 2] ICP30 unregister session Session# [4:7587241379251004332:2049] VirtualId# [4:7587241379251004321:0]
2025-12-24T02:11:33.267416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.267423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241374956036937:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.267427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10291) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.270846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:33.270978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241375333294592:2049] [node 2] ICS01 socket: 247 reason# EndOfStream
2025-12-24T02:11:33.270988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241375333294502:2049] [node 2] ICP30 unregister session Session# [6:7587241375333294592:2049] VirtualId# [6:7587241375333294586:0]
2025-12-24T02:11:33.270996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241375333294592:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.271048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.271076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.278743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241374956036937:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:33.290762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:33.290863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241375333294595:2050] [node 3] ICS01 socket: 251 reason# EndOfStream
2025-12-24T02:11:33.290871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241375333294503:2050] [node 3] ICP30 unregister session Session# [6:7587241375333294595:2050] VirtualId# [6:7587241375333294563:0]
2025-12-24T02:11:33.290878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241375333294595:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.290941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:33.290976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:33.290805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:33.290897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241371580539550:2050] [node 3] ICS07 socket disconnect 264 reason# EndOfStream
2025-12-24T02:11:33.290904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241371580539550:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.290942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:33.290971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241371580539550:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:33.291381Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241371580539460:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11748) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:33.291391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241371580539550:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:33.291395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241371580539460:2050] [node 3] ICP30 unregister session Session# [5:7587241371580539550:2050] VirtualId# [5:7587241371580539540:0]
2025-12-24T02:11:33.291413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:33.291420Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241371580539460:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.291423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11748) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.306839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:33.306936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241375333294590:2067] [node 4] ICS01 socket: 254 reason# EndOfStream
2025-12-24T02:11:33.306946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241375333294524:2067] [node 4] ICP30 unregister session Session# [6:7587241375333294590:2067] VirtualId# [6:7587241375333294581:0]
2025-12-24T02:11:33.306953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241375333294590:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.307013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:33.307038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:33.342821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241379209300603:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:33.342837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241379209300604:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:33.358782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241377660003593:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.358801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241377660003594:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.354833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241379221299162:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.354849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241379221299163:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:33.356920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378615154884:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.356929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378615154885:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.356933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378615154886:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.355919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241379111126408:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.355928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241379111126409:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.355934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241379111126410:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.374993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241377818838398:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.375011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241377818838399:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.395091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241377818838397:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.409419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241379221299223:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.411317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241379221299231:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:33.411189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126469:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.410667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241379209300665:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:33.410925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241379209300667:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:33.411110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241379209300664:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-12-24T02:11:33.415081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241377660003655:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.413831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241379221299236:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:33.415734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838409:2059] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.415754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838408:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.417282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241377818838408:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.417739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241377818838398:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.417902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299238:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.417911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241379221299238:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.418691Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838415:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.418868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241379221299238:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.419018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241377818838428:2067] [node 5] ICP01 ready to work
2025-12-24T02:11:33.419696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241377818838408:2058] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.420158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126469:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.420517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241379111126409:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.419722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241379221299236:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.419740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.419751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241379221299236:2112] [node 6] ICP22 created new session: [2:7587241379221299241:2112]
2025-12-24T02:11:33.419757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241379221299241:2112] [node 6] ICS09 handshake done sender: [2:7587241379221299238:2113] self: [2:7587241379221299237:0] peer: [6:7587241377818838413:0] socket: 239 qp: -1
2025-12-24T02:11:33.419766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241379221299241:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:33.419776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.419777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241377818838398:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.419790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.419796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241377818838398:2049] [node 2] ICP22 created new session: [6:7587241377818838432:2049]
2025-12-24T02:11:33.419802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241377818838432:2049] [node 2] ICS09 handshake done sender: [6:7587241377818838408:2058] self: [6:7587241377818838413:0] peer: [2:7587241379221299237:0] socket: 241 qp: -1
2025-12-24T02:11:33.419808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241377818838432:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.419826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.420721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299227:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.420726Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241379221299227:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.421395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126472:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.422433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241379221299227:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.422622Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379111126469:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.422392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378615154922:2081] [node 6] ICP01 ready to work
2025-12-24T02:11:33.422764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241379111126409:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.422778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.422785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241379111126409:2049] [node 2] ICP22 created new session: [4:7587241379111126473:2049]
2025-12-24T02:11:33.422793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379111126473:2049] [node 2] ICS09 handshake done sender: [4:7587241379111126469:2108] self: [4:7587241379111126471:0] peer: [2:7587241379221299226:0] socket: 238 qp: -1
2025-12-24T02:11:33.422800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379111126473:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.422836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.423231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241379221299223:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.423243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.423250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241379221299223:2107] [node 4] ICP22 created new session: [2:7587241379221299244:2107]
2025-12-24T02:11:33.423255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241379221299244:2107] [node 4] ICS09 handshake done sender: [2:7587241379221299227:2109] self: [2:7587241379221299226:0] peer: [4:7587241379111126471:0] socket: 235 qp: -1
2025-12-24T02:11:33.423259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241379221299244:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:33.423264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.422958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241377660003661:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:33.423081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241377660003662:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.427075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378615154935:2087] [node 4] ICP01 ready to work
2025-12-24T02:11:33.427135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154936:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.429675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241377818838445:2080] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.430782Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241379209300604:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19622841ns) {Configure:104(19622181ns) {ProcessConfigured:135(17251219ns) {StartInitialHandshake:157(17250539ns) {DropHandshakes:470(134ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(944ns) {SwitchToState:179(520ns)}}}}}
2025-12-24T02:11:33.431004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241379209300676:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-12-24T02:11:33.431203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241379209300667:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20230942ns) {Configure:104(20230657ns) {ProcessConfigured:135(17523078ns) {StartInitialHandshake:157(17522651ns) {DropHandshakes:470(323ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(19ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(450ns) {SwitchToState:179(210ns)}}}}}
2025-12-24T02:11:33.431529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300664:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.433153Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241379209300665:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22416009ns) {Configure:104(22414942ns) {ProcessConfigured:135(19369440ns) {StartInitialHandshake:157(19369127ns) {DropHandshakes:470(188ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(97ns) PrepareNewSessionHandshake:477(1183ns) {SwitchToState:179(33ns)}}}}}
2025-12-24T02:11:33.433035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241377660003593:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19612977ns) {Configure:104(19612193ns) {ProcessConfigured:135(19228058ns) {StartInitialHandshake:157(19227545ns) {DropHandshakes:470(580ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(172ns) PrepareNewSessionHandshake:477(1604ns) {SwitchToState:179(670ns)}}}}}
2025-12-24T02:11:33.436795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241379111126478:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:33.435177Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241377818838399:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17379810ns) {Configure:104(17378962ns) {ProcessConfigured:135(17176418ns) {StartInitialHandshake:157(17176126ns) {DropHandshakes:470(416ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(121ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1165ns) {SwitchToState:179(519ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.440032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241378615154922:2081] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17271831ns) {Configure:104(17259966ns) {ProcessConfigured:135(15301991ns) {StartInitialHandshake:157(15301561ns) {DropHandshakes:470(240ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(880ns) {SwitchToState:179(484ns)}}}}}
2025-12-24T02:11:33.440705Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003672:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.440733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003673:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.440797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003674:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.440815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003675:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.440954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241379221299231:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29244537ns) {Configure:104(29244231ns) {ProcessConfigured:135(29053568ns) {StartInitialHandshake:157(29053372ns) {DropHandshakes:470(204ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:477(657ns) {SwitchToState:179(34ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.441649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126480:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.441683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126481:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.441720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126482:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.441966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241379221299249:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2057
2025-12-24T02:11:33.441980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241379221299250:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2057
2025-12-24T02:11:33.442013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241379221299251:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2057
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.443159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299250:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.443739Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299251:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.443788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241379221299162:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.443956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300670:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.443968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241379209300670:2111] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.444778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241379221299254:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2057
2025-12-24T02:11:33.444920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241379209300670:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.445045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241379221299251:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.444041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126481:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.444076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126482:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.442828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838478:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.445124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241379221299162:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.445146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.445152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241379221299162:2048] [node 1] ICP22 created new session: [2:7587241379221299255:2048]
2025-12-24T02:11:33.445159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241379221299255:2048] [node 1] ICS09 handshake done sender: [2:7587241379221299251:2117] self: [2:7587241379221299253:0] peer: [1:7587241379209300669:0] socket: 276 qp: -1
2025-12-24T02:11:33.445166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241379221299255:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.445186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.446454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241379209300603:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.446475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.446481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241379209300603:2048] [node 2] ICP22 created new session: [1:7587241379209300685:2048]
2025-12-24T02:11:33.446488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241379209300685:2048] [node 2] ICS09 handshake done sender: [1:7587241379209300670:2111] self: [1:7587241379209300669:0] peer: [2:7587241379221299253:0] socket: 263 qp: -1
2025-12-24T02:11:33.446494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241379209300685:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:33.446500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.448772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300682:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.448779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241379209300682:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.449470Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241379209300682:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.449603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241379209300665:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.449617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.449624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241379209300665:2107] [node 6] ICP22 created new session: [1:7587241379209300688:2107]
2025-12-24T02:11:33.449630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241379209300688:2107] [node 6] ICS09 handshake done sender: [1:7587241379209300682:2115] self: [1:7587241379209300674:0] peer: [6:7587241377818838480:0] socket: 234 qp: -1
2025-12-24T02:11:33.449636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241379209300688:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:33.449642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.447093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838479:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.448606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241377818838479:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.448640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241377818838397:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.454135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003673:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.454295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241377660003593:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.454459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299249:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.454772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126480:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.458008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154963:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.458356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154963:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.456795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838481:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.457000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241377818838478:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.459230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241377818838445:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29437739ns) {Configure:104(29436533ns) {ProcessConfigured:135(29085862ns) {StartInitialHandshake:157(29084879ns) {DropHandshakes:470(482ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(291ns) PrepareNewSessionHandshake:477(1053ns) {SwitchToState:179(596ns)}}}}}
2025-12-24T02:11:33.460348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241377818838445:2080] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.460378Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241377818838479:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.460502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241377818838397:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.460527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.460538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241377818838397:2048] [node 1] ICP22 created new session: [6:7587241377818838484:2048]
2025-12-24T02:11:33.460545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241377818838484:2048] [node 1] ICS09 handshake done sender: [6:7587241377818838479:2117] self: [6:7587241377818838480:0] peer: [1:7587241379209300674:0] socket: 282 qp: -1
2025-12-24T02:11:33.460553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241377818838484:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.460578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.460775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003674:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.463122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241379111126410:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26235243ns) {Configure:104(26233962ns) {ProcessConfigured:135(25913242ns) {StartInitialHandshake:157(25912642ns) {DropHandshakes:470(1073ns) {DropIncomingHandshake:442(121ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(2224ns) {SwitchToState:179(833ns)}}}}}
2025-12-24T02:11:33.463202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241379111126410:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.463265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126484:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.463268Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241379111126484:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.464762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241379209300671:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:33.464847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241379111126477:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:33.464951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379111126484:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.465190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241379111126478:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.465202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.465211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241379111126478:2111] [node 6] ICP22 created new session: [4:7587241379111126492:2111]
2025-12-24T02:11:33.465218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379111126492:2111] [node 6] ICS09 handshake done sender: [4:7587241379111126484:2115] self: [4:7587241379111126483:0] peer: [6:7587241377818838483:0] socket: 274 qp: -1
2025-12-24T02:11:33.465224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379111126492:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:33.465231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.465566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003672:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.466730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300675:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.466749Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241379209300675:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.466823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299247:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.466836Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241379221299247:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.470884Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241379111126408:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43185757ns) {Configure:104(43184537ns) {ProcessConfigured:135(42798977ns) {StartInitialHandshake:157(42798189ns) {DropHandshakes:470(922ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(128ns)} GenerateSessionVirtualId:988(329ns) PrepareNewSessionHandshake:477(1552ns) {SwitchToState:179(1111ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.475933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003664:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.475947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241377660003664:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.475068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241379209300691:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-12-24T02:11:33.475494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241379111126408:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.478779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378615154885:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.478829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241378615154885:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20404979ns) {IncomingHandshake:261(20403945ns) {DropIncomingHandshake:442(239ns) PrepareNewSessionHandshake:477(705ns) {SwitchToState:179(302ns)} GenerateSessionVirtualId:988(328ns) IssueIncomingHandshakeReply:193(20398760ns) {SwitchToState:179(731ns)}}}
2025-12-24T02:11:33.478868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300691:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.480477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241377818838409:2059] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.480518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241377818838399:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.479228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003679:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.480644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003657:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.480649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241377660003657:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.481260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241377660003657:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.481344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241377660003655:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.481361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.481370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241377660003655:2107] [node 6] ICP22 created new session: [3:7587241377660003681:2107]
2025-12-24T02:11:33.481376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241377660003681:2107] [node 6] ICS09 handshake done sender: [3:7587241377660003657:2108] self: [3:7587241377660003656:0] peer: [6:7587241377818838486:0] socket: 240 qp: -1
2025-12-24T02:11:33.481383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241377660003681:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:33.481393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.483421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154936:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.484834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241379209300675:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.485510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241379221299233:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.485530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241379221299233:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.485644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241377818838399:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12089) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241377818838409:2059] held: no
2025-12-24T02:11:33.483304Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003675:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.483347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241377660003594:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.486295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300679:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.486312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241379209300679:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.486720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241379209300604:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.486743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.486757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241379209300604:2049] [node 3] ICP22 created new session: [1:7587241379209300694:2049]
2025-12-24T02:11:33.486767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241379209300694:2049] [node 3] ICS09 handshake done sender: [1:7587241379209300675:2112] self: [1:7587241379209300672:0] peer: [3:7587241377660003676:0] socket: 262 qp: -1
2025-12-24T02:11:33.486775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241379209300694:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:33.486783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.490679Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154966:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.490797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241377660003664:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.491315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154966:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.491360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378615154935:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.492273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154969:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.492905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378615154966:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.493015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378615154935:2087] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.493048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.493054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378615154935:2087] [node 4] ICP22 created new session: [5:7587241378615154970:2087]
2025-12-24T02:11:33.493061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378615154970:2087] [node 4] ICS09 handshake done sender: [5:7587241378615154966:2117] self: [5:7587241378615154968:0] peer: [4:7587241379111126494:0] socket: 291 qp: -1
2025-12-24T02:11:33.493069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378615154970:2087] [node 4] ICS10 traffic start
2025-12-24T02:11:33.493078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.491267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241379111126478:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(25989997ns) {HandleHandshakeStatus:329(25989385ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(263ns) SwitchToState:179(168ns) ProcessPendingSessionEvents:495(12372347ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 12358799ns max# 12302319ns}}}
2025-12-24T02:11:33.491530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241379111126495:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.491536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241379111126495:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.492713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379111126495:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.494905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241377660003661:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.494924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.494932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241377660003661:2109] [node 4] ICP22 created new session: [3:7587241377660003686:2109]
2025-12-24T02:11:33.494940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241377660003686:2109] [node 4] ICS09 handshake done sender: [3:7587241377660003664:2111] self: [3:7587241377660003663:0] peer: [4:7587241379111126489:0] socket: 251 qp: -1
2025-12-24T02:11:33.494947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241377660003686:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:33.494986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.495518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154973:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.495749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838487:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.495521Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241379209300671:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30598851ns) {Configure:104(30597022ns) {ProcessConfigured:135(30196313ns) {StartInitialHandshake:157(30195712ns) {DropHandshakes:470(883ns) {DropIncomingHandshake:442(173ns) DropOutgoingHandshake:457(182ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(1532ns) {SwitchToState:179(1149ns)}}}}}
2025-12-24T02:11:33.495755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241379209300697:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-12-24T02:11:33.495827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241379111126477:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.495851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.495860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241379111126477:2110] [node 5] ICP22 created new session: [4:7587241379111126501:2110]
2025-12-24T02:11:33.495867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379111126501:2110] [node 5] ICS09 handshake done sender: [4:7587241379111126495:2118] self: [4:7587241379111126494:0] peer: [5:7587241378615154968:0] socket: 290 qp: -1
2025-12-24T02:11:33.495872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379111126501:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:33.495879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.495947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241379221299247:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.496662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241379221299231:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.496677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.496683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241379221299231:2110] [node 5] ICP22 created new session: [2:7587241379221299259:2110]
2025-12-24T02:11:33.496689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241379221299259:2110] [node 5] ICS09 handshake done sender: [2:7587241379221299247:2114] self: [2:7587241379221299234:0] peer: [5:7587241378615154964:0] socket: 271 qp: -1
2025-12-24T02:11:33.496695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241379221299259:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:33.496705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.496542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378615154963:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.498588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003679:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.498937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241377818838478:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.499165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241377818838445:2080] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.499183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.499191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241377818838445:2080] [node 4] ICP22 created new session: [6:7587241377818838488:2080]
2025-12-24T02:11:33.499196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241377818838488:2080] [node 4] ICS09 handshake done sender: [6:7587241377818838478:2116] self: [6:7587241377818838483:0] peer: [4:7587241379111126483:0] socket: 277 qp: -1
2025-12-24T02:11:33.499203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241377818838488:2080] [node 4] ICS10 traffic start
2025-12-24T02:11:33.499213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.498358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300697:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.502891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126499:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.502839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241379221299233:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.503651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241379221299163:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.503670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.503677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241379221299163:2049] [node 3] ICP22 created new session: [2:7587241379221299262:2049]
2025-12-24T02:11:33.503684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241379221299262:2049] [node 3] ICS09 handshake done sender: [2:7587241379221299233:2111] self: [2:7587241379221299232:0] peer: [3:7587241377660003684:0] socket: 267 qp: -1
2025-12-24T02:11:33.503691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241379221299262:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:33.503698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.504007Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241379209300665:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:33.506718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154967:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.506815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378615154885:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.506829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.506837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378615154885:2049] [node 2] ICP22 created new session: [5:7587241378615154975:2049]
2025-12-24T02:11:33.506843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378615154975:2049] [node 2] ICS09 handshake done sender: [5:7587241378615154963:2116] self: [5:7587241378615154964:0] peer: [2:7587241379221299234:0] socket: 278 qp: -1
2025-12-24T02:11:33.506849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378615154975:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.506856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.506847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379111126480:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.507094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241379111126410:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.507103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.507110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241379111126410:2050] [node 3] ICP22 created new session: [4:7587241379111126503:2050]
2025-12-24T02:11:33.507117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379111126503:2050] [node 3] ICS09 handshake done sender: [4:7587241379111126480:2112] self: [4:7587241379111126489:0] peer: [3:7587241377660003663:0] socket: 255 qp: -1
2025-12-24T02:11:33.507122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379111126503:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.507128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.515013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241379209300604:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(28169391ns) {HandleHandshakeStatus:329(28168616ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(386ns) SwitchToState:179(264ns) ProcessPendingSessionEvents:495(28050853ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 28029327ns max# 27949903ns}}}
2025-12-24T02:11:33.515104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241379209300604:2049] [node 3] ICP18 handshake (actor [1:7587241379209300697:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.515495Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003689:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.515635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241377660003690:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14639
2025-12-24T02:11:33.515824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241377660003675:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.515913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241377660003594:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.515932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.515940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241377660003594:2049] [node 2] ICP22 created new session: [3:7587241377660003691:2049]
2025-12-24T02:11:33.515947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241377660003691:2049] [node 2] ICS09 handshake done sender: [3:7587241377660003675:2118] self: [3:7587241377660003684:0] peer: [2:7587241379221299232:0] socket: 270 qp: -1
2025-12-24T02:11:33.515955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241377660003691:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.515963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.516905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241377660003673:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.516988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241377660003593:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.517005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.517011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241377660003593:2048] [node 1] ICP22 created new session: [3:7587241377660003693:2048]
2025-12-24T02:11:33.517017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241377660003693:2048] [node 1] ICS09 handshake done sender: [3:7587241377660003673:2116] self: [3:7587241377660003676:0] peer: [1:7587241379209300672:0] socket: 268 qp: -1
2025-12-24T02:11:33.517023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241377660003693:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.517044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.522963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838491:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.522982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838492:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.523077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241377818838409:2059] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.523245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241377818838399:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.523263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.523268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241377818838399:2050] [node 3] ICP22 created new session: [6:7587241377818838493:2050]
2025-12-24T02:11:33.523275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241377818838493:2050] [node 3] ICS09 handshake done sender: [6:7587241377818838409:2059] self: [6:7587241377818838486:0] peer: [3:7587241377660003656:0] socket: 242 qp: -1
2025-12-24T02:11:33.523282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241377818838493:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.523289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.530914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241377818838492:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.530975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241377818838428:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.538985Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241377660003661:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(43950926ns) {HandleHandshakeStatus:329(43950197ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(102ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(36020853ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 35999635ns max# 35917793ns}}}
2025-12-24T02:11:33.539076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241377660003661:2109] [node 4] ICP18 handshake (actor [3:7587241377660003679:2119]) is too old
2025-12-24T02:11:33.539962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241379209300679:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.540072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241379209300667:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.540095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.540103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241379209300667:2109] [node 4] ICP22 created new session: [1:7587241379209300700:2109]
2025-12-24T02:11:33.540110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241379209300700:2109] [node 4] ICS09 handshake done sender: [1:7587241379209300679:2114] self: [1:7587241379209300673:0] peer: [4:7587241379111126497:0] socket: 264 qp: -1
2025-12-24T02:11:33.540116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241379209300700:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:33.540124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.543329Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241379221299163:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(39549830ns) {HandleHandshakeStatus:329(39549337ns) {DropIncomingHandshake:442(4384ns) DropOutgoingHandshake:457(110ns) SwitchToState:179(366ns) ProcessPendingSessionEvents:495(39464138ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 39416386ns max# 39327539ns}}}
2025-12-24T02:11:33.544334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154967:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.544421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154960:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.544425Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241378615154960:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.544928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378615154886:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.546964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241377660003668:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.546976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241377660003668:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.548613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241379209300702:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57920
2025-12-24T02:11:33.547824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241379111126410:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(40623210ns) {HandleHandshakeStatus:329(40622772ns) {DropIncomingHandshake:442(139ns) DropOutgoingHandshake:457(819ns) SwitchToState:179(92ns) ProcessPendingSessionEvents:495(40560394ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40534270ns max# 40440937ns}}}
2025-12-24T02:11:33.548113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241379111126408:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241379111126482:2114] held: no
2025-12-24T02:11:33.548300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241379111126505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57617
2025-12-24T02:11:33.548848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154974:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.548939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378615154960:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.554859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241379111126482:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.555012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300702:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.555057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241379209300696:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.555060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241379209300696:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.554766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378615154974:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.554888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378615154884:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.558795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241378615154922:2081] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.558824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.558834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378615154922:2081] [node 6] ICP22 created new session: [5:7587241378615154981:2081]
2025-12-24T02:11:33.558842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378615154981:2081] [node 6] ICS09 handshake done sender: [5:7587241378615154960:2114] self: [5:7587241378615154926:0] peer: [6:7587241377818838495:0] socket: 300 qp: -1
2025-12-24T02:11:33.558852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378615154981:2081] [node 6] ICS10 traffic start
2025-12-24T02:11:33.558862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.558954Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241378615154885:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(52035841ns) {HandleHandshakeStatus:329(52035187ns) {DropIncomingHandshake:442(177ns) DropOutgoingHandshake:457(1277ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(27886891ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 27856764ns max# 27780672ns}}}
2025-12-24T02:11:33.559011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241378615154885:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:33.560219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241377818838428:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6627) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241377818838492:2122] held: no
2025-12-24T02:11:33.559370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241379111126408:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.559391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.559401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241379111126408:2048] [node 1] ICP22 created new session: [4:7587241379111126506:2048]
2025-12-24T02:11:33.559408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241379111126506:2048] [node 1] ICS09 handshake done sender: [4:7587241379111126482:2114] self: [4:7587241379111126497:0] peer: [1:7587241379209300673:0] socket: 273 qp: -1
2025-12-24T02:11:33.559416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241379111126506:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.559423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.575748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241377660003668:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.575895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241377660003662:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.575924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.575935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241377660003662:2110] [node 5] ICP22 created new session: [3:7587241377660003696:2110]
2025-12-24T02:11:33.575943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241377660003696:2110] [node 5] ICS09 handshake done sender: [3:7587241377660003668:2113] self: [3:7587241377660003667:0] peer: [5:7587241378615154979:0] socket: 293 qp: -1
2025-12-24T02:11:33.575950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241377660003696:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:33.575962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.579109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154983:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.579310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378615154967:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.579356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378615154886:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.579373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.579381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378615154886:2050] [node 3] ICP22 created new session: [5:7587241378615154984:2050]
2025-12-24T02:11:33.579389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378615154984:2050] [node 3] ICS09 handshake done sender: [5:7587241378615154967:2118] self: [5:7587241378615154979:0] peer: [3:7587241377660003667:0] socket: 296 qp: -1
2025-12-24T02:11:33.579395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378615154984:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.579403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.587475Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241379209300696:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.587607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241379209300671:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.587645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.587656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241379209300671:2110] [node 5] ICP22 created new session: [1:7587241379209300705:2110]
2025-12-24T02:11:33.587665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241379209300705:2110] [node 5] ICS09 handshake done sender: [1:7587241379209300696:2117] self: [1:7587241379209300690:0] peer: [5:7587241378615154980:0] socket: 306 qp: -1
2025-12-24T02:11:33.587674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241379209300705:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:33.587683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.593124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378615154884:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16610) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241378615154974:2121] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.595001Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241377660003662:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(18997262ns) {HandleHandshakeStatus:329(18995882ns) {DropIncomingHandshake:442(5840ns) DropOutgoingHandshake:457(304ns) SwitchToState:179(281ns) ProcessPendingSessionEvents:495(18878724ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18848147ns max# 18757589ns}}}
2025-12-24T02:11:33.594945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378615154986:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58137
2025-12-24T02:11:33.595125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378615154974:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.595222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378615154884:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.595233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.595239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378615154884:2048] [node 1] ICP22 created new session: [5:7587241378615154987:2048]
2025-12-24T02:11:33.595245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378615154987:2048] [node 1] ICS09 handshake done sender: [5:7587241378615154974:2121] self: [5:7587241378615154980:0] peer: [1:7587241379209300690:0] socket: 307 qp: -1
2025-12-24T02:11:33.595252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378615154987:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.595263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.598412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241377818838496:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46603
2025-12-24T02:11:33.598783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241377818838492:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.598901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241377818838428:2067] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.598916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.598923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241377818838428:2067] [node 5] ICP22 created new session: [6:7587241377818838497:2067]
2025-12-24T02:11:33.598930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241377818838497:2067] [node 5] ICS09 handshake done sender: [6:7587241377818838492:2122] self: [6:7587241377818838495:0] peer: [5:7587241378615154926:0] socket: 252 qp: -1
2025-12-24T02:11:33.598937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241377818838497:2067] [node 5] ICS10 traffic start
2025-12-24T02:11:33.598948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.614777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:33.614828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:33.614898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241377660003693:2048] [node 1] ICS01 socket: 268 reason# EndOfStream
2025-12-24T02:11:33.614905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241377660003593:2048] [node 1] ICP30 unregister session Session# [3:7587241377660003693:2048] VirtualId# [3:7587241377660003676:0]
2025-12-24T02:11:33.614887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241379221299255:2048] [node 1] ICS01 socket: 276 reason# EndOfStream
2025-12-24T02:11:33.614911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241377660003693:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.614959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.614898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241379221299162:2048] [node 1] ICP30 unregister session Session# [2:7587241379221299255:2048] VirtualId# [2:7587241379221299253:0]
2025-12-24T02:11:33.614989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.614909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241379221299255:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.614959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.614989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.620625Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:33.620704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241377818838484:2048] [node 1] ICS01 socket: 282 reason# EndOfStream
2025-12-24T02:11:33.620711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241377818838397:2048] [node 1] ICP30 unregister session Session# [6:7587241377818838484:2048] VirtualId# [6:7587241377818838480:0]
2025-12-24T02:11:33.620718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241377818838484:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.620764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.620789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.618761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:33.618854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241379111126506:2048] [node 1] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:33.618862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241379111126408:2048] [node 1] ICP30 unregister session Session# [4:7587241379111126506:2048] VirtualId# [4:7587241379111126497:0]
2025-12-24T02:11:33.618869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241379111126506:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.618909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.618931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.622730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:33.622812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378615154987:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-12-24T02:11:33.622819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378615154884:2048] [node 1] ICP30 unregister session Session# [5:7587241378615154987:2048] VirtualId# [5:7587241378615154980:0]
2025-12-24T02:11:33.622825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378615154987:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.622859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:33.622894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:33.640017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:33.640126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241377660003691:2049] [node 2] ICS01 socket: 270 reason# EndOfStream
2025-12-24T02:11:33.640137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241377660003594:2049] [node 2] ICP30 unregister session Session# [3:7587241377660003691:2049] VirtualId# [3:7587241377660003684:0]
2025-12-24T02:11:33.640146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241377660003691:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.640207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.640235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.644163Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:33.644249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241378615154975:2049] [node 2] ICS07 socket disconnect 278 reason# EndOfStream
2025-12-24T02:11:33.644255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378615154975:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.644307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.644332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241378615154975:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:33.644524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:33.644601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241377818838432:2049] [node 2] ICS01 socket: 241 reason# EndOfStream
2025-12-24T02:11:33.644608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241377818838398:2049] [node 2] ICP30 unregister session Session# [6:7587241377818838432:2049] VirtualId# [6:7587241377818838413:0]
2025-12-24T02:11:33.644614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241377818838432:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.644644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.644669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.645054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378615154885:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:33.645066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378615154975:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:33.645073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378615154885:2049] [node 2] ICP30 unregister session Session# [5:7587241378615154975:2049] VirtualId# [5:7587241378615154964:0]
2025-12-24T02:11:33.645120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.645126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241378615154885:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.645129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.642742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:33.642816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241379111126473:2049] [node 2] ICS07 socket disconnect 238 reason# EndOfStream
2025-12-24T02:11:33.642822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241379111126473:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.642856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:33.642875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241379111126473:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:33.643253Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241379111126409:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:33.643261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241379111126473:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:33.643266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241379111126409:2049] [node 2] ICP30 unregister session Session# [4:7587241379111126473:2049] VirtualId# [4:7587241379111126471:0]
2025-12-24T02:11:33.643291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:33.643299Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241379111126409:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.643301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2312) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:33.655266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241378615154885:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:33.662129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:33.662237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241377818838493:2050] [node 3] ICS01 socket: 242 reason# EndOfStream
2025-12-24T02:11:33.662246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241377818838399:2050] [node 3] ICP30 unregister session Session# [6:7587241377818838493:2050] VirtualId# [6:7587241377818838486:0]
2025-12-24T02:11:33.662254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241377818838493:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.662308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:33.662333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:33.662152Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:33.662237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378615154984:2050] [node 3] ICS01 socket: 296 reason# EndOfStream
2025-12-24T02:11:33.662247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378615154886:2050] [node 3] ICP30 unregister session Session# [5:7587241378615154984:2050] VirtualId# [5:7587241378615154979:0]
2025-12-24T02:11:33.662253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378615154984:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.662305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:33.662333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:33.700555Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:33.700683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241377818838488:2080] [node 4] ICS01 socket: 277 reason# EndOfStream
2025-12-24T02:11:33.700696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241377818838445:2080] [node 4] ICP30 unregister session Session# [6:7587241377818838488:2080] VirtualId# [6:7587241377818838483:0]
2025-12-24T02:11:33.700705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241377818838488:2080] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:33.700760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:33.700791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:33.774910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241378221942411:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.774926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241378221942412:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.774931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241378221942413:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.770861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241379256954662:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.770881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241379256954663:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.771469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378871055428:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.771478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378871055429:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.784285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241378426501071:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.774738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241378019173843:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:33.784304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241378426501072:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:33.774748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241378019173844:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:33.774753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241378019173845:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.771483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378871055430:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:33.770831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241376804335167:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:33.770848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241376804335168:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:33.851009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241378221942433:2065] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.851085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241378019173879:2078] [node 5] ICP01 ready to work
2025-12-24T02:11:33.852646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378871055486:2098] [node 4] ICP01 ready to work
2025-12-24T02:11:33.851139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241379256954705:2089] [node 6] ICP01 ready to work
2025-12-24T02:11:33.851576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241379256954708:2091] [node 4] ICP01 ready to work
2025-12-24T02:11:33.851892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954712:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.852636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954717:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.851255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241378019173880:2075] [node 6] ICP01 ready to work
2025-12-24T02:11:33.857301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241378426501132:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.868898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173915:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.867510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241378221942471:2087] [node 5] ICP01 ready to work
2025-12-24T02:11:33.867665Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942470:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.868222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942469:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.870943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241378871055503:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:33.871675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173915:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.871716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241378019173845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.871089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241376804335237:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:33.871146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241376804335236:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3118
2025-12-24T02:11:33.871850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954714:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.871862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241379256954714:2094] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.872792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241376804335167:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17057170ns) {Configure:104(17056193ns) {ProcessConfigured:135(16548646ns) {StartInitialHandshake:157(16548267ns) {DropHandshakes:470(644ns) {DropIncomingHandshake:442(38ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(654ns) {SwitchToState:179(86ns)}}}}}
2025-12-24T02:11:33.873433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335236:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.874162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241379256954662:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16558946ns) {Configure:104(16558361ns) {ProcessConfigured:135(16208137ns) {StartInitialHandshake:157(16207758ns) {DropHandshakes:470(486ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1250ns) {SwitchToState:179(1069ns)}}}}}
2025-12-24T02:11:33.873546Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241378426501132:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16139808ns) {Configure:104(16138899ns) {ProcessConfigured:135(15755573ns) {StartInitialHandshake:157(15754413ns) {DropHandshakes:470(1041ns) {DropIncomingHandshake:442(149ns) DropOutgoingHandshake:457(148ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(1635ns) {SwitchToState:179(831ns)}}}}}
2025-12-24T02:11:33.874301Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241378221942411:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23399738ns) {Configure:104(23393076ns) {ProcessConfigured:135(15925293ns) {StartInitialHandshake:157(15923687ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(181ns) PrepareNewSessionHandshake:477(980ns) {SwitchToState:179(624ns)}}}}}
2025-12-24T02:11:33.874382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241378221942469:2100] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.874794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241378221942433:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23719930ns) {Configure:104(23718862ns) {ProcessConfigured:135(16292731ns) {StartInitialHandshake:157(16292423ns) {DropHandshakes:470(311ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(646ns) {SwitchToState:179(69ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.877507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173918:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.882837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241376804335232:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.886983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501144:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.887094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501145:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.887132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501146:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.887158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501147:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.890812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241378426501072:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27776050ns) {Configure:104(27775215ns) {ProcessConfigured:135(27438803ns) {StartInitialHandshake:157(27437632ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(35ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1903ns) {SwitchToState:179(616ns)}}}}}
2025-12-24T02:11:33.894227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501144:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.894260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501145:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.895911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501147:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.895947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241378426501071:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.896103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335241:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.896116Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241376804335241:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.896770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241376804335244:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3118
2025-12-24T02:11:33.897171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241376804335234:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:33.897029Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241378221942412:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23954686ns) {Configure:104(23954017ns) {ProcessConfigured:135(16236624ns) {StartInitialHandshake:157(16235883ns) {DropHandshakes:470(488ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1204ns) {SwitchToState:179(336ns)}}}}}
2025-12-24T02:11:33.897922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241378426501138:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:33.898517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241379256954737:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:33.896867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055506:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.899773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173918:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.899152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241378426501136:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:33.900011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241379256954714:2094] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.899926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055506:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.899902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501146:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.900075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241378221942470:2101] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.900113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241378221942433:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.900235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954717:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.900355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335244:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.900495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954712:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.900515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241379256954662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.900611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241379256954708:2091] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.900625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.900631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241379256954708:2091] [node 4] ICP22 created new session: [3:7587241379256954745:2091]
2025-12-24T02:11:33.900636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241379256954745:2091] [node 4] ICS09 handshake done sender: [3:7587241379256954714:2094] self: [3:7587241379256954713:0] peer: [4:7587241378019173917:0] socket: 243 qp: -1
2025-12-24T02:11:33.900644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241379256954745:2091] [node 4] ICS10 traffic start
2025-12-24T02:11:33.900677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.907087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055509:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.907153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055510:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.907017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241376804335232:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24004918ns) {Configure:104(24004217ns) {ProcessConfigured:135(23589721ns) {StartInitialHandshake:157(23588945ns) {DropHandshakes:470(1579ns) {DropIncomingHandshake:442(517ns) DropOutgoingHandshake:457(218ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1325ns) {SwitchToState:179(637ns)}}}}}
2025-12-24T02:11:33.911718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335228:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.911725Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241376804335228:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.911880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241376804335241:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.911957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241376804335167:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.911973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.911981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241376804335167:2048] [node 2] ICP22 created new session: [1:7587241376804335249:2048]
2025-12-24T02:11:33.911990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241376804335249:2048] [node 2] ICS09 handshake done sender: [1:7587241376804335241:2113] self: [1:7587241376804335230:0] peer: [2:7587241378426501152:0] socket: 265 qp: -1
2025-12-24T02:11:33.911998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241376804335249:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:33.912007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.907294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954748:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.913937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173889:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.913953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241378019173889:2089] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.913994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173891:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.914009Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241378019173891:2090] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.913037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055509:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.913074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055510:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.913107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378871055428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.913121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378871055429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.914721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055508:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.914934Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241376804335234:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17621590ns) {Configure:104(17620647ns) {ProcessConfigured:135(17016191ns) {StartInitialHandshake:157(17015212ns) {DropHandshakes:470(701ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(234ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(1111ns) {SwitchToState:179(808ns)}}}}}
2025-12-24T02:11:33.915028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335239:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.915031Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241376804335239:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.915570Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241376804335239:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.915661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241376804335237:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.915679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.915685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241376804335237:2110] [node 5] ICP22 created new session: [1:7587241376804335255:2110]
2025-12-24T02:11:33.915690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241376804335255:2110] [node 5] ICS09 handshake done sender: [1:7587241376804335239:2112] self: [1:7587241376804335238:0] peer: [5:7587241378871055511:0] socket: 264 qp: -1
2025-12-24T02:11:33.915694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241376804335255:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:33.915710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.917454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055508:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.917544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378871055428:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241378871055509:2117] held: no
2025-12-24T02:11:33.914890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954748:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.914928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241379256954663:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.918960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501139:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.918975Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241378426501139:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.919410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501148:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.919413Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241378426501148:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.920011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241378426501139:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.920120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241378426501148:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.920178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241378426501132:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.920191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.920197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241378426501132:2107] [node 5] ICP22 created new session: [2:7587241378426501157:2107]
2025-12-24T02:11:33.920203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378426501157:2107] [node 5] ICS09 handshake done sender: [2:7587241378426501139:2110] self: [2:7587241378426501134:0] peer: [5:7587241378871055512:0] socket: 266 qp: -1
2025-12-24T02:11:33.920208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378426501157:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:33.920223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.920447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241378426501072:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.920457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.920460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241378426501072:2049] [node 3] ICP22 created new session: [2:7587241378426501159:2049]
2025-12-24T02:11:33.920464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378426501159:2049] [node 3] ICS09 handshake done sender: [2:7587241378426501148:2116] self: [2:7587241378426501137:0] peer: [3:7587241379256954749:0] socket: 278 qp: -1
2025-12-24T02:11:33.920467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378426501159:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:33.920471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.922836Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241378221942413:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(71358955ns) {Configure:104(71358328ns) {ProcessConfigured:135(64019784ns) {StartInitialHandshake:157(64019167ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(814ns)}}}}}
2025-12-24T02:11:33.922946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241378221942413:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.922847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241379256954737:2102] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24222963ns) {Configure:104(24222605ns) {ProcessConfigured:135(23827805ns) {StartInitialHandshake:157(23827440ns) {DropHandshakes:470(241ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(946ns) {SwitchToState:179(444ns)}}}}}
2025-12-24T02:11:33.923555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241379256954663:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241379256954748:2117] held: no
2025-12-24T02:11:33.923709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954710:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.923712Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241379256954710:2092] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.924099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954752:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.923396Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241378426501136:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24118886ns) {Configure:104(24118573ns) {ProcessConfigured:135(23868383ns) {StartInitialHandshake:157(23868177ns) {DropHandshakes:470(232ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(647ns) {SwitchToState:179(220ns)}}}}}
2025-12-24T02:11:33.925396Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378871055429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241378871055510:2118] held: no
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.926968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
2025-12-24T02:11:33.927046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
2025-12-24T02:11:33.927059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.927446Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241376804335258:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3118
2025-12-24T02:11:33.927496Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241376804335259:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3118
2025-12-24T02:11:33.928395Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241376804335167:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(16347077ns) {HandleHandshakeStatus:329(16345807ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(251ns) SwitchToState:179(87ns) ProcessPendingSessionEvents:495(16255911ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16235502ns max# 16151783ns}}}
2025-12-24T02:11:33.928819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241379256954710:2092] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.928150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501164:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.928724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241378426501147:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.929039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241378426501071:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.929051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.929057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241378426501071:2048] [node 1] ICP22 created new session: [2:7587241378426501165:2048]
2025-12-24T02:11:33.929062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378426501165:2048] [node 1] ICS09 handshake done sender: [2:7587241378426501147:2115] self: [2:7587241378426501152:0] peer: [1:7587241376804335230:0] socket: 277 qp: -1
2025-12-24T02:11:33.929067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378426501165:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.929074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.929837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378871055486:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.929867Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241378871055486:2098] [node 4] ICP35 event processing took too much time PendingConnection:279(29887810ns) {IncomingHandshake:261(29887397ns) {DropIncomingHandshake:442(62ns) PrepareNewSessionHandshake:477(506ns) {SwitchToState:179(76ns)} GenerateSessionVirtualId:988(265ns) IssueIncomingHandshakeReply:193(29883058ns) {SwitchToState:179(688ns)}}}
2025-12-24T02:11:33.929906Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173926:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.929948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241378019173843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.928966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954754:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.930285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241379256954705:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.930296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.930302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241379256954705:2089] [node 6] ICP22 created new session: [3:7587241379256954756:2089]
2025-12-24T02:11:33.930307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241379256954756:2089] [node 6] ICS09 handshake done sender: [3:7587241379256954710:2092] self: [3:7587241379256954709:0] peer: [6:7587241378221942492:0] socket: 241 qp: -1
2025-12-24T02:11:33.930313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241379256954756:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:33.930327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.930391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241379256954748:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.930972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335258:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.931002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335259:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.931054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241376804335228:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.933587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241376804335168:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.933605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.933612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241376804335168:2049] [node 3] ICP22 created new session: [1:7587241376804335261:2049]
2025-12-24T02:11:33.933619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241376804335261:2049] [node 3] ICS09 handshake done sender: [1:7587241376804335228:2107] self: [1:7587241376804335227:0] peer: [3:7587241379256954744:0] socket: 240 qp: -1
2025-12-24T02:11:33.933625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241376804335261:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:33.933631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.933895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241376804335167:2048] [node 2] ICP18 handshake (actor [1:7587241376804335258:2117]) is too old
2025-12-24T02:11:33.933912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241376804335168:2049] [node 3] ICP18 handshake (actor [1:7587241376804335259:2118]) is too old
2025-12-24T02:11:33.930840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173928:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.930908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173925:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
2025-12-24T02:11:33.934775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241378019173889:2089] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.934904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241378019173891:2090] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.934907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241378019173879:2078] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.934925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.934932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241378019173879:2078] [node 5] ICP22 created new session: [4:7587241378019173933:2078]
2025-12-24T02:11:33.934938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241378019173933:2078] [node 5] ICS09 handshake done sender: [4:7587241378019173889:2089] self: [4:7587241378019173888:0] peer: [5:7587241378871055507:0] socket: 252 qp: -1
2025-12-24T02:11:33.934941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241378019173915:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.934944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241378019173933:2078] [node 5] ICS10 traffic start
2025-12-24T02:11:33.934974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.935435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241376804335263:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3118
2025-12-24T02:11:33.935653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241378019173880:2075] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.935668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.935674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241378019173880:2075] [node 6] ICP22 created new session: [4:7587241378019173937:2075]
2025-12-24T02:11:33.935679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241378019173937:2075] [node 6] ICS09 handshake done sender: [4:7587241378019173891:2090] self: [4:7587241378019173890:0] peer: [6:7587241378221942490:0] socket: 253 qp: -1
2025-12-24T02:11:33.935683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241378019173937:2075] [node 6] ICS10 traffic start
2025-12-24T02:11:33.935697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.938339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335263:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.938827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241378019173845:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.938847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.938855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241378019173845:2050] [node 3] ICP22 created new session: [4:7587241378019173939:2050]
2025-12-24T02:11:33.938860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241378019173939:2050] [node 3] ICS09 handshake done sender: [4:7587241378019173915:2109] self: [4:7587241378019173917:0] peer: [3:7587241379256954713:0] socket: 256 qp: -1
2025-12-24T02:11:33.938866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241378019173939:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.938873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.939160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241379256954708:2091] [node 4] ICP35 event processing took too much time PendingConnection:279(38467312ns) {HandleHandshakeStatus:329(38466718ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(223ns) SwitchToState:179(68ns) ProcessPendingSessionEvents:495(26155134ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 26138241ns max# 26081729ns}}}
2025-12-24T02:11:33.940126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241378221942433:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241378221942470:2101] held: no
2025-12-24T02:11:33.942950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378871055486:2098] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13700) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241378871055506:2115] held: no
2025-12-24T02:11:33.942733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335247:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.942751Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241376804335247:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.943031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241378019173927:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.943058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241378019173844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.943147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501161:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.943158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241378426501161:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.946899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241376804335247:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.946993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241376804335232:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.947011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.947019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241376804335232:2108] [node 4] ICP22 created new session: [1:7587241376804335266:2108]
2025-12-24T02:11:33.947024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241376804335266:2108] [node 4] ICS09 handshake done sender: [1:7587241376804335247:2115] self: [1:7587241376804335245:0] peer: [4:7587241378019173929:0] socket: 288 qp: -1
2025-12-24T02:11:33.947029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241376804335266:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:33.947037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.947503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241379256954663:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.947517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.947523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241379256954663:2049] [node 2] ICP22 created new session: [3:7587241379256954758:2049]
2025-12-24T02:11:33.947529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241379256954758:2049] [node 2] ICS09 handshake done sender: [3:7587241379256954748:2117] self: [3:7587241379256954749:0] peer: [2:7587241378426501137:0] socket: 279 qp: -1
2025-12-24T02:11:33.947533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241379256954758:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.947551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.948507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241379256954662:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241379256954712:2093] held: no
2025-12-24T02:11:33.948685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241379256954712:2093] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.947333Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241378426501138:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49274342ns) {Configure:104(49273852ns) {ProcessConfigured:135(48787647ns) {StartInitialHandshake:157(48787068ns) {DropHandshakes:470(149ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(168ns) PrepareNewSessionHandshake:477(696ns) {SwitchToState:179(220ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.946823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055515:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.949949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378871055509:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.950021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378871055428:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.950031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.950037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378871055428:2048] [node 1] ICP22 created new session: [5:7587241378871055519:2048]
2025-12-24T02:11:33.950042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378871055519:2048] [node 1] ICS09 handshake done sender: [5:7587241378871055509:2117] self: [5:7587241378871055511:0] peer: [1:7587241376804335238:0] socket: 283 qp: -1
2025-12-24T02:11:33.950048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378871055519:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.950054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.950315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055517:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.950329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055518:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.950585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241379256954760:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36728
2025-12-24T02:11:33.950624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241379256954662:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.950633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.950637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241379256954662:2048] [node 1] ICP22 created new session: [3:7587241379256954761:2048]
2025-12-24T02:11:33.950641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241379256954761:2048] [node 1] ICS09 handshake done sender: [3:7587241379256954712:2093] self: [3:7587241379256954744:0] peer: [1:7587241376804335227:0] socket: 242 qp: -1
2025-12-24T02:11:33.950646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241379256954761:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.950651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.951040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942493:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.951047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942496:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.951092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942494:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.951111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942495:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.951306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241378221942496:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.951341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241378221942412:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.951544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241378221942470:2101] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.951455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241378426501167:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.951465Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241378426501167:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.954743Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378871055506:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:33.954838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378871055486:2098] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.954850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.954856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378871055486:2098] [node 4] ICP22 created new session: [5:7587241378871055521:2098]
2025-12-24T02:11:33.954862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378871055521:2098] [node 4] ICS09 handshake done sender: [5:7587241378871055506:2115] self: [5:7587241378871055507:0] peer: [4:7587241378019173888:0] socket: 260 qp: -1
2025-12-24T02:11:33.954867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378871055521:2098] [node 4] ICS10 traffic start
2025-12-24T02:11:33.954882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.955319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055516:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.954776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241378019173843:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11788) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241378019173926:2118] held: no
2025-12-24T02:11:33.954738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241378221942433:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.954759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.954766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241378221942433:2065] [node 4] ICP22 created new session: [6:7587241378221942498:2065]
2025-12-24T02:11:33.954772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241378221942498:2065] [node 4] ICS09 handshake done sender: [6:7587241378221942470:2101] self: [6:7587241378221942490:0] peer: [4:7587241378019173890:0] socket: 261 qp: -1
2025-12-24T02:11:33.954778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241378221942498:2065] [node 4] ICS10 traffic start
2025-12-24T02:11:33.954794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.959093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378871055510:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.959079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241378221942493:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.959166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241378221942469:2100] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.963956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378871055429:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.963979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.963985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378871055429:2049] [node 2] ICP22 created new session: [5:7587241378871055524:2049]
2025-12-24T02:11:33.963991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378871055524:2049] [node 2] ICS09 handshake done sender: [5:7587241378871055510:2118] self: [5:7587241378871055512:0] peer: [2:7587241378426501134:0] socket: 284 qp: -1
2025-12-24T02:11:33.963999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378871055524:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.964009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.964463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055517:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.964492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241378871055430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.966344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241378871055430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30863) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241378871055517:2121] held: no
2025-12-24T02:11:33.966572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241378871055527:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44110
2025-12-24T02:11:33.963234Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241379256954705:2089] [node 6] ICP35 event processing took too much time PendingConnection:279(32836485ns) {HandleHandshakeStatus:329(32835945ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(136ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(23138956ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23113300ns max# 23022691ns}}}
2025-12-24T02:11:33.966716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378871055517:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:33.964589Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241379256954750:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.964594Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241379256954750:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.965225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241379256954750:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:33.965293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241379256954737:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.965306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:33.965312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241379256954737:2102] [node 5] ICP22 created new session: [3:7587241379256954764:2102]
2025-12-24T02:11:33.965317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241379256954764:2102] [node 5] ICS09 handshake done sender: [3:7587241379256954750:2118] self: [3:7587241379256954743:0] peer: [5:7587241378871055526:0] socket: 299 qp: -1
2025-12-24T02:11:33.965322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241379256954764:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:33.965329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:33.967131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241378426501161:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.967254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241378426501136:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.967272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:33.967279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241378426501136:2108] [node 4] ICP22 created new session: [2:7587241378426501170:2108]
2025-12-24T02:11:33.967284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378426501170:2108] [node 4] ICS09 handshake done sender: [2:7587241378426501161:2117] self: [2:7587241378426501154:0] peer: [4:7587241378019173941:0] socket: 275 qp: -1
2025-12-24T02:11:33.967290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378426501170:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:33.967297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:33.967301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173942:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.974874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241378221942413:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.974904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.974914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241378221942413:2050] [node 3] ICP22 created new session: [6:7587241378221942501:2050]
2025-12-24T02:11:33.974924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241378221942501:2050] [node 3] ICS09 handshake done sender: [6:7587241378221942469:2100] self: [6:7587241378221942492:0] peer: [3:7587241379256954709:0] socket: 254 qp: -1
2025-12-24T02:11:33.974932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241378221942501:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.974941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.975362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942500:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.975536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241378221942411:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.979110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241378426501172:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37442
2025-12-24T02:11:33.980095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241378871055430:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.980117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:33.980124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378871055430:2050] [node 3] ICP22 created new session: [5:7587241378871055528:2050]
2025-12-24T02:11:33.980131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241378871055528:2050] [node 3] ICS09 handshake done sender: [5:7587241378871055517:2121] self: [5:7587241378871055526:0] peer: [3:7587241379256954743:0] socket: 297 qp: -1
2025-12-24T02:11:33.980138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241378871055528:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:33.980157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:33.980397Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241378426501167:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.980566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241378426501138:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.980576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.980582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241378426501138:2109] [node 6] ICP22 created new session: [2:7587241378426501174:2109]
2025-12-24T02:11:33.980586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241378426501174:2109] [node 6] ICS09 handshake done sender: [2:7587241378426501167:2119] self: [2:7587241378426501153:0] peer: [6:7587241378221942497:0] socket: 305 qp: -1
2025-12-24T02:11:33.980591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241378426501174:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:33.980599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.979290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241376804335251:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.979304Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241376804335251:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:33.980460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241376804335251:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:33.980722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241376804335234:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:33.980735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:33.980740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241376804335234:2109] [node 6] ICP22 created new session: [1:7587241376804335269:2109]
2025-12-24T02:11:33.980744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241376804335269:2109] [node 6] ICS09 handshake done sender: [1:7587241376804335251:2116] self: [1:7587241376804335246:0] peer: [6:7587241378221942503:0] socket: 257 qp: -1
2025-12-24T02:11:33.980748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241376804335269:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:33.980753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:33.972984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241378019173943:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33845
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.982959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241378019173845:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44032417ns) {HandleHandshakeStatus:329(44031492ns) {DropIncomingHandshake:442(204ns) DropOutgoingHandshake:457(4953ns) SwitchToState:179(92ns) ProcessPendingSessionEvents:495(43942670ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 43923323ns max# 43852381ns}}}
2025-12-24T02:11:33.982993Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241378019173844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17042) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241378019173927:2119] held: no
2025-12-24T02:11:33.983074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241378019173926:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:33.983479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942505:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.983966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241378221942496:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.984012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241378221942412:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.984024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.984029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241378221942412:2049] [node 2] ICP22 created new session: [6:7587241378221942507:2049]
2025-12-24T02:11:33.984035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241378221942507:2049] [node 2] ICS09 handshake done sender: [6:7587241378221942496:2120] self: [6:7587241378221942497:0] peer: [2:7587241378426501153:0] socket: 309 qp: -1
2025-12-24T02:11:33.984042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241378221942507:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.984048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.984321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241378221942506:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:33.984432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241378221942493:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:33.984473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241378221942411:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.984483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.984487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241378221942411:2048] [node 1] ICP22 created new session: [6:7587241378221942509:2048]
2025-12-24T02:11:33.984491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241378221942509:2048] [node 1] ICS09 handshake done sender: [6:7587241378221942493:2117] self: [6:7587241378221942503:0] peer: [1:7587241376804335246:0] socket: 289 qp: -1
2025-12-24T02:11:33.984496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241378221942509:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.984501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.990844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241378221942500:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.990885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241378221942471:2087] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:33.992247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241378019173927:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:33.992750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241378019173843:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.992765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:33.992771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241378019173843:2048] [node 1] ICP22 created new session: [4:7587241378019173944:2048]
2025-12-24T02:11:33.992776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241378019173944:2048] [node 1] ICS09 handshake done sender: [4:7587241378019173926:2118] self: [4:7587241378019173929:0] peer: [1:7587241376804335245:0] socket: 293 qp: -1
2025-12-24T02:11:33.992782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241378019173944:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:33.992788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:33.992939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241378019173844:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:33.992944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:33.992948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241378019173844:2049] [node 2] ICP22 created new session: [4:7587241378019173946:2049]
2025-12-24T02:11:33.992953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241378019173946:2049] [node 2] ICS09 handshake done sender: [4:7587241378019173927:2119] self: [4:7587241378019173941:0] peer: [2:7587241378426501154:0] socket: 294 qp: -1
2025-12-24T02:11:33.992957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241378019173946:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:33.992962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:33.991882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241376804335232:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(44778188ns) {HandleHandshakeStatus:329(44777262ns) {DropIncomingHandshake:442(3228ns) DropOutgoingHandshake:457(91ns) SwitchToState:179(361ns) ProcessPendingSessionEvents:495(32871434ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32847766ns max# 32764972ns}}}
2025-12-24T02:11:33.993682Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241379256954708:2091] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:33.998891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241378871055505:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:33.998917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241378871055505:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.003093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241378426501138:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22400814ns) {HandleHandshakeStatus:329(22400115ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(123ns) SwitchToState:179(41ns) ProcessPendingSessionEvents:495(22340390ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22310032ns max# 22225649ns}}}
2025-12-24T02:11:34.003590Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241376804335234:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(22767421ns) {HandleHandshakeStatus:329(22767227ns) {DropIncomingHandshake:442(4440ns) DropOutgoingHandshake:457(50ns) SwitchToState:179(146ns) ProcessPendingSessionEvents:495(22715568ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22696417ns max# 22621735ns}}}
2025-12-24T02:11:34.016493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241378871055505:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.016633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241378871055503:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.016660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.016671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241378871055503:2109] [node 6] ICP22 created new session: [5:7587241383166022827:2109]
2025-12-24T02:11:34.016679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241383166022827:2109] [node 6] ICS09 handshake done sender: [5:7587241378871055505:2114] self: [5:7587241378871055504:0] peer: [6:7587241378221942511:0] socket: 246 qp: -1
2025-12-24T02:11:34.016695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241383166022827:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:34.016706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.019771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241378221942471:2087] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20140) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241378221942500:2121] held: no
2025-12-24T02:11:34.019938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382516909808:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12856
2025-12-24T02:11:34.020253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241378221942500:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.020373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241378221942471:2087] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.020388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.020394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241378221942471:2087] [node 5] ICP22 created new session: [6:7587241382516909809:2087]
2025-12-24T02:11:34.020401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382516909809:2087] [node 5] ICS09 handshake done sender: [6:7587241378221942500:2121] self: [6:7587241378221942511:0] peer: [5:7587241378871055504:0] socket: 282 qp: -1
2025-12-24T02:11:34.020407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382516909809:2087] [node 5] ICS10 traffic start
2025-12-24T02:11:34.020419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.020886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241378426501138:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:34.035147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:34.035247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241378426501165:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-12-24T02:11:34.035257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241378426501071:2048] [node 1] ICP30 unregister session Session# [2:7587241378426501165:2048] VirtualId# [2:7587241378426501152:0]
2025-12-24T02:11:34.035265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241378426501165:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.035308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.035332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.035896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:34.035969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241379256954761:2048] [node 1] ICS01 socket: 242 reason# EndOfStream
2025-12-24T02:11:34.035975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241379256954662:2048] [node 1] ICP30 unregister session Session# [3:7587241379256954761:2048] VirtualId# [3:7587241379256954744:0]
2025-12-24T02:11:34.035980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241379256954761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.036013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.036032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.042129Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:34.042206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378871055519:2048] [node 1] ICS01 socket: 283 reason# EndOfStream
2025-12-24T02:11:34.042213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378871055428:2048] [node 1] ICP30 unregister session Session# [5:7587241378871055519:2048] VirtualId# [5:7587241378871055511:0]
2025-12-24T02:11:34.042221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378871055519:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.042260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.042282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.042728Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:34.042818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241378019173944:2048] [node 1] ICS01 socket: 293 reason# EndOfStream
2025-12-24T02:11:34.042825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241378019173843:2048] [node 1] ICP30 unregister session Session# [4:7587241378019173944:2048] VirtualId# [4:7587241378019173929:0]
2025-12-24T02:11:34.042832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241378019173944:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.042880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.042909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.042871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:34.042946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241378221942509:2048] [node 1] ICS01 socket: 289 reason# EndOfStream
2025-12-24T02:11:34.042953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241378221942411:2048] [node 1] ICP30 unregister session Session# [6:7587241378221942509:2048] VirtualId# [6:7587241378221942503:0]
2025-12-24T02:11:34.042959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241378221942509:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.043002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.043030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.045264Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:34.045337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241379256954758:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-12-24T02:11:34.045343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241379256954663:2049] [node 2] ICP30 unregister session Session# [3:7587241379256954758:2049] VirtualId# [3:7587241379256954749:0]
2025-12-24T02:11:34.045348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241379256954758:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.045381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.045401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.053770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:34.053841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241378019173946:2049] [node 2] ICS01 socket: 294 reason# EndOfStream
2025-12-24T02:11:34.053847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241378019173844:2049] [node 2] ICP30 unregister session Session# [4:7587241378019173946:2049] VirtualId# [4:7587241378019173941:0]
2025-12-24T02:11:34.053855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241378019173946:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.053898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.053921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.053956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:34.054019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241378221942507:2049] [node 2] ICS01 socket: 309 reason# EndOfStream
2025-12-24T02:11:34.054024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241378221942412:2049] [node 2] ICP30 unregister session Session# [6:7587241378221942507:2049] VirtualId# [6:7587241378221942497:0]
2025-12-24T02:11:34.054028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241378221942507:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.054059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.054080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.054752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:34.054862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378871055524:2049] [node 2] ICS01 socket: 284 reason# EndOfStream
2025-12-24T02:11:34.054868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378871055429:2049] [node 2] ICP30 unregister session Session# [5:7587241378871055524:2049] VirtualId# [5:7587241378871055512:0]
2025-12-24T02:11:34.054874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378871055524:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.054914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.054934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.087608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:34.087723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241378871055528:2050] [node 3] ICS01 socket: 297 reason# EndOfStream
2025-12-24T02:11:34.087736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241378871055430:2050] [node 3] ICP30 unregister session Session# [5:7587241378871055528:2050] VirtualId# [5:7587241378871055526:0]
2025-12-24T02:11:34.087744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241378871055528:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.087794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:34.087820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:34.090846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:34.090935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241378221942501:2050] [node 3] ICS01 socket: 254 reason# EndOfStream
2025-12-24T02:11:34.090942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241378221942413:2050] [node 3] ICP30 unregister session Session# [6:7587241378221942501:2050] VirtualId# [6:7587241378221942492:0]
2025-12-24T02:11:34.090949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241378221942501:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.090995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:34.091021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:34.126587Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:34.126708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241378221942498:2065] [node 4] ICS01 socket: 261 reason# EndOfStream
2025-12-24T02:11:34.126720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241378221942433:2065] [node 4] ICP30 unregister session Session# [6:7587241378221942498:2065] VirtualId# [6:7587241378221942490:0]
2025-12-24T02:11:34.126729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241378221942498:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.126790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:34.126816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:34.164418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:34.164550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241382516909809:2087] [node 5] ICS01 socket: 282 reason# EndOfStream
2025-12-24T02:11:34.164564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241378221942471:2087] [node 5] ICP30 unregister session Session# [6:7587241382516909809:2087] VirtualId# [6:7587241378221942511:0]
2025-12-24T02:11:34.164573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241382516909809:2087] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.164623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:34.164651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:34.230952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381207143423:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:34.230969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381207143424:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:34.238832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241381117786789:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.238841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241381117786790:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.238774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241383135605984:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.238787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241383135605985:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:34.246859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383376237523:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.246868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383376237524:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.246873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383376237525:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:34.258815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382321940461:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.258832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382321940462:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.258839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382321940463:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:34.293836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241382849457536:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.293883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241382849457537:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.293888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241382849457538:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.329793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786849:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.328564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381207143484:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:34.330572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940522:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.333809Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786850:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.330819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241383135606048:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.334086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381207143495:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:34.334190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381207143496:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:34.335183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381207143493:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43012
2025-12-24T02:11:34.335242Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381207143494:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43012
2025-12-24T02:11:34.334932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237584:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.340679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940522:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.341103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382321940461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.338939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241382849457563:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:34.339048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241382849457567:2069] [node 5] ICP01 ready to work
2025-12-24T02:11:34.342857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237584:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.343287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383376237524:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.343108Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143493:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.343146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143486:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.343149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381207143486:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.344328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381207143486:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.347301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786850:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.347767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241381117786790:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.346800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606050:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.346811Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241383135606050:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.348067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241383135606050:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.350983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237586:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.351013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237587:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.351186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940524:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.351388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381207143484:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.351404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.351411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381207143484:2107] [node 5] ICP22 created new session: [1:7587241381207143506:2107]
2025-12-24T02:11:34.351419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381207143506:2107] [node 5] ICS09 handshake done sender: [1:7587241381207143486:2108] self: [1:7587241381207143485:0] peer: [5:7587241382321940523:0] socket: 246 qp: -1
2025-12-24T02:11:34.351426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381207143506:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:34.351435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.354820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786849:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.355203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241381117786789:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.359020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241381117786854:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:34.364632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143488:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.364648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381207143488:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.362748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606052:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.362766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241383135606052:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.365373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383376237584:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.365403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382321940522:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.374805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381207143488:2109] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.374956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381207143424:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.374977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.374987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381207143424:2049] [node 3] ICP22 created new session: [1:7587241381207143510:2049]
2025-12-24T02:11:34.374994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381207143510:2049] [node 3] ICS09 handshake done sender: [1:7587241381207143488:2109] self: [1:7587241381207143487:0] peer: [3:7587241381117786852:0] socket: 247 qp: -1
2025-12-24T02:11:34.375002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381207143510:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:34.375012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.378836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457609:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.382864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143494:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.386818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382321940461:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.386845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.386857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382321940461:2048] [node 1] ICP22 created new session: [5:7587241382321940525:2048]
2025-12-24T02:11:34.386866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382321940525:2048] [node 1] ICS09 handshake done sender: [5:7587241382321940522:2108] self: [5:7587241382321940523:0] peer: [1:7587241381207143485:0] socket: 250 qp: -1
2025-12-24T02:11:34.386878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382321940525:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.386903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.388077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383376237524:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.388097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.388102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383376237524:2049] [node 2] ICP22 created new session: [4:7587241383376237589:2049]
2025-12-24T02:11:34.388110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383376237589:2049] [node 2] ICS09 handshake done sender: [4:7587241383376237584:2108] self: [4:7587241383376237585:0] peer: [2:7587241383135606049:0] socket: 259 qp: -1
2025-12-24T02:11:34.388116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383376237589:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.388131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.390961Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241381117786854:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31883866ns) {Configure:104(31882777ns) {ProcessConfigured:135(31577205ns) {StartInitialHandshake:157(31576780ns) {DropHandshakes:470(740ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(339ns) PrepareNewSessionHandshake:477(1448ns) {SwitchToState:179(731ns)}}}}}
2025-12-24T02:11:34.391251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241381117786861:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:34.391333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241381117786862:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:34.391383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786860:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.392079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786859:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.392371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241383135606048:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.392390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.392395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241383135606048:2108] [node 4] ICP22 created new session: [2:7587241383135606058:2108]
2025-12-24T02:11:34.392401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241383135606058:2108] [node 4] ICS09 handshake done sender: [2:7587241383135606050:2109] self: [2:7587241383135606049:0] peer: [4:7587241383376237585:0] socket: 254 qp: -1
2025-12-24T02:11:34.392407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241383135606058:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:34.392415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.392833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241383135606060:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:34.392916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241383135606061:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:34.393707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241383135606062:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45344
2025-12-24T02:11:34.393718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241383135606063:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45344
2025-12-24T02:11:34.394179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241383135606052:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.407065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241381117786849:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.407301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241381117786789:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.407315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.407323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241381117786789:2048] [node 1] ICP22 created new session: [3:7587241381117786871:2048]
2025-12-24T02:11:34.407331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241381117786871:2048] [node 1] ICS09 handshake done sender: [3:7587241381117786849:2107] self: [3:7587241381117786852:0] peer: [1:7587241381207143487:0] socket: 248 qp: -1
2025-12-24T02:11:34.407337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241381117786871:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.407372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.407789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786868:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.408977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241381117786850:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.412214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241381117786790:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.412226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.412233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241381117786790:2049] [node 2] ICP22 created new session: [3:7587241381117786873:2049]
2025-12-24T02:11:34.412239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241381117786873:2049] [node 2] ICS09 handshake done sender: [3:7587241381117786850:2108] self: [3:7587241381117786851:0] peer: [2:7587241383135606051:0] socket: 256 qp: -1
2025-12-24T02:11:34.412245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241381117786873:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.412253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.410768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241383135605985:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.410791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.410801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241383135605985:2049] [node 3] ICP22 created new session: [2:7587241383135606070:2049]
2025-12-24T02:11:34.410808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241383135606070:2049] [node 3] ICS09 handshake done sender: [2:7587241383135606052:2110] self: [2:7587241383135606051:0] peer: [3:7587241381117786851:0] socket: 255 qp: -1
2025-12-24T02:11:34.410814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241383135606070:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:34.410823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.411441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606063:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.412627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241383135605984:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.411368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382321940532:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:34.411616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382321940533:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:34.413635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940534:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.414028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241383376237524:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(25872802ns) {HandleHandshakeStatus:329(25872314ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(73ns) SwitchToState:179(74ns) ProcessPendingSessionEvents:495(19221062ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 19206681ns max# 19154259ns}}}
2025-12-24T02:11:34.414857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383376237597:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:34.414990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383376237598:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:34.415034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237586:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.415137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237594:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.415185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237595:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.415213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237596:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.416006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383376237523:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.422723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143502:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.422745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381207143502:2116] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.427465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786875:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.430967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241382849457609:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.442479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237595:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.440296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241383135606074:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45344
2025-12-24T02:11:34.441780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940534:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.441988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940543:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.442020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940544:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.442038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382321940463:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.445497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383376237525:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.446755Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786860:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.446801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786864:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.446804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241381117786864:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.451121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606074:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.451156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606068:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.451159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241383135606068:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.451496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606062:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.451772Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241383135606068:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.451841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241383135606061:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.450784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940543:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.450911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382321940462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.454872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237596:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.454905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237594:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.451855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.451862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241383135606061:2112] [node 5] ICP22 created new session: [2:7587241383135606078:2112]
2025-12-24T02:11:34.451869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241383135606078:2112] [node 5] ICS09 handshake done sender: [2:7587241383135606068:2116] self: [2:7587241383135606067:0] peer: [5:7587241382321940546:0] socket: 287 qp: -1
2025-12-24T02:11:34.451876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241383135606078:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:34.451887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.450971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241382849457536:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.451073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457616:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.451109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457617:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.451140Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457615:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.454794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241382849457616:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.454830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241382849457537:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.454881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786866:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.454892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241381117786866:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.455763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237610:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.458775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241382849457615:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.458810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241382849457538:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.458880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241382849457617:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.458896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241382849457563:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.462829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241383135606066:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.462843Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241383135606066:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.462774Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241381117786864:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.470796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143498:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.470822Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381207143498:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.474747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237600:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.474772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241383376237600:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.478407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241381117786861:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.478435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.478446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241381117786861:2110] [node 5] ICP22 created new session: [3:7587241381117786879:2110]
2025-12-24T02:11:34.478454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241381117786879:2110] [node 5] ICS09 handshake done sender: [3:7587241381117786864:2115] self: [3:7587241381117786863:0] peer: [5:7587241382321940545:0] socket: 283 qp: -1
2025-12-24T02:11:34.478460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241381117786879:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:34.478472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.478956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383376237600:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.479588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381207143500:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.479602Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381207143500:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.481841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381207143502:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.481996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381207143498:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.482185Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381207143500:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.482258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381207143423:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.482276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.482282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381207143423:2048] [node 2] ICP22 created new session: [1:7587241381207143517:2048]
2025-12-24T02:11:34.482289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381207143517:2048] [node 2] ICS09 handshake done sender: [1:7587241381207143502:2116] self: [1:7587241381207143501:0] peer: [2:7587241383135606073:0] socket: 272 qp: -1
2025-12-24T02:11:34.482294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381207143517:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:34.482301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.482520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381207143495:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.482526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.482529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381207143495:2110] [node 4] ICP22 created new session: [1:7587241381207143519:2110]
2025-12-24T02:11:34.482533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381207143519:2110] [node 4] ICS09 handshake done sender: [1:7587241381207143498:2114] self: [1:7587241381207143497:0] peer: [4:7587241383376237604:0] socket: 261 qp: -1
2025-12-24T02:11:34.482537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381207143519:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:34.482542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.480769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940544:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.480887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940547:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.481074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382321940532:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.481194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382321940462:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241382321940543:2117] held: no
2025-12-24T02:11:34.481293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382321940543:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.481357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382321940462:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.481368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.481374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382321940462:2049] [node 2] ICP22 created new session: [5:7587241382321940551:2049]
2025-12-24T02:11:34.481380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382321940551:2049] [node 2] ICS09 handshake done sender: [5:7587241382321940543:2117] self: [5:7587241382321940546:0] peer: [2:7587241383135606067:0] socket: 296 qp: -1
2025-12-24T02:11:34.481385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382321940551:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.481391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.481542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940548:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.482760Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241383135606066:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.482811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382321940534:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.486908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381207143496:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.486924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.486928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381207143496:2111] [node 6] ICP22 created new session: [1:7587241381207143521:2111]
2025-12-24T02:11:34.486933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381207143521:2111] [node 6] ICS09 handshake done sender: [1:7587241381207143500:2115] self: [1:7587241381207143499:0] peer: [6:7587241382849457618:0] socket: 268 qp: -1
2025-12-24T02:11:34.486937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381207143521:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:34.486942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.487343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457622:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.486901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382321940463:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.486920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.486927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382321940463:2050] [node 3] ICP22 created new session: [5:7587241382321940553:2050]
2025-12-24T02:11:34.486934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382321940553:2050] [node 3] ICS09 handshake done sender: [5:7587241382321940534:2113] self: [5:7587241382321940545:0] peer: [3:7587241381117786863:0] socket: 286 qp: -1
2025-12-24T02:11:34.486940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382321940553:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:34.486947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.490940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241381117786866:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.498839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786875:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.498881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241381117786858:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.498885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241381117786858:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.499632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241381117786862:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.499656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.499664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241381117786862:2111] [node 4] ICP22 created new session: [3:7587241381117786881:2111]
2025-12-24T02:11:34.499670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241381117786881:2111] [node 4] ICS09 handshake done sender: [3:7587241381117786866:2116] self: [3:7587241381117786865:0] peer: [4:7587241383376237607:0] socket: 284 qp: -1
2025-12-24T02:11:34.499677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241381117786881:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:34.499685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.499954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241381117786862:2111] [node 4] ICP18 handshake (actor [3:7587241381117786875:2118]) is too old
2025-12-24T02:11:34.500199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241381117786858:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.500457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241381117786854:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.500468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.500472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241381117786854:2109] [node 6] ICP22 created new session: [3:7587241381117786884:2109]
2025-12-24T02:11:34.500476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241381117786884:2109] [node 6] ICS09 handshake done sender: [3:7587241381117786858:2112] self: [3:7587241381117786855:0] peer: [6:7587241382849457620:0] socket: 276 qp: -1
2025-12-24T02:11:34.500482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241381117786884:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:34.500488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.502770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241383135606060:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.502798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.502860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457623:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.502806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241383135606060:2111] [node 6] ICP22 created new session: [2:7587241383135606082:2111]
2025-12-24T02:11:34.502814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241383135606082:2111] [node 6] ICS09 handshake done sender: [2:7587241383135606066:2115] self: [2:7587241383135606065:0] peer: [6:7587241382849457619:0] socket: 280 qp: -1
2025-12-24T02:11:34.502820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241383135606082:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:34.502828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.506952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241382849457616:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.507047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383376237602:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.507061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241383376237602:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.511218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241382849457537:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.511245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.511252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241382849457537:2049] [node 2] ICP22 created new session: [6:7587241382849457626:2049]
2025-12-24T02:11:34.511260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382849457626:2049] [node 2] ICS09 handshake done sender: [6:7587241382849457616:2117] self: [6:7587241382849457619:0] peer: [2:7587241383135606065:0] socket: 302 qp: -1
2025-12-24T02:11:34.511266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382849457626:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.511275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.513286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457624:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.510946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241383135605984:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241383135606063:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.515136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457625:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.514965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241383135606081:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45344
2025-12-24T02:11:34.515921Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241382849457536:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241382849457609:2113] held: no
No RDMA context found for GID: 2025-12-24T02:11:34.516052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241382849457617:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.516139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457629:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.516266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241382849457609:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.516339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241382849457563:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.516352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.516357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241382849457563:2063] [node 4] ICP22 created new session: [6:7587241382849457630:2063]
2025-12-24T02:11:34.516365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382849457630:2063] [node 4] ICS09 handshake done sender: [6:7587241382849457617:2118] self: [6:7587241382849457621:0] peer: [4:7587241383376237599:0] socket: 303 qp: -1
2025-12-24T02:11:34.516370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382849457630:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:34.516378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.516603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241382849457536:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.516610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.516614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241382849457536:2048] [node 1] ICP22 created new session: [6:7587241382849457632:2048]
2025-12-24T02:11:34.516618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382849457632:2048] [node 1] ICS09 handshake done sender: [6:7587241382849457609:2113] self: [6:7587241382849457618:0] peer: [1:7587241381207143499:0] socket: 269 qp: -1
2025-12-24T02:11:34.516623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382849457632:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.516629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
::ffff:127.1.0.0
2025-12-24T02:11:34.518931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940555:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.518788Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241383376237525:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29856) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241383376237595:2115] held: no
2025-12-24T02:11:34.519009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383376237595:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.523293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241383135606063:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.529927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382321940556:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4421
2025-12-24T02:11:34.526948Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241382849457615:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:34.530802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241383376237597:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.530831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.530840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383376237597:2111] [node 6] ICP22 created new session: [4:7587241383376237614:2111]
2025-12-24T02:11:34.530848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383376237614:2111] [node 6] ICS09 handshake done sender: [4:7587241383376237600:2117] self: [4:7587241383376237599:0] peer: [6:7587241382849457621:0] socket: 294 qp: -1
2025-12-24T02:11:34.530856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383376237614:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:34.530864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.534777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241383135605984:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.534795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.534802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241383135605984:2048] [node 1] ICP22 created new session: [2:7587241383135606084:2048]
2025-12-24T02:11:34.534808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241383135606084:2048] [node 1] ICS09 handshake done sender: [2:7587241383135606063:2114] self: [2:7587241383135606073:0] peer: [1:7587241381207143501:0] socket: 278 qp: -1
2025-12-24T02:11:34.534815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241383135606084:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.534823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.535177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241382849457538:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.535193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.535198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241382849457538:2050] [node 3] ICP22 created new session: [6:7587241382849457635:2050]
2025-12-24T02:11:34.535204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382849457635:2050] [node 3] ICS09 handshake done sender: [6:7587241382849457615:2116] self: [6:7587241382849457620:0] peer: [3:7587241381117786855:0] socket: 301 qp: -1
2025-12-24T02:11:34.535208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382849457635:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:34.535214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.535890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241382849457625:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.538855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940555:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.541770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241382849457567:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.542582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383376237525:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.542600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.542607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383376237525:2050] [node 3] ICP22 created new session: [4:7587241383376237616:2050]
2025-12-24T02:11:34.542613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383376237616:2050] [node 3] ICS09 handshake done sender: [4:7587241383376237595:2115] self: [4:7587241383376237607:0] peer: [3:7587241381117786865:0] socket: 291 qp: -1
2025-12-24T02:11:34.542619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383376237616:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:34.542628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.542916Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382321940540:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.542925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241382321940540:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.543575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382321940540:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.543658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241382321940533:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.543672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.543678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382321940533:2112] [node 6] ICP22 created new session: [5:7587241382321940559:2112]
2025-12-24T02:11:34.543682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382321940559:2112] [node 6] ICS09 handshake done sender: [5:7587241382321940540:2116] self: [5:7587241382321940539:0] peer: [6:7587241382849457637:0] socket: 314 qp: -1
2025-12-24T02:11:34.543688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382321940559:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:34.543695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.547157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241381117786886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41076
2025-12-24T02:11:34.550860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383376237602:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.551546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241382849457567:2069] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17681) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241382849457625:2122] held: no
2025-12-24T02:11:34.551768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241382849457638:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9793
2025-12-24T02:11:34.551925Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241382849457625:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.551975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241382849457567:2069] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.551986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.551991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241382849457567:2069] [node 5] ICP22 created new session: [6:7587241382849457639:2069]
2025-12-24T02:11:34.551996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241382849457639:2069] [node 5] ICS09 handshake done sender: [6:7587241382849457625:2122] self: [6:7587241382849457637:0] peer: [5:7587241382321940539:0] socket: 317 qp: -1
2025-12-24T02:11:34.552001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241382849457639:2069] [node 5] ICS10 traffic start
2025-12-24T02:11:34.552007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.558860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241383376237598:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.558883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.558890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383376237598:2112] [node 5] ICP22 created new session: [4:7587241383376237618:2112]
2025-12-24T02:11:34.558896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383376237618:2112] [node 5] ICS09 handshake done sender: [4:7587241383376237602:2118] self: [4:7587241383376237601:0] peer: [5:7587241382321940550:0] socket: 295 qp: -1
2025-12-24T02:11:34.558900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383376237618:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:34.559090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382321940544:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.558906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.559711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383376237612:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53005
2025-12-24T02:11:34.559869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383376237586:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.559915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383376237523:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.559923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.559927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383376237523:2048] [node 1] ICP22 created new session: [4:7587241383376237620:2048]
2025-12-24T02:11:34.559931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383376237620:2048] [node 1] ICS09 handshake done sender: [4:7587241383376237586:2109] self: [4:7587241383376237604:0] peer: [1:7587241381207143497:0] socket: 263 qp: -1
2025-12-24T02:11:34.559937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383376237620:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.559943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.578833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382321940532:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.578863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.578875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382321940532:2111] [node 4] ICP22 created new session: [5:7587241382321940561:2111]
2025-12-24T02:11:34.578883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382321940561:2111] [node 4] ICS09 handshake done sender: [5:7587241382321940544:2118] self: [5:7587241382321940550:0] peer: [4:7587241383376237601:0] socket: 297 qp: -1
2025-12-24T02:11:34.578894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382321940561:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:34.578908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.579058Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241382321940532:2111] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:34.619108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:34.619219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241383135606084:2048] [node 1] ICS01 socket: 278 reason# EndOfStream
2025-12-24T02:11:34.619230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241383135605984:2048] [node 1] ICP30 unregister session Session# [2:7587241383135606084:2048] VirtualId# [2:7587241383135606073:0]
2025-12-24T02:11:34.619239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241383135606084:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.619279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.619301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.622740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:34.622827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241381117786871:2048] [node 1] ICS07 socket disconnect 248 reason# EndOfStream
2025-12-24T02:11:34.622838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241381117786871:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.622886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.622912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241381117786871:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:34.626754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:34.626859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241382849457632:2048] [node 1] ICS07 socket disconnect 269 reason# EndOfStream
2025-12-24T02:11:34.626867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241382849457632:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.626741Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:34.626759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:34.626856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241382321940525:2048] [node 1] ICS07 socket disconnect 250 reason# EndOfStream
2025-12-24T02:11:34.626863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241382321940525:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.626918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.626941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241382321940525:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:34.629948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241381117786789:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.629962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241381117786871:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.629968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241381117786789:2048] [node 1] ICP30 unregister session Session# [3:7587241381117786871:2048] VirtualId# [3:7587241381117786852:0]
2025-12-24T02:11:34.630003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.630009Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241381117786789:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.630011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.626933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.626959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241382849457632:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:34.629870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241382849457536:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.629880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241382849457632:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.629886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241382849457536:2048] [node 1] ICP30 unregister session Session# [6:7587241382849457632:2048] VirtualId# [6:7587241382849457618:0]
2025-12-24T02:11:34.629908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.629915Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241382849457536:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.629918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.630142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382321940461:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.630155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241382321940525:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.630158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382321940461:2048] [node 1] ICP30 unregister session Session# [5:7587241382321940525:2048] VirtualId# [5:7587241382321940523:0]
2025-12-24T02:11:34.630190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.630194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241382321940461:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.630196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.630764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241383376237620:2048] [node 1] ICS07 socket disconnect 263 reason# EndOfStream
2025-12-24T02:11:34.630777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241383376237620:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.630815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:34.630836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241383376237620:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:34.631201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:34.631293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241381117786873:2049] [node 2] ICS07 socket disconnect 256 reason# EndOfStream
2025-12-24T02:11:34.631298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241381117786873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.631328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.631344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241381117786873:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:34.634878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241383376237523:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.634893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241383376237620:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.634898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241383376237523:2048] [node 1] ICP30 unregister session Session# [4:7587241383376237620:2048] VirtualId# [4:7587241383376237604:0]
2025-12-24T02:11:34.634911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:34.634918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241383376237523:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.634920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.638783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241382321940461:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:34.638810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241381117786789:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:34.643160Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241381117786790:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:34.643173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241381117786873:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:34.643419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241381117786790:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.643429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241381117786873:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.643435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241381117786790:2049] [node 2] ICP30 unregister session Session# [3:7587241381117786873:2049] VirtualId# [3:7587241381117786851:0]
2025-12-24T02:11:34.643488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.643495Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241381117786790:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.643497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.642731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241382849457536:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:34.649597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:34.649687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241382849457626:2049] [node 2] ICS07 socket disconnect 302 reason# EndOfStream
2025-12-24T02:11:34.649695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241382849457626:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.649741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.649762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241382849457626:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:34.649487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241383376237523:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:34.650125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:34.650177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241383376237589:2049] [node 2] ICS07 socket disconnect 259 reason# EndOfStream
2025-12-24T02:11:34.650182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241383376237589:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.650206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.650223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241383376237589:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:34.650587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241383376237524:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.650594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241383376237589:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.650599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241383376237524:2049] [node 2] ICP30 unregister session Session# [4:7587241383376237589:2049] VirtualId# [4:7587241383376237585:0]
2025-12-24T02:11:34.650645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.650650Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241383376237524:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.650653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.649945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:34.650017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241382321940551:2049] [node 2] ICS01 socket: 296 reason# EndOfStream
2025-12-24T02:11:34.650023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382321940462:2049] [node 2] ICP30 unregister session Session# [5:7587241382321940551:2049] VirtualId# [5:7587241382321940546:0]
2025-12-24T02:11:34.650027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241382321940551:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.650053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:34.650075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.650782Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241382849457537:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:34.650792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241382849457626:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:34.650796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241382849457537:2049] [node 2] ICP30 unregister session Session# [6:7587241382849457626:2049] VirtualId# [6:7587241382849457619:0]
2025-12-24T02:11:34.650807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:34.650811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241382849457537:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.650813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8369) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:34.662782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241382849457537:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:34.663265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241383376237524:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:34.674787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:34.674914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:34.675004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241382849457635:2050] [node 3] ICS01 socket: 301 reason# EndOfStream
2025-12-24T02:11:34.675011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241382849457538:2050] [node 3] ICP30 unregister session Session# [6:7587241382849457635:2050] VirtualId# [6:7587241382849457620:0]
2025-12-24T02:11:34.675028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241382849457635:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.675082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:34.675114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:34.674900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241382321940553:2050] [node 3] ICS01 socket: 286 reason# EndOfStream
2025-12-24T02:11:34.674912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382321940463:2050] [node 3] ICP30 unregister session Session# [5:7587241382321940553:2050] VirtualId# [5:7587241382321940545:0]
2025-12-24T02:11:34.674921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241382321940553:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.674979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:34.675004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:34.680040Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:34.680159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241382849457630:2063] [node 4] ICS01 socket: 303 reason# EndOfStream
2025-12-24T02:11:34.680170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241382849457563:2063] [node 4] ICP30 unregister session Session# [6:7587241382849457630:2063] VirtualId# [6:7587241382849457621:0]
2025-12-24T02:11:34.680180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241382849457630:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:34.680233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:34.680291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:34.713515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241382904099228:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.713533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241382904099229:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.713787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241380912632746:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.713803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241380912632747:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:34.730711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381722663420:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:34.730756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381722663421:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:34.716122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241380406712678:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.716134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241380406712679:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.716139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241380406712680:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:34.739302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383876908662:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.739354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383876908663:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.739361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383876908664:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:34.734782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382087851333:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:34.734817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382087851334:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:34.734822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382087851335:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:34.759166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382087851379:2087] [node 6] ICP01 ready to work
2025-12-24T02:11:34.758985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381722663481:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:34.759006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383876908724:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:34.762986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241380406712730:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:34.763094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241380406712731:2093] [node 4] ICP01 ready to work
2025-12-24T02:11:34.771214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381722663488:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:34.773963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241380912632807:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:34.774742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241380912632813:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.779458Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712736:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.779559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712737:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.782961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241380406712736:2102] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.787040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241380406712731:2093] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.787688Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241380406712737:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.787719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241380406712678:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.788429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712757:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.788700Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241380406712737:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.789173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241380406712678:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.789190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.789195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241380406712678:2048] [node 1] ICP22 created new session: [6:7587241380406712758:2048]
2025-12-24T02:11:34.789200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241380406712758:2048] [node 1] ICS09 handshake done sender: [6:7587241380406712737:2103] self: [6:7587241380406712756:0] peer: [1:7587241381722663489:0] socket: 259 qp: -1
2025-12-24T02:11:34.789205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241380406712758:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.789226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.789532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241382904099289:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:34.786725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241381722663486:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:34.787820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663490:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.787826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381722663490:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.788515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381722663490:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.788802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381722663488:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.788817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.788824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381722663488:2110] [node 6] ICP22 created new session: [1:7587241381722663500:2110]
2025-12-24T02:11:34.788831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381722663500:2110] [node 6] ICS09 handshake done sender: [1:7587241381722663490:2111] self: [1:7587241381722663489:0] peer: [6:7587241380406712756:0] socket: 257 qp: -1
2025-12-24T02:11:34.788838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381722663500:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:34.788850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.790822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908728:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.790799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241382087851335:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31054551ns) {Configure:104(31053233ns) {ProcessConfigured:135(30835437ns) {StartInitialHandshake:157(30835241ns) {DropHandshakes:470(323ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(111ns) PrepareNewSessionHandshake:477(1346ns) {SwitchToState:179(833ns)}}}}}
2025-12-24T02:11:34.794754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908726:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.794771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241383876908726:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.798793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241382087851334:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36020654ns) {Configure:104(36016849ns) {ProcessConfigured:135(35555121ns) {StartInitialHandshake:157(35553841ns) {DropHandshakes:470(1439ns) {DropIncomingHandshake:442(194ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(1549ns) {SwitchToState:179(1027ns)}}}}}
2025-12-24T02:11:34.799005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241382087851403:2084] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.802962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712764:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.803109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381722663503:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49915
2025-12-24T02:11:34.810844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241380912632813:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36029505ns) {Configure:104(36028647ns) {ProcessConfigured:135(35810190ns) {StartInitialHandshake:157(35809279ns) {DropHandshakes:470(155ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1971ns) {SwitchToState:179(269ns)}}}}}
2025-12-24T02:11:34.811054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241380912632819:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.814810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241380406712764:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.814909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241380406712730:2086] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.818858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908729:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.820015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241382904099295:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:34.822713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382087851382:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.822729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241382087851382:2094] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.818940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.824772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.824788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712768:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.825557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241380406712764:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.825626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241380406712730:2086] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.825639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.825644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241380406712730:2086] [node 5] ICP22 created new session: [6:7587241380406712769:2086]
2025-12-24T02:11:34.825649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241380406712769:2086] [node 5] ICS09 handshake done sender: [6:7587241380406712764:2118] self: [6:7587241380406712766:0] peer: [5:7587241382087851381:0] socket: 278 qp: -1
2025-12-24T02:11:34.825654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241380406712769:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:34.825659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.823458Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908728:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.824458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383876908664:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.824889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382087851382:2094] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.825019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241382087851379:2087] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.825031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.825037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382087851379:2087] [node 6] ICP22 created new session: [5:7587241382087851413:2087]
2025-12-24T02:11:34.825043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382087851413:2087] [node 6] ICS09 handshake done sender: [5:7587241382087851382:2094] self: [5:7587241382087851381:0] peer: [6:7587241380406712766:0] socket: 276 qp: -1
2025-12-24T02:11:34.825050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382087851413:2087] [node 6] ICS10 traffic start
2025-12-24T02:11:34.825067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.825336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851412:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.826813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908730:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.826884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383876908726:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.827045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241380406712736:2102] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.831046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632827:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.831112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.831387Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908729:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.831139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632825:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.831183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632826:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.830714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099293:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
2025-12-24T02:11:34.831282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241382904099300:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:34.831736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099291:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.831739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241382904099291:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.834736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382087851412:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.834824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382087851333:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.838910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632827:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.838936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632825:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.839560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099297:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
2025-12-24T02:11:34.839614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099301:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
2025-12-24T02:11:34.839659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099302:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.842993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851417:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.846778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099301:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.846802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241383876908724:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.846826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.846834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383876908724:2108] [node 6] ICP22 created new session: [4:7587241383876908740:2108]
2025-12-24T02:11:34.850148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241380406712731:2093] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.846843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383876908740:2108] [node 6] ICS09 handshake done sender: [4:7587241383876908726:2109] self: [4:7587241383876908725:0] peer: [6:7587241380406712755:0] socket: 252 qp: -1
2025-12-24T02:11:34.850169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.846851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383876908740:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:34.850176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241380406712731:2093] [node 4] ICP22 created new session: [6:7587241380406712771:2093]
2025-12-24T02:11:34.846865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.850182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241380406712771:2093] [node 4] ICS09 handshake done sender: [6:7587241380406712736:2102] self: [6:7587241380406712755:0] peer: [4:7587241383876908725:0] socket: 255 qp: -1
2025-12-24T02:11:34.847153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241383876908742:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:34.850190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241380406712771:2093] [node 4] ICS10 traffic start
2025-12-24T02:11:34.849889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241383876908724:2108] [node 6] ICP18 handshake (actor [4:7587241383876908729:2111]) is too old
2025-12-24T02:11:34.850199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.851069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381722663504:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49915
2025-12-24T02:11:34.854717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663483:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.854739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381722663483:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.854777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099293:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.854888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099297:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.854949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241382904099228:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.857564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241382904099291:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.857668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241382904099289:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.857681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.857689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241382904099289:2107] [node 4] ICP22 created new session: [3:7587241382904099315:2107]
2025-12-24T02:11:34.857696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241382904099315:2107] [node 4] ICS09 handshake done sender: [3:7587241382904099291:2108] self: [3:7587241382904099290:0] peer: [4:7587241383876908736:0] socket: 270 qp: -1
2025-12-24T02:11:34.857702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241382904099315:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:34.857711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.857964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382087851417:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.858007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382087851335:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.863029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632826:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.866648Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632831:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.867212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241380406712765:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.867251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241380406712679:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.870714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241380912632746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.870755Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241380912632746:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31742063ns) {IncomingHandshake:261(31741838ns) {DropIncomingHandshake:442(157ns) PrepareNewSessionHandshake:477(695ns) {SwitchToState:179(514ns)} GenerateSessionVirtualId:988(221ns) IssueIncomingHandshakeReply:193(31739078ns) {SwitchToState:179(1219ns)}}}
2025-12-24T02:11:34.870757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663494:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.870776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381722663494:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.875145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663492:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.875154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381722663492:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.875955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099299:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.875968Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241382904099299:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.878765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632828:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.878789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663504:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.878858Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381722663483:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.879141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908745:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.879060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632818:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.879078Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241380912632818:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.882876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381722663492:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.882972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381722663481:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.882989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.882997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381722663481:2107] [node 5] ICP22 created new session: [1:7587241381722663507:2107]
2025-12-24T02:11:34.883004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381722663507:2107] [node 5] ICS09 handshake done sender: [1:7587241381722663483:2108] self: [1:7587241381722663482:0] peer: [5:7587241382087851416:0] socket: 253 qp: -1
2025-12-24T02:11:34.883010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381722663507:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:34.883020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.883242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381722663420:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.883249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.883255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381722663420:2048] [node 2] ICP22 created new session: [1:7587241381722663509:2048]
2025-12-24T02:11:34.883259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381722663509:2048] [node 2] ICS09 handshake done sender: [1:7587241381722663492:2112] self: [1:7587241381722663491:0] peer: [2:7587241380912632830:0] socket: 258 qp: -1
2025-12-24T02:11:34.883264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381722663509:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:34.883269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.878868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241382904099299:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.883009Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241382904099289:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(25291895ns) {HandleHandshakeStatus:329(25291259ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(154ns) SwitchToState:179(58ns) ProcessPendingSessionEvents:495(12158203ns) {unordered ForwardSessionEventToSession:612 num# 30 sum# 12149938ns max# 12118763ns}}}
2025-12-24T02:11:34.883035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241382904099295:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.883049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.883054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241382904099295:2110] [node 5] ICP22 created new session: [3:7587241382904099319:2110]
2025-12-24T02:11:34.883058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241382904099319:2110] [node 5] ICS09 handshake done sender: [3:7587241382904099299:2113] self: [3:7587241382904099298:0] peer: [5:7587241382087851418:0] socket: 300 qp: -1
2025-12-24T02:11:34.883063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241382904099319:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:34.883069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:34.886851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908745:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.888876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099321:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
2025-12-24T02:11:34.889497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382087851335:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8955) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241382087851417:2116] held: no
2025-12-24T02:11:34.890866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381722663494:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.894936Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851421:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.895034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851422:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.895088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851420:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.895216Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712774:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.898943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908748:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.898962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908749:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.899034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383876908728:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:34.899095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381722663421:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.899122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.899131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381722663421:2049] [node 3] ICP22 created new session: [1:7587241381722663514:2049]
2025-12-24T02:11:34.899140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381722663514:2049] [node 3] ICS09 handshake done sender: [1:7587241381722663494:2113] self: [1:7587241381722663493:0] peer: [3:7587241382904099313:0] socket: 269 qp: -1
2025-12-24T02:11:34.899148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381722663514:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:34.899160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.899263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383876908664:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.899287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.899293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383876908664:2050] [node 3] ICP22 created new session: [4:7587241383876908750:2050]
2025-12-24T02:11:34.899299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383876908750:2050] [node 3] ICS09 handshake done sender: [4:7587241383876908728:2110] self: [4:7587241383876908736:0] peer: [3:7587241382904099290:0] socket: 272 qp: -1
2025-12-24T02:11:34.899304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383876908750:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:34.899325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.899708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908749:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.899735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383876908662:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.899442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381722663516:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49915
2025-12-24T02:11:34.899467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241381722663517:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49915
2025-12-24T02:11:34.900011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663498:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.900019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241381722663498:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.900611Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241381722663498:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.900684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241381722663486:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.900701Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.900708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241381722663486:2109] [node 4] ICP22 created new session: [1:7587241381722663519:2109]
2025-12-24T02:11:34.900715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241381722663519:2109] [node 4] ICS09 handshake done sender: [1:7587241381722663498:2114] self: [1:7587241381722663497:0] peer: [4:7587241383876908752:0] socket: 273 qp: -1
2025-12-24T02:11:34.900720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241381722663519:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:34.900725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.899903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241382904099297:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.902851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241380406712774:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.902677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241382904099228:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.902690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.902709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241382904099228:2048] [node 1] ICP22 created new session: [3:7587241382904099323:2048]
2025-12-24T02:11:34.902729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241382904099323:2048] [node 1] ICS09 handshake done sender: [3:7587241382904099297:2112] self: [3:7587241382904099313:0] peer: [1:7587241381722663493:0] socket: 284 qp: -1
2025-12-24T02:11:34.902734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241382904099323:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.902741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.902950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663516:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.902895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241380406712680:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.903201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632831:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.903890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241380912632818:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.907575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241381722663421:2049] [node 3] ICP18 handshake (actor [1:7587241381722663516:2117]) is too old
2025-12-24T02:11:34.907007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241380912632813:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.907023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.907028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241380912632813:2108] [node 6] ICP22 created new session: [2:7587241380912632837:2108]
2025-12-24T02:11:34.907035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241380912632837:2108] [node 6] ICS09 handshake done sender: [2:7587241380912632818:2112] self: [2:7587241380912632814:0] peer: [6:7587241380406712773:0] socket: 281 qp: -1
2025-12-24T02:11:34.907040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241380912632837:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:34.907059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.919464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382087851422:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.919568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382087851412:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.922342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382087851417:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.922784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241381722663517:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.922849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241381722663481:2107] [node 5] ICP18 handshake (actor [1:7587241381722663517:2118]) is too old
2025-12-24T02:11:34.923938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382087851335:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.923963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.923972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382087851335:2050] [node 3] ICP22 created new session: [5:7587241382087851423:2050]
2025-12-24T02:11:34.923979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382087851423:2050] [node 3] ICS09 handshake done sender: [5:7587241382087851417:2116] self: [5:7587241382087851418:0] peer: [3:7587241382904099298:0] socket: 302 qp: -1
2025-12-24T02:11:34.923988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382087851423:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:34.923997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.924406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099302:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.924802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241382904099229:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.924192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241380912632836:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56682
2025-12-24T02:11:34.924287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241380912632825:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.926719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241382904099306:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.926734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241382904099306:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.930888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382087851333:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.930913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.930922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382087851333:2048] [node 1] ICP22 created new session: [5:7587241382087851425:2048]
2025-12-24T02:11:34.930929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382087851425:2048] [node 1] ICS09 handshake done sender: [5:7587241382087851412:2115] self: [5:7587241382087851416:0] peer: [1:7587241381722663482:0] socket: 262 qp: -1
2025-12-24T02:11:34.930938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382087851425:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.930946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.930870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908730:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.931351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241383876908662:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241383876908749:2119] held: no
2025-12-24T02:11:34.931488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241383876908663:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.934897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241380406712679:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27357) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241380406712765:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.932774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382087851403:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.935122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712776:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.932883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851427:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.935484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241380406712765:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.935290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.942306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241383876908744:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.942323Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241383876908744:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.939281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632810:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.939299Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241380912632810:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.941177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241380912632746:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.941196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.941205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241380912632746:2048] [node 1] ICP22 created new session: [2:7587241380912632841:2048]
2025-12-24T02:11:34.941212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241380912632841:2048] [node 1] ICS09 handshake done sender: [2:7587241380912632825:2115] self: [2:7587241380912632830:0] peer: [1:7587241381722663491:0] socket: 260 qp: -1
2025-12-24T02:11:34.941222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241380912632841:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.941232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.941613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632812:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.941617Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241380912632812:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:34.942429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241380912632810:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.942783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383876908749:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:34.946843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383876908662:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.946870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:34.946879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383876908662:2048] [node 1] ICP22 created new session: [4:7587241383876908757:2048]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.947414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241380912632812:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:34.947475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241380406712679:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.947491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.947498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241380406712679:2049] [node 2] ICP22 created new session: [6:7587241380406712777:2049]
2025-12-24T02:11:34.947505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241380406712777:2049] [node 2] ICS09 handshake done sender: [6:7587241380406712765:2119] self: [6:7587241380406712773:0] peer: [2:7587241380912632814:0] socket: 283 qp: -1
2025-12-24T02:11:34.947511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241380406712777:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.947518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.948288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241380912632807:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.948306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.948311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241380912632807:2107] [node 4] ICP22 created new session: [2:7587241380912632844:2107]
2025-12-24T02:11:34.948316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241380912632844:2107] [node 4] ICS09 handshake done sender: [2:7587241380912632810:2109] self: [2:7587241380912632809:0] peer: [4:7587241383876908754:0] socket: 266 qp: -1
2025-12-24T02:11:34.948320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241380912632844:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:34.948326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.948796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241380912632747:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.948805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:34.948809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241380912632747:2049] [node 3] ICP22 created new session: [2:7587241380912632846:2049]
2025-12-24T02:11:34.948812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241380912632846:2049] [node 3] ICS09 handshake done sender: [2:7587241380912632812:2110] self: [2:7587241380912632811:0] peer: [3:7587241382904099325:0] socket: 280 qp: -1
2025-12-24T02:11:34.948816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241380912632846:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:34.948820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:34.946886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383876908757:2048] [node 1] ICS09 handshake done sender: [4:7587241383876908749:2119] self: [4:7587241383876908752:0] peer: [1:7587241381722663497:0] socket: 318 qp: -1
2025-12-24T02:11:34.946898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383876908757:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:34.946907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:34.946970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241383876908755:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7695
2025-12-24T02:11:34.947230Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383876908730:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:34.947285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241383876908663:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.947297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.947301Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241383876908744:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:34.947303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383876908663:2049] [node 2] ICP22 created new session: [4:7587241383876908759:2049]
2025-12-24T02:11:34.947309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383876908759:2049] [node 2] ICS09 handshake done sender: [4:7587241383876908730:2112] self: [4:7587241383876908754:0] peer: [2:7587241380912632809:0] socket: 282 qp: -1
2025-12-24T02:11:34.947314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383876908759:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.947319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.947633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241383876908742:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.947656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:34.947661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241383876908742:2115] [node 5] ICP22 created new session: [4:7587241383876908761:2115]
2025-12-24T02:11:34.947666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241383876908761:2115] [node 5] ICS09 handshake done sender: [4:7587241383876908744:2116] self: [4:7587241383876908743:0] peer: [5:7587241382087851428:0] socket: 312 qp: -1
2025-12-24T02:11:34.947671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241383876908761:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:34.947677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.950790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241382087851427:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.950840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241382087851334:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:34.952771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241382904099326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64290
2025-12-24T02:11:34.959003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241382904099302:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.963049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241382904099229:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.963075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:34.963083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241382904099229:2049] [node 2] ICP22 created new session: [3:7587241382904099328:2049]
2025-12-24T02:11:34.963090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241382904099328:2049] [node 2] ICS09 handshake done sender: [3:7587241382904099302:2115] self: [3:7587241382904099325:0] peer: [2:7587241380912632811:0] socket: 299 qp: -1
2025-12-24T02:11:34.963100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241382904099328:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:34.963106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:34.962890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382087851403:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3870) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241382087851422:2119] held: no
2025-12-24T02:11:34.963491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241382087851430:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:34.975871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382087851422:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:34.979166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241382904099306:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:34.979337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241382904099300:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:34.979359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:34.979371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241382904099300:2111] [node 6] ICP22 created new session: [3:7587241382904099330:2111]
2025-12-24T02:11:34.979378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241382904099330:2111] [node 6] ICS09 handshake done sender: [3:7587241382904099306:2117] self: [3:7587241382904099305:0] peer: [6:7587241380406712775:0] socket: 308 qp: -1
2025-12-24T02:11:34.979385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241382904099330:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:34.979396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:34.982923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382087851403:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:34.982942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:34.982950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382087851403:2084] [node 4] ICP22 created new session: [5:7587241382087851431:2084]
2025-12-24T02:11:34.982957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241382087851431:2084] [node 4] ICS09 handshake done sender: [5:7587241382087851422:2119] self: [5:7587241382087851428:0] peer: [4:7587241383876908743:0] socket: 317 qp: -1
2025-12-24T02:11:34.982963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241382087851431:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:34.982974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:34.990855Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241380406712680:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8955) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241380406712774:2122] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:34.994830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241380406712779:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36382
2025-12-24T02:11:34.995294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241380912632821:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:34.995313Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241380912632821:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.002838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241380406712774:2122] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.006841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241380912632821:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:35.006960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241380912632819:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.006974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.006982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241380912632819:2111] [node 5] ICP22 created new session: [2:7587241385207600145:2111]
2025-12-24T02:11:35.006989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241385207600145:2111] [node 5] ICS09 handshake done sender: [2:7587241380912632821:2113] self: [2:7587241380912632820:0] peer: [5:7587241382087851429:0] socket: 286 qp: -1
2025-12-24T02:11:35.006993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241385207600145:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:35.007000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:35.006867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241380406712680:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.006889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.006898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241380406712680:2050] [node 3] ICP22 created new session: [6:7587241384701680076:2050]
2025-12-24T02:11:35.006905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241384701680076:2050] [node 3] ICS09 handshake done sender: [6:7587241380406712774:2122] self: [6:7587241380406712775:0] peer: [3:7587241382904099305:0] socket: 311 qp: -1
2025-12-24T02:11:35.006913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241384701680076:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:35.006927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.006899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386382818729:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43118
2025-12-24T02:11:35.007087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241382087851427:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.007390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241382087851334:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.007401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.007405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241382087851334:2049] [node 2] ICP22 created new session: [5:7587241386382818730:2049]
2025-12-24T02:11:35.007411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386382818730:2049] [node 2] ICS09 handshake done sender: [5:7587241382087851427:2120] self: [5:7587241382087851429:0] peer: [2:7587241380912632820:0] socket: 290 qp: -1
2025-12-24T02:11:35.007415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386382818730:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.007421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:35.032850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:35.032950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241382904099323:2048] [node 1] ICS07 socket disconnect 284 reason# EndOfStream
2025-12-24T02:11:35.032957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241382904099323:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.032989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.033006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241382904099323:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:35.034782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:35.034871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241380912632841:2048] [node 1] ICS01 socket: 260 reason# EndOfStream
2025-12-24T02:11:35.034878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241380912632746:2048] [node 1] ICP30 unregister session Session# [2:7587241380912632841:2048] VirtualId# [2:7587241380912632830:0]
2025-12-24T02:11:35.034885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241380912632841:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.034930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.034959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.035125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241382904099228:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:35.035137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241382904099323:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:35.035351Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241382904099228:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.035357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241382904099323:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.035362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241382904099228:2048] [node 1] ICP30 unregister session Session# [3:7587241382904099323:2048] VirtualId# [3:7587241382904099313:0]
2025-12-24T02:11:35.035405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.035410Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241382904099228:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.035413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.041556Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:35.041649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241380406712758:2048] [node 1] ICS07 socket disconnect 259 reason# EndOfStream
2025-12-24T02:11:35.041656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241380406712758:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.041700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.041722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241380406712758:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:35.042029Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241380406712678:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.042036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241380406712758:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.042042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241380406712678:2048] [node 1] ICP30 unregister session Session# [6:7587241380406712758:2048] VirtualId# [6:7587241380406712756:0]
2025-12-24T02:11:35.042069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.042076Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241380406712678:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.042078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.042820Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:35.042898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241383876908757:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:35.042904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241383876908662:2048] [node 1] ICP30 unregister session Session# [4:7587241383876908757:2048] VirtualId# [4:7587241383876908752:0]
2025-12-24T02:11:35.042909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241383876908757:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.042950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.042974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.042859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:35.042926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241382087851425:2048] [node 1] ICS07 socket disconnect 262 reason# EndOfStream
2025-12-24T02:11:35.042930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241382087851425:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.042960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.042979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241382087851425:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:35.046861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382087851333:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.046876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241382087851425:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.046880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382087851333:2048] [node 1] ICP30 unregister session Session# [5:7587241382087851425:2048] VirtualId# [5:7587241382087851416:0]
2025-12-24T02:11:35.046904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.046909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241382087851333:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.046911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64450) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.046880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241382904099228:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:35.049335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241380406712678:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:35.050780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:35.050845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241382904099328:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-12-24T02:11:35.050851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241382904099229:2049] [node 2] ICP30 unregister session Session# [3:7587241382904099328:2049] VirtualId# [3:7587241382904099325:0]
2025-12-24T02:11:35.050855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241382904099328:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.050895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.050920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.057397Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:35.057458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241386382818730:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-12-24T02:11:35.057463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382087851334:2049] [node 2] ICP30 unregister session Session# [5:7587241386382818730:2049] VirtualId# [5:7587241382087851429:0]
2025-12-24T02:11:35.057468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241386382818730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.057490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.057511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.057814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241382087851333:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:35.058869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:35.058939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241383876908759:2049] [node 2] ICS01 socket: 282 reason# EndOfStream
2025-12-24T02:11:35.058945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241383876908663:2049] [node 2] ICP30 unregister session Session# [4:7587241383876908759:2049] VirtualId# [4:7587241383876908754:0]
2025-12-24T02:11:35.058951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241383876908759:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.058992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.059014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.057083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:35.057152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241380406712777:2049] [node 2] ICS07 socket disconnect 283 reason# EndOfStream
2025-12-24T02:11:35.057157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241380406712777:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.057206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.057226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241380406712777:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:35.057752Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241380406712679:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27357) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.057758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241380406712777:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.057762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241380406712679:2049] [node 2] ICP30 unregister session Session# [6:7587241380406712777:2049] VirtualId# [6:7587241380406712773:0]
2025-12-24T02:11:35.057779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.057784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241380406712679:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.057787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27357) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.074738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241380406712679:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:35.086771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:35.086870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241382087851423:2050] [node 3] ICS07 socket disconnect 302 reason# EndOfStream
2025-12-24T02:11:35.086877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241382087851423:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.086924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:35.086948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241382087851423:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:35.087131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:35.087202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241384701680076:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-12-24T02:11:35.087208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241380406712680:2050] [node 3] ICP30 unregister session Session# [6:7587241384701680076:2050] VirtualId# [6:7587241380406712775:0]
2025-12-24T02:11:35.087213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241384701680076:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.087237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:35.087259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:35.087606Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241382087851335:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8955) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.087617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241382087851423:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.087622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241382087851335:2050] [node 3] ICP30 unregister session Session# [5:7587241382087851423:2050] VirtualId# [5:7587241382087851418:0]
2025-12-24T02:11:35.087662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:35.087667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241382087851335:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8955) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.087670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8955) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.102869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:35.102966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241380406712771:2093] [node 4] ICS07 socket disconnect 255 reason# EndOfStream
2025-12-24T02:11:35.102973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241380406712771:2093] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.103024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:35.103046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241380406712771:2093] [node 4] ICS15 start handshake
2025-12-24T02:11:35.103418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241380406712731:2093] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3870) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.103432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241380406712771:2093] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.103436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241380406712731:2093] [node 4] ICP30 unregister session Session# [6:7587241380406712771:2093] VirtualId# [6:7587241380406712755:0]
2025-12-24T02:11:35.103463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:35.103469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241380406712731:2093] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3870) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.103472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3870) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.114517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241380406712731:2093] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:35.162878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241387309194869:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.162901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241387309194870:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:35.166802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386957180240:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.166813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386957180241:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.166819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386957180242:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.170715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241385550083447:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.170737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241385550083448:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.161499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385154554090:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:35.161753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385154554091:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:35.186804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386301427261:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.190744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386301427259:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.190802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386301427260:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.194836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241386988683089:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.194849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241386988683090:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.194855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241386988683091:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.236421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385154554151:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:35.234976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241387309194930:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.239273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194934:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.242981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241386988683121:2072] [node 4] ICP01 ready to work
2025-12-24T02:11:35.242949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241385550083511:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:35.244181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683123:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.244205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683122:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.245188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083515:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.244496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385154554159:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:35.246868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241386988683136:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:35.250914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386301427321:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.252787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083516:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.259531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241387309194937:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:35.261327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194941:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
2025-12-24T02:11:35.262922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194934:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.262112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386301427329:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:35.263363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180304:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.263383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180305:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.266814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241385154554091:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19973293ns) {Configure:104(19972946ns) {ProcessConfigured:135(19688381ns) {StartInitialHandshake:157(19688034ns) {DropHandshakes:470(326ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1409ns) {SwitchToState:179(992ns)}}}}}
2025-12-24T02:11:35.266942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180305:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.267061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385154554167:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59961
2025-12-24T02:11:35.267078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385154554168:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59961
2025-12-24T02:11:35.267267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554168:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.268269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554167:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.271561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194942:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
2025-12-24T02:11:35.271932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241387309194870:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24881812ns) {Configure:104(24880786ns) {ProcessConfigured:135(24510629ns) {StartInitialHandshake:157(24509977ns) {DropHandshakes:470(675ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(688ns) PrepareNewSessionHandshake:477(894ns) {SwitchToState:179(478ns)}}}}}
2025-12-24T02:11:35.271660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683131:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.271781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683167:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.273272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241386988683131:2084] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.273294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241386988683122:2079] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.274548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241386988683091:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.274565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241386988683090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.272236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083520:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.274794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083515:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.274903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241386988683167:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.275157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180304:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.275341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386957180313:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:35.275667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386957180313:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.275783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241386988683128:2083] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.275789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241386988683128:2083] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.276335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180315:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.276420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241386988683128:2083] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:35.276675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386957180304:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:35.276747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241386988683121:2072] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.276764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.276770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241386988683121:2072] [node 4] ICP22 created new session: [6:7587241386988683172:2072]
2025-12-24T02:11:35.276776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241386988683172:2072] [node 4] ICS09 handshake done sender: [6:7587241386988683128:2083] self: [6:7587241386988683127:0] peer: [4:7587241386957180314:0] socket: 270 qp: -1
2025-12-24T02:11:35.276782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241386988683172:2072] [node 4] ICS10 traffic start
2025-12-24T02:11:35.276791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.277208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386957180313:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.277219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:35.277224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386957180313:2113] [node 6] ICP22 created new session: [4:7587241386957180316:2113]
2025-12-24T02:11:35.277228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386957180316:2113] [node 6] ICS09 handshake done sender: [4:7587241386957180304:2109] self: [4:7587241386957180314:0] peer: [6:7587241386988683127:0] socket: 276 qp: -1
2025-12-24T02:11:35.277233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386957180316:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:35.277238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.280061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194932:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.280074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241387309194932:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.278860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427335:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.278887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427336:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.278947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427337:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.282804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083513:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.282293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386957180302:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:35.282829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241385550083513:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.282898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385154554165:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:35.286757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427336:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.286794Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427337:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.287233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386301427260:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.287419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241386988683123:2080] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.287563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427335:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.287591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386301427259:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.286953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241385550083523:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:35.290999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194946:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
2025-12-24T02:11:35.291077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194942:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.293291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554161:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.293311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385154554161:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.293686Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083516:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.293861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241385550083526:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:35.294241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241385550083526:2115] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.294841Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241386988683136:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23801609ns) {Configure:104(23801051ns) {ProcessConfigured:135(23455559ns) {StartInitialHandshake:157(23455110ns) {DropHandshakes:470(406ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1675ns) {SwitchToState:179(1205ns)}}}}}
2025-12-24T02:11:35.294935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241386988683136:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.295311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194941:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.295345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194946:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.295514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241387309194948:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:35.296072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241387309194948:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.296511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241387309194869:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.298851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083520:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.298881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241385550083448:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.302787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241387309194932:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.307414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241385154554165:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24404552ns) {Configure:104(24404101ns) {ProcessConfigured:135(24062903ns) {StartInitialHandshake:157(24044281ns) {DropHandshakes:470(504ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:477(1786ns) {SwitchToState:179(1050ns)}} IncomingHandshake:261(8105ns) {DropIncomingHandshake:442(103ns) PrepareNewSessionHandshake:477(235ns) {SwitchToState:179(174ns)} GenerateSessionVirtualId:988(260ns) IssueIncomingHandshakeReply:193(340ns)}}}}
2025-12-24T02:11:35.307774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554153:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.307778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385154554153:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.309138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385154554153:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:35.309224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385154554151:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.309233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:35.309238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385154554151:2107] [node 6] ICP22 created new session: [1:7587241385154554174:2107]
2025-12-24T02:11:35.309243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385154554174:2107] [node 6] ICS09 handshake done sender: [1:7587241385154554153:2108] self: [1:7587241385154554152:0] peer: [6:7587241386988683178:0] socket: 260 qp: -1
2025-12-24T02:11:35.309248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385154554174:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:35.309265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:35.308468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241385550083513:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:35.308574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241385550083511:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.308595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:35.308604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241385550083511:2108] [node 6] ICP22 created new session: [3:7587241385550083530:2108]
2025-12-24T02:11:35.308611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241385550083530:2108] [node 6] ICS09 handshake done sender: [3:7587241385550083513:2109] self: [3:7587241385550083512:0] peer: [6:7587241386988683169:0] socket: 268 qp: -1
2025-12-24T02:11:35.308618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241385550083530:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:35.308628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:35.309599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083532:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.310975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241385550083523:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23948069ns) {Configure:104(23947501ns) {ProcessConfigured:135(23596447ns) {StartInitialHandshake:157(23595754ns) {DropHandshakes:470(259ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(99ns) PrepareNewSessionHandshake:477(707ns) {SwitchToState:179(227ns)}}}}}
2025-12-24T02:11:35.307574Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241386988683089:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20747899ns) {Configure:104(20747208ns) {ProcessConfigured:135(20330388ns) {StartInitialHandshake:157(20329791ns) {DropHandshakes:470(656ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(967ns) {SwitchToState:179(28ns)}}}}}
2025-12-24T02:11:35.307613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241386988683089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.310419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241386988683091:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61263) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241386988683131:2084] held: no
2025-12-24T02:11:35.311347Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180310:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.311359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241386957180310:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.314892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194939:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.314908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241387309194939:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.314825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554157:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.314844Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385154554157:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.318720Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427326:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.318741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241386301427326:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.318769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427331:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.318779Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241386301427331:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.318718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241387309194943:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.318733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241387309194943:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.320890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241387309194930:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.320911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:35.320919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241387309194930:2107] [node 6] ICP22 created new session: [2:7587241387309194953:2107]
2025-12-24T02:11:35.320927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241387309194953:2107] [node 6] ICS09 handshake done sender: [2:7587241387309194932:2108] self: [2:7587241387309194931:0] peer: [6:7587241386988683170:0] socket: 258 qp: -1
2025-12-24T02:11:35.320935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241387309194953:2107] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.320946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:35.320793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683180:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.321293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683181:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.322927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385154554161:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.326986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385154554178:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59961
2025-12-24T02:11:35.327105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083532:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.327154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241385550083447:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.327289Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554166:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.327294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385154554166:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.327817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083536:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.327928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385154554166:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.328091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241385550083532:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:35.328169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241385550083447:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.328190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:35.328195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241385550083447:2048] [node 1] ICP22 created new session: [3:7587241385550083537:2048]
2025-12-24T02:11:35.328201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241385550083537:2048] [node 1] ICS09 handshake done sender: [3:7587241385550083532:2116] self: [3:7587241385550083535:0] peer: [1:7587241385154554163:0] socket: 307 qp: -1
2025-12-24T02:11:35.328208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241385550083537:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:35.328216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:35.329137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385154554091:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.329157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.329161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385154554091:2049] [node 3] ICP22 created new session: [1:7587241385154554180:2049]
2025-12-24T02:11:35.329167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385154554180:2049] [node 3] ICS09 handshake done sender: [1:7587241385154554166:2113] self: [1:7587241385154554163:0] peer: [3:7587241385550083535:0] socket: 306 qp: -1
2025-12-24T02:11:35.329171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385154554180:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:35.329176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.330776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385154554159:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.330795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.330801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385154554159:2109] [node 5] ICP22 created new session: [1:7587241385154554183:2109]
2025-12-24T02:11:35.330807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385154554183:2109] [node 5] ICS09 handshake done sender: [1:7587241385154554161:2111] self: [1:7587241385154554160:0] peer: [5:7587241386301427340:0] socket: 271 qp: -1
2025-12-24T02:11:35.330844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385154554183:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:35.330851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.331913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194955:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
2025-12-24T02:11:35.332061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241387309194939:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:35.332159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241387309194937:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.332168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.332174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241387309194937:2110] [node 5] ICP22 created new session: [2:7587241387309194958:2110]
2025-12-24T02:11:35.332181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241387309194958:2110] [node 5] ICS09 handshake done sender: [2:7587241387309194939:2111] self: [2:7587241387309194938:0] peer: [5:7587241386301427339:0] socket: 282 qp: -1
2025-12-24T02:11:35.332186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241387309194958:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:35.332193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:35.332326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241387309194957:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45124
2025-12-24T02:11:35.334801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241386957180302:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(52381073ns) {Configure:104(52380614ns) {ProcessConfigured:135(52004661ns) {StartInitialHandshake:157(51986832ns) {DropHandshakes:470(815ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(1100ns) PrepareNewSessionHandshake:477(1378ns) {SwitchToState:179(251ns)}} IncomingHandshake:261(8154ns) {DropIncomingHandshake:442(67ns) PrepareNewSessionHandshake:477(187ns) {SwitchToState:179(139ns)} GenerateSessionVirtualId:988(321ns) IssueIncomingHandshakeReply:193(856ns)}}}}
2025-12-24T02:11:35.334886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386957180310:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.336860Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241387309194943:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.337073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241387309194870:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.337092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.337098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241387309194870:2049] [node 3] ICP22 created new session: [2:7587241387309194961:2049]
2025-12-24T02:11:35.337103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241387309194961:2049] [node 3] ICS09 handshake done sender: [2:7587241387309194943:2114] self: [2:7587241387309194935:0] peer: [3:7587241385550083528:0] socket: 294 qp: -1
2025-12-24T02:11:35.337108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241387309194961:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:35.337113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.337197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241387309194942:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:35.337487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241387309194948:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.337497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.337501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241387309194948:2116] [node 4] ICP22 created new session: [2:7587241387309194963:2116]
2025-12-24T02:11:35.337505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241387309194963:2116] [node 4] ICS09 handshake done sender: [2:7587241387309194942:2113] self: [2:7587241387309194950:0] peer: [4:7587241386957180309:0] socket: 291 qp: -1
2025-12-24T02:11:35.337509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241387309194963:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:35.337515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.337567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241387309194941:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:35.337744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241387309194869:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.337755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:35.337759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241387309194869:2048] [node 1] ICP22 created new session: [2:7587241387309194965:2048]
2025-12-24T02:11:35.337763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241387309194965:2048] [node 1] ICS09 handshake done sender: [2:7587241387309194941:2112] self: [2:7587241387309194951:0] peer: [1:7587241385154554156:0] socket: 275 qp: -1
2025-12-24T02:11:35.337768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241387309194965:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:35.337773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:35.336758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385154554157:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.338982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683182:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.339101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386301427326:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.342727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241386957180242:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(55968495ns) {Configure:104(55967658ns) {ProcessConfigured:135(55587975ns) {StartInitialHandshake:157(55587399ns) {DropHandshakes:470(744ns) {DropIncomingHandshake:442(203ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:477(1096ns) {SwitchToState:179(701ns)}}}}}
2025-12-24T02:11:35.342763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241386957180241:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.342772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.342782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386957180241:2049] [node 2] ICP22 created new session: [4:7587241386957180323:2049]
2025-12-24T02:11:35.342790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386957180323:2049] [node 2] ICS09 handshake done sender: [4:7587241386957180310:2112] self: [4:7587241386957180309:0] peer: [2:7587241387309194950:0] socket: 290 qp: -1
2025-12-24T02:11:35.342797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386957180323:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.342816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:35.342929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180324:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.342993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.342986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241385550083448:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241385550083520:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.343627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385154554090:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.343643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.343661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385154554090:2048] [node 2] ICP22 created new session: [1:7587241385154554186:2048]
2025-12-24T02:11:35.343667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385154554186:2048] [node 2] ICS09 handshake done sender: [1:7587241385154554157:2110] self: [1:7587241385154554156:0] peer: [2:7587241387309194951:0] socket: 267 qp: -1
2025-12-24T02:11:35.343673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385154554186:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:35.343681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.344772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.344846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083541:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.344864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241385550083540:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20463
2025-12-24T02:11:35.343260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241386301427261:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.343276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.343283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386301427261:2050] [node 3] ICP22 created new session: [5:7587241386301427345:2050]
2025-12-24T02:11:35.343289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386301427345:2050] [node 3] ICS09 handshake done sender: [5:7587241386301427326:2110] self: [5:7587241386301427325:0] peer: [3:7587241385550083527:0] socket: 279 qp: -1
2025-12-24T02:11:35.343295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386301427345:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:35.343313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.343563Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.347098Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241386988683090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241386988683122:2079] held: no
2025-12-24T02:11:35.347286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241386988683122:2079] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.347596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241386988683131:2084] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.346982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.347428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.347536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386301427335:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:35.347719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241386988683123:2080] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:35.347823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241386988683091:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.347836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.347841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241386988683091:2050] [node 3] ICP22 created new session: [6:7587241386988683184:2050]
2025-12-24T02:11:35.347846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241386988683184:2050] [node 3] ICS09 handshake done sender: [6:7587241386988683131:2084] self: [6:7587241386988683169:0] peer: [3:7587241385550083512:0] socket: 273 qp: -1
2025-12-24T02:11:35.347851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241386988683184:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:35.347856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.348016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241386988683089:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.348032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:35.348037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241386988683089:2048] [node 1] ICP22 created new session: [6:7587241386988683186:2048]
2025-12-24T02:11:35.348041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241386988683186:2048] [node 1] ICS09 handshake done sender: [6:7587241386988683123:2080] self: [6:7587241386988683178:0] peer: [1:7587241385154554152:0] socket: 266 qp: -1
2025-12-24T02:11:35.348044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241386988683186:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:35.348048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:35.348345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386301427259:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.348357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:35.348362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386301427259:2048] [node 1] ICP22 created new session: [5:7587241386301427349:2048]
2025-12-24T02:11:35.348366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386301427349:2048] [node 1] ICS09 handshake done sender: [5:7587241386301427335:2114] self: [5:7587241386301427340:0] peer: [1:7587241385154554160:0] socket: 277 qp: -1
2025-12-24T02:11:35.348371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386301427349:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:35.348375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:35.346919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180326:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.349388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386301427331:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:35.349461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386301427336:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.349540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241386301427329:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.349553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:35.349557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386301427329:2111] [node 6] ICP22 created new session: [5:7587241386301427351:2111]
2025-12-24T02:11:35.349561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386301427351:2111] [node 6] ICS09 handshake done sender: [5:7587241386301427331:2112] self: [5:7587241386301427330:0] peer: [6:7587241386988683176:0] socket: 283 qp: -1
2025-12-24T02:11:35.349565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386301427351:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:35.349570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:35.349690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386301427260:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.349694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.349697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386301427260:2049] [node 2] ICP22 created new session: [5:7587241386301427353:2049]
2025-12-24T02:11:35.349700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386301427353:2049] [node 2] ICS09 handshake done sender: [5:7587241386301427336:2115] self: [5:7587241386301427339:0] peer: [2:7587241387309194938:0] socket: 299 qp: -1
2025-12-24T02:11:35.349704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386301427353:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.349708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.349971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241386988683090:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.349983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.349987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241386988683090:2049] [node 2] ICP22 created new session: [6:7587241386988683188:2049]
2025-12-24T02:11:35.349992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241386988683188:2049] [node 2] ICS09 handshake done sender: [6:7587241386988683122:2079] self: [6:7587241386988683170:0] peer: [2:7587241387309194931:0] socket: 265 qp: -1
2025-12-24T02:11:35.349996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241386988683188:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.350001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:35.350119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241386988683183:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54095
2025-12-24T02:11:35.350897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554178:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.351461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241386988683136:2077] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30658) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241386988683167:2117] held: no
2025-12-24T02:11:35.351625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241386988683167:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:35.350748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386957180240:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.353899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241385154554151:2107] [node 6] ICP18 handshake (actor [1:7587241385154554178:2117]) is too old
2025-12-24T02:11:35.355478Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385154554171:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.355493Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385154554171:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.358922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385154554188:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59961
2025-12-24T02:11:35.362535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241385550083520:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.362794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241385550083516:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:35.362897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083541:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.362905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241385550083448:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.362916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.362922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241385550083448:2049] [node 2] ICP22 created new session: [3:7587241385550083542:2049]
2025-12-24T02:11:35.362927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241385550083542:2049] [node 2] ICS09 handshake done sender: [3:7587241385550083520:2113] self: [3:7587241385550083528:0] peer: [2:7587241387309194935:0] socket: 295 qp: -1
2025-12-24T02:11:35.362932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241385550083542:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.362939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:35.363016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241385550083526:2115] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.363023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.363027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241385550083526:2115] [node 5] ICP22 created new session: [3:7587241385550083544:2115]
2025-12-24T02:11:35.363031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241385550083544:2115] [node 5] ICS09 handshake done sender: [3:7587241385550083516:2111] self: [3:7587241385550083527:0] peer: [5:7587241386301427325:0] socket: 280 qp: -1
2025-12-24T02:11:35.363034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241385550083544:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:35.363039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:35.362680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241386988683136:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.362717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.362727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241386988683136:2077] [node 5] ICP22 created new session: [6:7587241386988683190:2077]
2025-12-24T02:11:35.362735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241386988683190:2077] [node 5] ICS09 handshake done sender: [6:7587241386988683167:2117] self: [6:7587241386988683176:0] peer: [5:7587241386301427330:0] socket: 292 qp: -1
2025-12-24T02:11:35.362746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241386988683190:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:35.362754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:35.366776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180324:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.366828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386957180242:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.367056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241385550083533:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.367066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241385550083533:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.370981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180331:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.371170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386957180324:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.371255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386957180242:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.371270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.371276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386957180242:2050] [node 3] ICP22 created new session: [4:7587241386957180332:2050]
2025-12-24T02:11:35.371281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386957180332:2050] [node 3] ICS09 handshake done sender: [4:7587241386957180324:2117] self: [4:7587241386957180330:0] peer: [3:7587241385550083524:0] socket: 310 qp: -1
2025-12-24T02:11:35.371287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386957180332:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:35.371294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.370802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241385550083533:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:35.370986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241385550083523:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.371008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.371052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241385550083523:2114] [node 4] ICP22 created new session: [3:7587241385550083548:2114]
2025-12-24T02:11:35.371060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241385550083548:2114] [node 4] ICS09 handshake done sender: [3:7587241385550083533:2117] self: [3:7587241385550083524:0] peer: [4:7587241386957180330:0] socket: 308 qp: -1
2025-12-24T02:11:35.371068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241385550083548:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:35.371077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.374820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386301427344:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.374879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386301427321:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.375026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386957180321:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.375050Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241386957180321:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.375636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385154554171:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:35.375800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385154554165:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.375821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.375830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385154554165:2112] [node 4] ICP22 created new session: [1:7587241385154554190:2112]
2025-12-24T02:11:35.375838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385154554190:2112] [node 4] ICS09 handshake done sender: [1:7587241385154554171:2116] self: [1:7587241385154554170:0] peer: [4:7587241386957180329:0] socket: 285 qp: -1
2025-12-24T02:11:35.375843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385154554190:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:35.375850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.386910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386957180321:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:35.387186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241386957180302:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.387210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:35.387219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386957180302:2108] [node 5] ICP22 created new session: [4:7587241386957180335:2108]
2025-12-24T02:11:35.387228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386957180335:2108] [node 5] ICS09 handshake done sender: [4:7587241386957180321:2116] self: [4:7587241386957180318:0] peer: [5:7587241386301427355:0] socket: 323 qp: -1
2025-12-24T02:11:35.387234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386957180335:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:35.387246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.402923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386301427356:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49783
2025-12-24T02:11:35.403317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386301427344:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:35.403380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386301427321:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.403398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.403405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386301427321:2108] [node 4] ICP22 created new session: [5:7587241386301427357:2108]
2025-12-24T02:11:35.403412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241386301427357:2108] [node 4] ICS09 handshake done sender: [5:7587241386301427344:2119] self: [5:7587241386301427355:0] peer: [4:7587241386957180318:0] socket: 325 qp: -1
2025-12-24T02:11:35.403422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241386301427357:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:35.403430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.406990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241386957180302:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(19700068ns) {HandleHandshakeStatus:329(19698374ns) {DropIncomingHandshake:442(6119ns) DropOutgoingHandshake:457(210ns) SwitchToState:179(360ns) ProcessPendingSessionEvents:495(19592284ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 19571063ns max# 19507873ns}}}
2025-12-24T02:11:35.408358Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241386957180240:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14826) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241386957180326:2118] held: no
2025-12-24T02:11:35.410859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241385154554165:2112] [node 4] ICP35 event processing took too much time PendingConnection:279(35006662ns) {HandleHandshakeStatus:329(35005825ns) {DropIncomingHandshake:442(6464ns) DropOutgoingHandshake:457(135ns) SwitchToState:179(93ns) ProcessPendingSessionEvents:495(34917897ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 34910168ns max# 34891507ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.422720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386957180337:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25453
2025-12-24T02:11:35.427288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386957180326:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:35.427401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386957180240:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.427419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:35.427433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386957180240:2048] [node 1] ICP22 created new session: [4:7587241386957180338:2048]
2025-12-24T02:11:35.427441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386957180338:2048] [node 1] ICS09 handshake done sender: [4:7587241386957180326:2118] self: [4:7587241386957180329:0] peer: [1:7587241385154554170:0] socket: 321 qp: -1
2025-12-24T02:11:35.427451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386957180338:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:35.427463Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:35.667771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:35.667842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241385550083537:2048] [node 1] ICS01 socket: 307 reason# EndOfStream
2025-12-24T02:11:35.667848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241385550083447:2048] [node 1] ICP30 unregister session Session# [3:7587241385550083537:2048] VirtualId# [3:7587241385550083535:0]
2025-12-24T02:11:35.667853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241385550083537:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.667877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.667894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.667196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:35.667313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241387309194965:2048] [node 1] ICS01 socket: 275 reason# EndOfStream
2025-12-24T02:11:35.667323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241387309194869:2048] [node 1] ICP30 unregister session Session# [2:7587241387309194965:2048] VirtualId# [2:7587241387309194951:0]
2025-12-24T02:11:35.667332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241387309194965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.667369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.667390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.674830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:35.674922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241386988683186:2048] [node 1] ICS07 socket disconnect 266 reason# EndOfStream
2025-12-24T02:11:35.674932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241386988683186:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.674984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.675006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241386988683186:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:35.675437Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241386988683089:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14826) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.675449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241386988683186:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.675453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241386988683089:2048] [node 1] ICP30 unregister session Session# [6:7587241386988683186:2048] VirtualId# [6:7587241386988683178:0]
2025-12-24T02:11:35.675470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.675476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241386988683089:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14826) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.675479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14826) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.674776Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:35.674882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241386301427349:2048] [node 1] ICS01 socket: 277 reason# EndOfStream
2025-12-24T02:11:35.674892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386301427259:2048] [node 1] ICP30 unregister session Session# [5:7587241386301427349:2048] VirtualId# [5:7587241386301427340:0]
2025-12-24T02:11:35.674899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241386301427349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.674970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.674997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.675447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:35.675525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241386957180338:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:35.675532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241386957180240:2048] [node 1] ICP30 unregister session Session# [4:7587241386957180338:2048] VirtualId# [4:7587241386957180329:0]
2025-12-24T02:11:35.675538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241386957180338:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.675583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:35.675609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:35.686739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241386988683089:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:35.695493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:35.695590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241385550083542:2049] [node 2] ICS01 socket: 295 reason# EndOfStream
2025-12-24T02:11:35.695599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241385550083448:2049] [node 2] ICP30 unregister session Session# [3:7587241385550083542:2049] VirtualId# [3:7587241385550083528:0]
2025-12-24T02:11:35.695607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241385550083542:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.695640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.695668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.701524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:35.701623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241386988683188:2049] [node 2] ICS07 socket disconnect 265 reason# EndOfStream
2025-12-24T02:11:35.701630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241386988683188:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.701683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.701703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241386988683188:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:35.705159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:35.705241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241386957180323:2049] [node 2] ICS07 socket disconnect 290 reason# EndOfStream
2025-12-24T02:11:35.705249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241386957180323:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.705296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.705318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241386957180323:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:35.705502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:35.705566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241386301427353:2049] [node 2] ICS01 socket: 299 reason# EndOfStream
2025-12-24T02:11:35.705571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386301427260:2049] [node 2] ICP30 unregister session Session# [5:7587241386301427353:2049] VirtualId# [5:7587241386301427339:0]
2025-12-24T02:11:35.705577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241386301427353:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.705603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:35.705624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.705901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241386988683090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.705917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241386988683188:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.705922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241386988683090:2049] [node 2] ICP30 unregister session Session# [6:7587241386988683188:2049] VirtualId# [6:7587241386988683170:0]
2025-12-24T02:11:35.705939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.705949Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241386988683090:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.705952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.710014Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241386957180241:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:35.710029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241386957180323:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:35.710034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241386957180241:2049] [node 2] ICP30 unregister session Session# [4:7587241386957180323:2049] VirtualId# [4:7587241386957180309:0]
2025-12-24T02:11:35.710049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:35.710054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241386957180241:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.710057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17584) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:35.718740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241386988683090:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:35.718784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241386957180241:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:35.722971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:35.723072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241386301427345:2050] [node 3] ICS01 socket: 279 reason# EndOfStream
2025-12-24T02:11:35.723081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386301427261:2050] [node 3] ICP30 unregister session Session# [5:7587241386301427345:2050] VirtualId# [5:7587241386301427325:0]
2025-12-24T02:11:35.723089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241386301427345:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.723136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:35.723169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:35.723354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:35.723419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241386988683184:2050] [node 3] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:35.723424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241386988683091:2050] [node 3] ICP30 unregister session Session# [6:7587241386988683184:2050] VirtualId# [6:7587241386988683169:0]
2025-12-24T02:11:35.723428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241386988683184:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.723453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:35.723476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:35.738902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:35.739008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241386988683172:2072] [node 4] ICS01 socket: 270 reason# EndOfStream
2025-12-24T02:11:35.739018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241386988683121:2072] [node 4] ICP30 unregister session Session# [6:7587241386988683172:2072] VirtualId# [6:7587241386988683127:0]
2025-12-24T02:11:35.739026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241386988683172:2072] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:35.739062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:35.739095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:35.810975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385030216303:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:35.814764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385030216302:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:35.818901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241387945466880:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.818919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241387945466881:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.818771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241385189027933:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.822271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241385189027934:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:35.846831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386053057753:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.846842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386053057754:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.846848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386053057755:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.842904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386054385915:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.850817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241384805556021:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:35.850829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241384805556022:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.850838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241384805556023:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.842914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386054385916:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:35.842919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386054385917:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:35.888651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386054385956:2087] [node 4] ICP01 ready to work
2025-12-24T02:11:35.889547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385030216364:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:35.889594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385030216365:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.903056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241384805556072:2089] [node 5] ICP01 ready to work
2025-12-24T02:11:35.903173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241384805556074:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:35.903820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241385189027996:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.906958Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057818:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.906984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057819:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.913676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241385189027934:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22838023ns) {Configure:104(22837144ns) {ProcessConfigured:135(21707436ns) {StartInitialHandshake:157(21706615ns) {DropHandshakes:470(404ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(494ns) PrepareNewSessionHandshake:477(1855ns) {SwitchToState:179(601ns)}}}}}
2025-12-24T02:11:35.914139Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241385189027933:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22851259ns) {Configure:104(22849767ns) {ProcessConfigured:135(21944075ns) {StartInitialHandshake:157(21943613ns) {DropHandshakes:470(198ns) {DropIncomingHandshake:442(98ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(596ns) {SwitchToState:179(152ns)}}}}}
2025-12-24T02:11:35.914324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241385189028004:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.915055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241386054385915:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22088436ns) {Configure:104(22087615ns) {ProcessConfigured:135(21706360ns) {StartInitialHandshake:157(21706167ns) {DropHandshakes:470(439ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1640ns) {SwitchToState:179(313ns)}}}}}
2025-12-24T02:11:35.915636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386054385985:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:35.918790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241385030216376:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:35.918717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057812:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.919065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241387945466941:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:35.919189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241385189028009:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:35.918825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386053057821:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:35.918961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057810:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.919317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386054385985:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.926993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241387945466948:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:35.927102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241387945466949:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:35.927162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241387945466946:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:35.927217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241387945466947:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:35.929486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241386054385977:2090] [node 6] ICP01 ready to work
2025-12-24T02:11:35.930953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241386053057817:2102] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:35.931299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466946:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.932510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385030216380:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64753
2025-12-24T02:11:35.932543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385030216381:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64753
2025-12-24T02:11:35.931029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057810:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.931053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057818:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.931340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241384805556023:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40494792ns) {Configure:104(40493347ns) {ProcessConfigured:135(40037941ns) {StartInitialHandshake:157(40037761ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:477(708ns) {SwitchToState:179(359ns)}}}}}
2025-12-24T02:11:35.931071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057819:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.931630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241385189028011:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
2025-12-24T02:11:35.931087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057812:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.931653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241385189028012:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
2025-12-24T02:11:35.935456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386053057754:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.938795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241387945466880:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19907489ns) {Configure:104(19906099ns) {ProcessConfigured:135(19005250ns) {StartInitialHandshake:157(19004628ns) {DropHandshakes:470(439ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1055ns) {SwitchToState:179(79ns)}}}}}
2025-12-24T02:11:35.938870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241387945466880:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.939047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241387945466941:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19916629ns) {Configure:104(19916253ns) {ProcessConfigured:135(19119991ns) {StartInitialHandshake:157(19119474ns) {DropHandshakes:470(401ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(718ns) {SwitchToState:179(110ns)}}}}}
2025-12-24T02:11:35.939211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216381:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.939078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241385189027998:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.939093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241385189027998:2108] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.939478Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241386053057755:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20551468ns) {Configure:104(20550962ns) {ProcessConfigured:135(19283244ns) {StartInitialHandshake:157(19282794ns) {DropHandshakes:470(133ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(949ns) {SwitchToState:179(506ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.940047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386054385992:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:35.940431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241385189028013:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
2025-12-24T02:11:35.940910Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241385189028009:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18284034ns) {Configure:104(18282400ns) {ProcessConfigured:135(17907040ns) {StartInitialHandshake:157(17906498ns) {DropHandshakes:470(536ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:477(756ns) {SwitchToState:179(112ns)}}}}}
2025-12-24T02:11:35.942795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241385030216379:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64753
2025-12-24T02:11:35.942903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216367:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.942908Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385030216367:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.943005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216380:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.943144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241385189028011:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.943254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216379:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.944928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241385030216302:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(50217267ns) {Configure:104(50216650ns) {ProcessConfigured:135(49842663ns) {StartInitialHandshake:157(49842314ns) {DropHandshakes:470(380ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:477(1335ns) {SwitchToState:179(703ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.946993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466947:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.947042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241387945466881:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.947228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241385189028000:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.947239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241385189028000:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.947396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241385189027998:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.946855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057837:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.948305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241386053057838:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:35.948472Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386053057818:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.948517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386053057754:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.948536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.948542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386053057754:2049] [node 2] ICP22 created new session: [4:7587241386053057839:2049]
2025-12-24T02:11:35.948550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241386053057839:2049] [node 2] ICS09 handshake done sender: [4:7587241386053057818:2111] self: [4:7587241386053057830:0] peer: [2:7587241385189027997:0] socket: 279 qp: -1
2025-12-24T02:11:35.948559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241386053057839:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.948580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.950822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241385189028000:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.948716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241386054385995:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.955152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241384805556100:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:35.955177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241384805556101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:35.955189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241384805556099:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:35.958902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385030216367:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:35.958989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241384805556099:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.959069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241384805556021:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.962820Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241386053057753:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43735978ns) {Configure:104(43735006ns) {ProcessConfigured:135(42634596ns) {StartInitialHandshake:157(42633650ns) {DropHandshakes:470(280ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(2494ns) {SwitchToState:179(1728ns)}}}}}
2025-12-24T02:11:35.962930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386053057753:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.963080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241385189027934:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.963106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.963115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241385189027934:2049] [node 3] ICP22 created new session: [2:7587241385189028019:2049]
2025-12-24T02:11:35.963124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241385189028019:2049] [node 3] ICS09 handshake done sender: [2:7587241385189028000:2109] self: [2:7587241385189027993:0] peer: [3:7587241387945466962:0] socket: 281 qp: -1
2025-12-24T02:11:35.963133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241385189028019:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:35.963145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.965380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241387945466963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:35.965456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241385189027996:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.965475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:35.965486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241385189027996:2107] [node 4] ICP22 created new session: [2:7587241385189028022:2107]
2025-12-24T02:11:35.965494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241385189028022:2107] [node 4] ICS09 handshake done sender: [2:7587241385189027998:2108] self: [2:7587241385189027997:0] peer: [4:7587241386053057830:0] socket: 274 qp: -1
2025-12-24T02:11:35.965501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241385189028022:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:35.965527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:35.965737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241387945466947:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:35.966780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385030216303:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:35.966801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:35.966808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385030216303:2049] [node 3] ICP22 created new session: [1:7587241385030216388:2049]
2025-12-24T02:11:35.966814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241385030216388:2049] [node 3] ICS09 handshake done sender: [1:7587241385030216367:2109] self: [1:7587241385030216366:0] peer: [3:7587241387945466960:0] socket: 265 qp: -1
2025-12-24T02:11:35.966820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241385030216388:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:35.966836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:35.970230Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241386054385917:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(47323205ns) {Configure:104(47322187ns) {ProcessConfigured:135(46972855ns) {StartInitialHandshake:157(46971966ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1473ns) {SwitchToState:179(957ns)}}}}}
2025-12-24T02:11:35.970404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241387945466881:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:35.970424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:35.970434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241387945466881:2049] [node 2] ICP22 created new session: [3:7587241387945466964:2049]
2025-12-24T02:11:35.970441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241387945466964:2049] [node 2] ICS09 handshake done sender: [3:7587241387945466947:2111] self: [3:7587241387945466962:0] peer: [2:7587241385189027993:0] socket: 289 qp: -1
2025-12-24T02:11:35.970448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241387945466964:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:35.970465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:35.974900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386054385995:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.974958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386054385916:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.975094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241384805556101:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.978767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386054385992:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.978803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386054385956:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.979096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241385189028006:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.979114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241385189028006:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.982725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216373:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.982748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385030216373:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.982758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216369:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.982769Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385030216369:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.982759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057827:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.982780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241386053057827:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:35.987440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241384805556104:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:35.990768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241384805556074:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.990883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241384805556100:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.990929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241384805556023:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.990847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057837:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.990874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241386053057825:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.990884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241386053057825:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:35.990903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241386053057755:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.991295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241385189027996:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(43710337ns) {HandleHandshakeStatus:329(43709148ns) {DropIncomingHandshake:442(863ns) DropOutgoingHandshake:457(732ns) SwitchToState:179(214ns) ProcessPendingSessionEvents:495(12189170ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12168374ns max# 12081472ns}}}
2025-12-24T02:11:35.991922Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241385030216303:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(25047923ns) {HandleHandshakeStatus:329(25047374ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(126ns) SwitchToState:179(207ns) ProcessPendingSessionEvents:495(12774365ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12756654ns max# 12676137ns}}}
2025-12-24T02:11:35.994853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241384805556104:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.994906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241384805556022:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:35.994834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466951:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:35.994849Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241387945466951:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.006820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241387945466966:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:36.006896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241387945466967:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:36.007545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241385189028006:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.007683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241385189028004:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.007711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.007721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241385189028004:2110] [node 5] ICP22 created new session: [2:7587241389483995321:2110]
2025-12-24T02:11:36.007728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389483995321:2110] [node 5] ICS09 handshake done sender: [2:7587241385189028006:2112] self: [2:7587241385189028005:0] peer: [5:7587241386054385997:0] socket: 306 qp: -1
2025-12-24T02:11:36.007736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389483995321:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:36.007749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.008511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241387945466946:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.008613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241387945466880:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.008630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.008637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241387945466880:2048] [node 1] ICP22 created new session: [3:7587241392240434265:2048]
2025-12-24T02:11:36.008644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241392240434265:2048] [node 1] ICS09 handshake done sender: [3:7587241387945466946:2110] self: [3:7587241387945466960:0] peer: [1:7587241385030216366:0] socket: 272 qp: -1
2025-12-24T02:11:36.008649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241392240434265:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.008686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.009560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241385189027934:2049] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.010709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523403:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.010932Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241386054385916:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5498) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241386054385995:2116] held: no
2025-12-24T02:11:36.011184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385030216373:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.014719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466953:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.014740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241387945466953:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.014781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241385189028014:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.014799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241385189028014:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.015835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353295:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.015896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353296:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.016400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353297:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.016417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353298:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.016164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466966:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.016544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385030216369:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:36.016687Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241387945466951:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:36.016714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385030216364:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.016729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.016734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385030216364:2107] [node 4] ICP22 created new session: [1:7587241389325183689:2107]
2025-12-24T02:11:36.016739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241389325183689:2107] [node 4] ICS09 handshake done sender: [1:7587241385030216369:2110] self: [1:7587241385030216368:0] peer: [4:7587241386053057843:0] socket: 266 qp: -1
2025-12-24T02:11:36.016745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241389325183689:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:36.016753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.016791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241387945466948:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.016808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.016815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241387945466948:2108] [node 4] ICP22 created new session: [3:7587241392240434270:2108]
2025-12-24T02:11:36.016821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241392240434270:2108] [node 4] ICS09 handshake done sender: [3:7587241387945466951:2112] self: [3:7587241387945466950:0] peer: [4:7587241386053057844:0] socket: 290 qp: -1
2025-12-24T02:11:36.016827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241392240434270:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:36.016834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.018756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241384805556099:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.018786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241390349353295:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.018825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386054385915:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.018979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386054385995:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.021660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386053057825:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.021771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385030216365:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.021793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.021800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385030216365:2108] [node 6] ICP22 created new session: [1:7587241389325183692:2108]
2025-12-24T02:11:36.021806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241389325183692:2108] [node 6] ICS09 handshake done sender: [1:7587241385030216373:2111] self: [1:7587241385030216372:0] peer: [6:7587241384805556102:0] socket: 269 qp: -1
2025-12-24T02:11:36.021811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241389325183692:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:36.021821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.022380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241384805556021:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.022399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.022407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241384805556021:2048] [node 1] ICP22 created new session: [6:7587241389100523405:2048]
2025-12-24T02:11:36.022412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241389100523405:2048] [node 1] ICS09 handshake done sender: [6:7587241384805556099:2115] self: [6:7587241384805556102:0] peer: [1:7587241385030216372:0] socket: 273 qp: -1
2025-12-24T02:11:36.022417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241389100523405:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.022435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.019266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386054385916:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.019288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.019298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386054385916:2049] [node 2] ICP22 created new session: [5:7587241390349353300:2049]
2025-12-24T02:11:36.019306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241390349353300:2049] [node 2] ICS09 handshake done sender: [5:7587241386054385995:2116] self: [5:7587241386054385997:0] peer: [2:7587241385189028005:0] socket: 308 qp: -1
2025-12-24T02:11:36.019314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241390349353300:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.019333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.022807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241386053057755:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11843) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241386053057837:2118] held: no
2025-12-24T02:11:36.022926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241386053057821:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.022939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.022945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386053057821:2113] [node 5] ICP22 created new session: [4:7587241390348025142:2113]
2025-12-24T02:11:36.022951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241390348025142:2113] [node 5] ICS09 handshake done sender: [4:7587241386053057825:2114] self: [4:7587241386053057822:0] peer: [5:7587241386054385998:0] socket: 298 qp: -1
2025-12-24T02:11:36.022956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241390348025142:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:36.022972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.022283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241389325183691:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64753
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.026685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523404:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.026835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523407:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.027190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386053057827:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.027314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241386053057817:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.027329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.027336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386053057817:2102] [node 6] ICP22 created new session: [4:7587241390348025146:2102]
2025-12-24T02:11:36.027341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241390348025146:2102] [node 6] ICS09 handshake done sender: [4:7587241386053057827:2115] self: [4:7587241386053057826:0] peer: [6:7587241384805556103:0] socket: 299 qp: -1
2025-12-24T02:11:36.027345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241390348025146:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:36.027350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.026169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386054385992:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:36.026967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216382:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.026979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385030216382:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.035535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241390348025148:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:36.038789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241390348025149:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39261
2025-12-24T02:11:36.038839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385030216382:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.039012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386053057837:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.039076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386053057755:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.039093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.039100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386053057755:2050] [node 3] ICP22 created new session: [4:7587241390348025150:2050]
2025-12-24T02:11:36.039106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241390348025150:2050] [node 3] ICS09 handshake done sender: [4:7587241386053057837:2118] self: [4:7587241386053057844:0] peer: [3:7587241387945466950:0] socket: 293 qp: -1
2025-12-24T02:11:36.039113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241390348025150:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.039122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.038961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385030216376:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.038992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.039001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385030216376:2112] [node 5] ICP22 created new session: [1:7587241389325183696:2112]
2025-12-24T02:11:36.039007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241389325183696:2112] [node 5] ICS09 handshake done sender: [1:7587241385030216382:2116] self: [1:7587241385030216378:0] peer: [5:7587241390349353299:0] socket: 303 qp: -1
2025-12-24T02:11:36.039014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241389325183696:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:36.039041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.043503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241385189028014:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.043664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241386053057812:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.044416Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241385030216364:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(27551898ns) {HandleHandshakeStatus:329(27551320ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(93ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(20381618ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20356438ns max# 20269994ns}}}
2025-12-24T02:11:36.046136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241387945466953:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.046272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241387945466949:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.046291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.046300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241387945466949:2109] [node 6] ICP22 created new session: [3:7587241392240434273:2109]
2025-12-24T02:11:36.046307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241392240434273:2109] [node 6] ICS09 handshake done sender: [3:7587241387945466953:2113] self: [3:7587241387945466952:0] peer: [6:7587241384805556105:0] socket: 291 qp: -1
2025-12-24T02:11:36.046317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241392240434273:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:36.046327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.046857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241386054385915:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241390349353295:2118] held: no
2025-12-24T02:11:36.047030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241385189028009:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.047050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.047058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241385189028009:2113] [node 6] ICP22 created new session: [2:7587241389483995324:2113]
2025-12-24T02:11:36.047065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389483995324:2113] [node 6] ICS09 handshake done sender: [2:7587241385189028014:2117] self: [2:7587241385189028010:0] peer: [6:7587241384805556106:0] socket: 314 qp: -1
2025-12-24T02:11:36.047071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389483995324:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:36.046861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241386053057753:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.046882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.046889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241386053057753:2048] [node 1] ICP22 created new session: [4:7587241390348025152:2048]
2025-12-24T02:11:36.046895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241390348025152:2048] [node 1] ICS09 handshake done sender: [4:7587241386053057812:2106] self: [4:7587241386053057843:0] peer: [1:7587241385030216368:0] socket: 270 qp: -1
2025-12-24T02:11:36.046901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241390348025152:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.046906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.047098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.051277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241384805556074:2096] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23961) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241384805556101:2117] held: no
2025-12-24T02:11:36.050791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386054385956:2087] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.050814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.050821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386054385956:2087] [node 4] ICP22 created new session: [5:7587241390349353304:2087]
2025-12-24T02:11:36.050830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241390349353304:2087] [node 4] ICS09 handshake done sender: [5:7587241386054385992:2114] self: [5:7587241386054385998:0] peer: [4:7587241386053057822:0] socket: 302 qp: -1
2025-12-24T02:11:36.050838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241390349353304:2087] [node 4] ICS10 traffic start
2025-12-24T02:11:36.050846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.054946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389483995326:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.062958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241390349353297:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.063124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353303:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.062829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389483995326:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.062907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241385189027933:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.063198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241389325183698:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64753
2025-12-24T02:11:36.063423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241385030216364:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.066823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241385030216385:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.066832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241385030216385:2117] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.067904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241385030216385:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.067990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241385030216302:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.068015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.068024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241385030216302:2048] [node 2] ICP22 created new session: [1:7587241389325183700:2048]
2025-12-24T02:11:36.068031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241389325183700:2048] [node 2] ICS09 handshake done sender: [1:7587241385030216385:2117] self: [1:7587241385030216377:0] peer: [2:7587241389483995327:0] socket: 319 qp: -1
2025-12-24T02:11:36.068039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241389325183700:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:36.068050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.071133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241384805556101:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.071199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241389100523404:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.074810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241386054385917:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.074864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241390349353295:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.075026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241384805556074:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.075043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.075052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241384805556074:2096] [node 4] ICP22 created new session: [6:7587241389100523412:2096]
2025-12-24T02:11:36.075063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241389100523412:2096] [node 4] ICS09 handshake done sender: [6:7587241384805556101:2117] self: [6:7587241384805556103:0] peer: [4:7587241386053057826:0] socket: 312 qp: -1
2025-12-24T02:11:36.075065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523409:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.075070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241389100523412:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:36.075078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.075165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241384805556072:2089] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.075428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241386054385993:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.075437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241386054385993:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.075235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241384805556100:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.075275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523408:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.077485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241386054385993:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.077576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241386054385977:2090] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.077599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.077606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386054385977:2090] [node 6] ICP22 created new session: [5:7587241390349353309:2090]
2025-12-24T02:11:36.077614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241390349353309:2090] [node 6] ICS09 handshake done sender: [5:7587241386054385993:2115] self: [5:7587241386054385991:0] peer: [6:7587241389100523414:0] socket: 285 qp: -1
2025-12-24T02:11:36.077621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241390349353309:2090] [node 6] ICS10 traffic start
2025-12-24T02:11:36.077630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.070815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389483995328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
2025-12-24T02:11:36.070938Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241385189027933:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61948) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241389483995326:2118] held: no
2025-12-24T02:11:36.071146Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389483995326:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.071354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241385189027933:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.071372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.071380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241385189027933:2048] [node 1] ICP22 created new session: [2:7587241389483995329:2048]
2025-12-24T02:11:36.071387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389483995329:2048] [node 1] ICS09 handshake done sender: [2:7587241389483995326:2118] self: [2:7587241389483995327:0] peer: [1:7587241385030216377:0] socket: 280 qp: -1
2025-12-24T02:11:36.071394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389483995329:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.071405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.078807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386054385915:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.078826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.078834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386054385915:2048] [node 1] ICP22 created new session: [5:7587241390349353311:2048]
2025-12-24T02:11:36.078839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241390349353311:2048] [node 1] ICS09 handshake done sender: [5:7587241390349353295:2118] self: [5:7587241390349353299:0] peer: [1:7587241385030216378:0] socket: 316 qp: -1
2025-12-24T02:11:36.078846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241390349353311:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.078859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.090521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241390349353313:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14944
2025-12-24T02:11:36.090837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241390349353297:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.086732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241387945466961:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.086760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241387945466961:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.087530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241387945466961:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.087608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241387945466941:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.087625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.087636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241387945466941:2107] [node 5] ICP22 created new session: [3:7587241392240434276:2107]
2025-12-24T02:11:36.087644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241392240434276:2107] [node 5] ICS09 handshake done sender: [3:7587241387945466961:2116] self: [3:7587241387945466943:0] peer: [5:7587241390349353307:0] socket: 287 qp: -1
2025-12-24T02:11:36.087651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241392240434276:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:36.087702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.091112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241392240434278:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:36.091139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241392240434279:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17198
2025-12-24T02:11:36.090956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241386054385917:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.090979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.090986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241386054385917:2050] [node 3] ICP22 created new session: [5:7587241390349353314:2050]
2025-12-24T02:11:36.090994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241390349353314:2050] [node 3] ICS09 handshake done sender: [5:7587241390349353297:2120] self: [5:7587241390349353307:0] peer: [3:7587241387945466943:0] socket: 323 qp: -1
2025-12-24T02:11:36.091001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241390349353314:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.091010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.094983Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241387945466941:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.095580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241384805556023:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.095609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.095619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241384805556023:2050] [node 3] ICP22 created new session: [6:7587241389100523415:2050]
2025-12-24T02:11:36.095626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241389100523415:2050] [node 3] ICS09 handshake done sender: [6:7587241384805556100:2116] self: [6:7587241384805556105:0] peer: [3:7587241387945466952:0] socket: 311 qp: -1
2025-12-24T02:11:36.095635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241389100523415:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.095642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.096170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241384805556072:2089] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24634) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241389100523404:2120] held: no
2025-12-24T02:11:36.096537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241384805556104:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.097812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241384805556022:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.097833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.097839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241384805556022:2049] [node 2] ICP22 created new session: [6:7587241389100523418:2049]
2025-12-24T02:11:36.097847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241389100523418:2049] [node 2] ICS09 handshake done sender: [6:7587241384805556104:2118] self: [6:7587241384805556106:0] peer: [2:7587241385189028010:0] socket: 315 qp: -1
2025-12-24T02:11:36.097853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241389100523418:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.097859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.098804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241389100523417:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20833
2025-12-24T02:11:36.099099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241389100523404:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.099450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241384805556072:2089] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.099461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.099468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241384805556072:2089] [node 5] ICP22 created new session: [6:7587241389100523420:2089]
2025-12-24T02:11:36.099476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241389100523420:2089] [node 5] ICS09 handshake done sender: [6:7587241389100523404:2120] self: [6:7587241389100523414:0] peer: [5:7587241386054385991:0] socket: 326 qp: -1
2025-12-24T02:11:36.099482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241389100523420:2089] [node 5] ICS10 traffic start
2025-12-24T02:11:36.099493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.098963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389483995331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31253
2025-12-24T02:11:36.099270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241385189028009:2113] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.130857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:36.130971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241389483995329:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-12-24T02:11:36.130982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241385189027933:2048] [node 1] ICP30 unregister session Session# [2:7587241389483995329:2048] VirtualId# [2:7587241389483995327:0]
2025-12-24T02:11:36.130991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241389483995329:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.131054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.131082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.131155Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:36.131239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241392240434265:2048] [node 1] ICS01 socket: 272 reason# EndOfStream
2025-12-24T02:11:36.131247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241387945466880:2048] [node 1] ICP30 unregister session Session# [3:7587241392240434265:2048] VirtualId# [3:7587241387945466960:0]
2025-12-24T02:11:36.131256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241392240434265:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.131288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.131308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.138964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:36.139074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241390349353311:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-12-24T02:11:36.139085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386054385915:2048] [node 1] ICP30 unregister session Session# [5:7587241390349353311:2048] VirtualId# [5:7587241390349353299:0]
2025-12-24T02:11:36.139092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241390349353311:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.139151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.139176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.142876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:36.142969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241390348025152:2048] [node 1] ICS01 socket: 270 reason# EndOfStream
2025-12-24T02:11:36.142978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241386053057753:2048] [node 1] ICP30 unregister session Session# [4:7587241390348025152:2048] VirtualId# [4:7587241386053057843:0]
2025-12-24T02:11:36.142984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241390348025152:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.143037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.143060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.146753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:36.146856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241389100523405:2048] [node 1] ICS01 socket: 273 reason# EndOfStream
2025-12-24T02:11:36.146865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241384805556021:2048] [node 1] ICP30 unregister session Session# [6:7587241389100523405:2048] VirtualId# [6:7587241384805556102:0]
2025-12-24T02:11:36.146872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241389100523405:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.146928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.146955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.165962Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:36.166077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241387945466964:2049] [node 2] ICS01 socket: 289 reason# EndOfStream
2025-12-24T02:11:36.166089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241387945466881:2049] [node 2] ICP30 unregister session Session# [3:7587241387945466964:2049] VirtualId# [3:7587241387945466962:0]
2025-12-24T02:11:36.166098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241387945466964:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.166148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.166171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.174767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:36.174867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241389100523418:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-12-24T02:11:36.174877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241384805556022:2049] [node 2] ICP30 unregister session Session# [6:7587241389100523418:2049] VirtualId# [6:7587241384805556106:0]
2025-12-24T02:11:36.174885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241389100523418:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.174937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.174962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.178758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:36.178872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241386053057839:2049] [node 2] ICS01 socket: 279 reason# EndOfStream
2025-12-24T02:11:36.178883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241386053057754:2049] [node 2] ICP30 unregister session Session# [4:7587241386053057839:2049] VirtualId# [4:7587241386053057830:0]
2025-12-24T02:11:36.178891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241386053057839:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.178949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.178976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.179013Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:36.179087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241390349353300:2049] [node 2] ICS01 socket: 308 reason# EndOfStream
2025-12-24T02:11:36.179092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386054385916:2049] [node 2] ICP30 unregister session Session# [5:7587241390349353300:2049] VirtualId# [5:7587241386054385997:0]
2025-12-24T02:11:36.179096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241390349353300:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.179126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.179147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.210838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:36.210962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241389100523415:2050] [node 3] ICS01 socket: 311 reason# EndOfStream
2025-12-24T02:11:36.210975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241384805556023:2050] [node 3] ICP30 unregister session Session# [6:7587241389100523415:2050] VirtualId# [6:7587241384805556105:0]
2025-12-24T02:11:36.210984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241389100523415:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.211046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:36.211076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:36.214860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:36.214950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241390349353314:2050] [node 3] ICS01 socket: 323 reason# EndOfStream
2025-12-24T02:11:36.214958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241386054385917:2050] [node 3] ICP30 unregister session Session# [5:7587241390349353314:2050] VirtualId# [5:7587241390349353307:0]
2025-12-24T02:11:36.214965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241390349353314:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.215020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:36.215043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:36.246785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:36.246915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241389100523412:2096] [node 4] ICS01 socket: 312 reason# EndOfStream
2025-12-24T02:11:36.246928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241384805556074:2096] [node 4] ICP30 unregister session Session# [6:7587241389100523412:2096] VirtualId# [6:7587241384805556103:0]
2025-12-24T02:11:36.246937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241389100523412:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.246990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:36.247014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:36.330824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241388335885754:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.330834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241388335885755:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.330813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241392230509670:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:36.338764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241392230509671:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:36.334756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389049727930:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.334772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389049727931:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:36.350805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241391866113936:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.350815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241391866113937:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.350821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241391866113938:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:36.361081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241388892045542:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.361096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241388892045543:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.361102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241388892045544:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:36.370800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241392188886858:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.370812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241392188886859:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.370818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241392188886860:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.407275Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885818:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.407217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241388892045569:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:36.406974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241391866113994:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:36.407315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241391866113995:2098] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.411730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866113999:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.415974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866113999:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.418912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241388892045598:2083] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.419855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509736:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.419251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389049727993:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:36.419821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241388335885823:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:36.420248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885824:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.420561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241392230509737:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:36.421431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241388335885828:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.425401Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241391866113936:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18579114ns) {Configure:104(18578107ns) {ProcessConfigured:135(16863947ns) {StartInitialHandshake:157(16863255ns) {DropHandshakes:470(374ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(1684ns) {SwitchToState:179(834ns)}}}}}
2025-12-24T02:11:36.425150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886906:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.425201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886907:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.429543Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241391866113995:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(22160134ns) {Configure:104(22159671ns) {ProcessConfigured:135(20873590ns) {StartInitialHandshake:157(20873037ns) {DropHandshakes:470(239ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1280ns) {SwitchToState:179(818ns)}}}}}
2025-12-24T02:11:36.426946Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389049728001:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34820
2025-12-24T02:11:36.430245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241392230509741:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:36.430798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509735:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-12-24T02:11:36.430895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509742:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-12-24T02:11:36.430057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389049728003:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:36.431063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885818:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.431273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509735:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.430781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241392188886886:2063] [node 5] ICP01 ready to work
2025-12-24T02:11:36.434908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241392230509671:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20125899ns) {Configure:104(20124757ns) {ProcessConfigured:135(19755449ns) {StartInitialHandshake:157(19754741ns) {DropHandshakes:470(1002ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1062ns) {SwitchToState:179(618ns)}}}}}
2025-12-24T02:11:36.438075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509736:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.438101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509742:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.439190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241392230509670:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.437812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241389049727931:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22794573ns) {Configure:104(22793654ns) {ProcessConfigured:135(22451697ns) {StartInitialHandshake:157(22451170ns) {DropHandshakes:470(494ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(1361ns) {SwitchToState:179(830ns)}}}}}
2025-12-24T02:11:36.439691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049727994:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.439697Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389049727994:2108] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.440562Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241388892045543:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(33305294ns) {Configure:104(33304351ns) {ProcessConfigured:135(24746561ns) {StartInitialHandshake:157(24745844ns) {DropHandshakes:470(552ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(1176ns) {SwitchToState:179(938ns)}}}}}
2025-12-24T02:11:36.440779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241388892045542:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33856411ns) {Configure:104(33855230ns) {ProcessConfigured:135(24927668ns) {StartInitialHandshake:157(24927211ns) {DropHandshakes:470(354ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(765ns) {SwitchToState:179(264ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.441397Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509748:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-12-24T02:11:36.441523Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389049727994:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.441705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389049727930:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.441724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.441731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389049727930:2048] [node 1] ICP22 created new session: [2:7587241389049728009:2048]
2025-12-24T02:11:36.441737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389049728009:2048] [node 1] ICS09 handshake done sender: [2:7587241389049727994:2108] self: [2:7587241389049727991:0] peer: [1:7587241392230509747:0] socket: 280 qp: -1
2025-12-24T02:11:36.441743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389049728009:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.441762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.441927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389049728008:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34820
2025-12-24T02:11:36.442032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241392230509736:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.442293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885824:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.442144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241392230509670:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.442155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.442159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241392230509670:2048] [node 2] ICP22 created new session: [1:7587241392230509749:2048]
2025-12-24T02:11:36.442164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241392230509749:2048] [node 2] ICS09 handshake done sender: [1:7587241392230509736:2111] self: [1:7587241392230509747:0] peer: [2:7587241389049727991:0] socket: 282 qp: -1
2025-12-24T02:11:36.442169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241392230509749:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:36.442197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:36.442649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728008:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.442774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241388892045598:2083] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23779796ns) {Configure:104(23779412ns) {ProcessConfigured:135(23374322ns) {StartInitialHandshake:157(23374079ns) {DropHandshakes:470(507ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(985ns) {SwitchToState:179(541ns)}}}}}
2025-12-24T02:11:36.445895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389049728000:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:36.445840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241388335885821:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:36.443068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241392188886905:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:36.443117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241392188886907:2091] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.446745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241392230509734:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:36.450809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241391866113994:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(43757375ns) {Configure:104(43756830ns) {ProcessConfigured:135(42057441ns) {StartInitialHandshake:157(42056944ns) {DropHandshakes:470(191ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(88ns) PrepareNewSessionHandshake:477(622ns) {SwitchToState:179(375ns)}}}}}
2025-12-24T02:11:36.451163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241392230509741:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20857393ns) {Configure:104(20856958ns) {ProcessConfigured:135(20509933ns) {StartInitialHandshake:157(20509597ns) {DropHandshakes:470(489ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:477(572ns) {SwitchToState:179(194ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.453069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045622:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.453088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045623:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.453135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045620:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.453175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045621:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.451327Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241392188886858:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(44225534ns) {Configure:104(44224865ns) {ProcessConfigured:135(43865811ns) {StartInitialHandshake:157(43865537ns) {DropHandshakes:470(682ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(92ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(622ns) {SwitchToState:179(55ns)}}}}}
2025-12-24T02:11:36.451357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241392188886860:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.450945Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241389049728003:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20832562ns) {Configure:104(20831972ns) {ProcessConfigured:135(20504341ns) {StartInitialHandshake:157(20503757ns) {DropHandshakes:470(515ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:477(1574ns) {SwitchToState:179(766ns)}}}}}
2025-12-24T02:11:36.453264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885830:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.453275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241388335885830:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.453891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241388335885830:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.453968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241388335885828:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.453983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.453989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241388335885828:2111] [node 6] ICP22 created new session: [3:7587241388335885836:2111]
2025-12-24T02:11:36.453996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241388335885836:2111] [node 6] ICS09 handshake done sender: [3:7587241388335885830:2114] self: [3:7587241388335885829:0] peer: [6:7587241392188886935:0] socket: 287 qp: -1
2025-12-24T02:11:36.454002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241388335885836:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:36.454011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.454837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241392188886886:2063] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23920311ns) {Configure:104(23919730ns) {ProcessConfigured:135(23572106ns) {StartInitialHandshake:157(23571773ns) {DropHandshakes:470(278ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(570ns) {SwitchToState:179(292ns)}}}}}
2025-12-24T02:11:36.455624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241392188886860:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61831) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241392188886907:2091] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.454792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045623:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.454803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045620:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.454825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241388892045544:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.454827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241388892045543:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.458817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045621:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.458874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241388892045542:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.456233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241392188886906:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.456297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241392188886859:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.459185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045622:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.459221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241388892045569:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.459015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728012:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.459027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389049728012:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.459885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728001:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.462903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509754:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-12-24T02:11:36.466640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509754:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.466800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241391866113937:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23924023ns) {Configure:104(23923193ns) {ProcessConfigured:135(23573303ns) {StartInitialHandshake:157(23572400ns) {DropHandshakes:470(368ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1032ns) {SwitchToState:179(716ns)}}}}}
2025-12-24T02:11:36.466939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241392188886905:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23719590ns) {Configure:104(23719086ns) {ProcessConfigured:135(23421988ns) {StartInitialHandshake:157(23421907ns) {DropHandshakes:470(438ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(245ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.470883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885838:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.470945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885839:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.471646Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241389049728000:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25670966ns) {Configure:104(25670311ns) {ProcessConfigured:135(25333308ns) {StartInitialHandshake:157(25333080ns) {DropHandshakes:470(440ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(955ns) {SwitchToState:179(119ns)}}}}}
2025-12-24T02:11:36.471793Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241392230509734:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24968272ns) {Configure:104(24967860ns) {ProcessConfigured:135(24632747ns) {StartInitialHandshake:157(24616610ns) {DropHandshakes:470(392ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(290ns) PrepareNewSessionHandshake:477(1250ns) {SwitchToState:179(24ns)}} IncomingHandshake:261(5378ns) {DropIncomingHandshake:442(305ns) PrepareNewSessionHandshake:477(202ns) {SwitchToState:179(158ns)} GenerateSessionVirtualId:988(200ns) IssueIncomingHandshakeReply:193(587ns)}}}}
2025-12-24T02:11:36.471964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509739:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.471972Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241392230509739:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.474724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114011:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.474750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241391866114011:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.471403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241388335885821:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25456013ns) {Configure:104(25454824ns) {ProcessConfigured:135(25104682ns) {StartInitialHandshake:157(25104207ns) {DropHandshakes:470(428ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:477(1395ns) {SwitchToState:179(870ns)}}}}}
2025-12-24T02:11:36.474841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885826:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.474851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241388335885826:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.475774Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.475883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885838:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.475924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885839:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.475960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241388335885754:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.476106Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241388335885755:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(68434639ns) {Configure:104(57096312ns) {ProcessConfigured:135(56663926ns) {StartInitialHandshake:157(56663447ns) {DropHandshakes:470(307ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(874ns) {SwitchToState:179(495ns)}}}}}
2025-12-24T02:11:36.476167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241388335885755:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.474769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049727997:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.474781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389049727997:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.482902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728005:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.482913Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389049728005:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.482593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045628:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.482663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045629:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.482989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045628:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.486793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389049728012:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.486971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389049728003:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.488080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241392230509739:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.488392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241392230509737:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.488404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.488411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241392230509737:2106] [node 5] ICP22 created new session: [1:7587241392230509759:2106]
2025-12-24T02:11:36.488418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241392230509759:2106] [node 5] ICS09 handshake done sender: [1:7587241392230509739:2112] self: [1:7587241392230509738:0] peer: [5:7587241388892045626:0] socket: 286 qp: -1
2025-12-24T02:11:36.488426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241392230509759:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:36.488434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.490799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241388892045623:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.487795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886941:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.486995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.487007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389049728003:2112] [node 5] ICP22 created new session: [2:7587241389049728018:2112]
2025-12-24T02:11:36.487014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389049728018:2112] [node 5] ICS09 handshake done sender: [2:7587241389049728012:2115] self: [2:7587241389049728004:0] peer: [5:7587241388892045625:0] socket: 307 qp: -1
2025-12-24T02:11:36.487021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389049728018:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:36.487033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.487871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886942:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.487918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886943:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.490777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241392188886942:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.490833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241392188886858:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.491725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241388892045543:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.491752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.491759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241388892045543:2049] [node 2] ICP22 created new session: [5:7587241388892045631:2049]
2025-12-24T02:11:36.491766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241388892045631:2049] [node 2] ICS09 handshake done sender: [5:7587241388892045623:2118] self: [5:7587241388892045625:0] peer: [2:7587241389049728004:0] socket: 312 qp: -1
2025-12-24T02:11:36.491772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241388892045631:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.491793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.491502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241392188886941:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.491530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241392188886907:2091] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.491571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241392188886905:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.492640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509752:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.492651Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241392230509752:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.493304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241392230509752:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.493365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241392230509741:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.493379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.493384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241392230509741:2108] [node 6] ICP22 created new session: [1:7587241392230509763:2108]
2025-12-24T02:11:36.493390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241392230509763:2108] [node 6] ICS09 handshake done sender: [1:7587241392230509752:2116] self: [1:7587241392230509743:0] peer: [6:7587241392188886944:0] socket: 308 qp: -1
2025-12-24T02:11:36.493396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241392230509763:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:36.493402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.494109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241392230509765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39945
2025-12-24T02:11:36.494956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241392188886860:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.494976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.494984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241392188886860:2050] [node 3] ICP22 created new session: [6:7587241392188886946:2050]
2025-12-24T02:11:36.494992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241392188886946:2050] [node 3] ICS09 handshake done sender: [6:7587241392188886907:2091] self: [6:7587241392188886935:0] peer: [3:7587241388335885829:0] socket: 291 qp: -1
2025-12-24T02:11:36.494997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241392188886946:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.495018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.498734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241392188886858:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2460) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241392188886942:2118] held: no
2025-12-24T02:11:36.498788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509744:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.498808Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241392230509744:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.498797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241391866114011:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.498861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114003:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.498870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241391866114003:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.499184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389049727997:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.499387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389049727993:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.499406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.499412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389049727993:2107] [node 6] ICP22 created new session: [2:7587241389049728022:2107]
2025-12-24T02:11:36.499418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389049728022:2107] [node 6] ICS09 handshake done sender: [2:7587241389049727997:2109] self: [2:7587241389049727996:0] peer: [6:7587241392188886938:0] socket: 281 qp: -1
2025-12-24T02:11:36.499423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389049728022:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:36.499430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.499641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241391866113994:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.499674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.499680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241391866113994:2101] [node 5] ICP22 created new session: [4:7587241391866114018:2101]
2025-12-24T02:11:36.499686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241391866114018:2101] [node 5] ICS09 handshake done sender: [4:7587241391866114011:2114] self: [4:7587241391866113998:0] peer: [5:7587241388892045627:0] socket: 306 qp: -1
2025-12-24T02:11:36.499691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241391866114018:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:36.499700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.500999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389049728024:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34820
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.498932Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241388335885826:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.499254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241388335885823:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.499276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.499286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241388335885823:2110] [node 5] ICP22 created new session: [3:7587241388335885846:2110]
2025-12-24T02:11:36.499294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241388335885846:2110] [node 5] ICS09 handshake done sender: [3:7587241388335885826:2113] self: [3:7587241388335885825:0] peer: [5:7587241388892045624:0] socket: 285 qp: -1
2025-12-24T02:11:36.499302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241388335885846:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:36.499312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.505101Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241392188886859:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1160) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241392188886906:2090] held: no
2025-12-24T02:11:36.505851Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241388892045569:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27021) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241388892045622:2117] held: no
2025-12-24T02:11:36.503041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509765:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.503156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241392230509737:2106] [node 5] ICP18 handshake (actor [1:7587241392230509765:2119]) is too old
2025-12-24T02:11:36.502534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389049728025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34820
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.506821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728025:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.506879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389049728005:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.506996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241389049728003:2112] [node 5] ICP35 event processing took too much time PendingConnection:279(19962791ns) {HandleHandshakeStatus:329(19961892ns) {DropIncomingHandshake:442(6836ns) DropOutgoingHandshake:457(355ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(19852260ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 19839283ns max# 19601597ns}}}
2025-12-24T02:11:36.507449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114022:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.507503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114023:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.510690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885849:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.511009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241388335885839:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.510736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045635:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.510842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045636:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.510977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114022:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.511017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114023:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.511045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241391866113936:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.511052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241391866113937:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.511196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241388335885755:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.511212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.511218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241388335885755:2049] [node 2] ICP22 created new session: [3:7587241388335885850:2049]
2025-12-24T02:11:36.511225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241388335885850:2049] [node 2] ICS09 handshake done sender: [3:7587241388335885839:2116] self: [3:7587241388335885844:0] peer: [2:7587241389049727992:0] socket: 315 qp: -1
2025-12-24T02:11:36.511232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241388335885850:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.511239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:36.511422Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241388892045620:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.511590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241388892045622:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.511673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241388892045569:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.511690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.511697Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241388892045569:2060] [node 4] ICP22 created new session: [5:7587241388892045637:2060]
2025-12-24T02:11:36.511703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241388892045637:2060] [node 4] ICS09 handshake done sender: [5:7587241388892045622:2117] self: [5:7587241388892045627:0] peer: [4:7587241391866113998:0] socket: 311 qp: -1
2025-12-24T02:11:36.511710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241388892045637:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:36.511718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.515088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241388892045634:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44393
2025-12-24T02:11:36.518081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241388892045544:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.518100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.518107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241388892045544:2050] [node 3] ICP22 created new session: [5:7587241388892045639:2050]
2025-12-24T02:11:36.518115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241388892045639:2050] [node 3] ICS09 handshake done sender: [5:7587241388892045620:2115] self: [5:7587241388892045624:0] peer: [3:7587241388335885825:0] socket: 288 qp: -1
2025-12-24T02:11:36.518121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241388892045639:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.518131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.518714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886948:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.518721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389049728016:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.518739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389049728016:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.518841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241392188886942:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.520432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241388892045621:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.522399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885852:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.522917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241392230509744:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.523070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241392188886858:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.523094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.523103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241392188886858:2048] [node 1] ICP22 created new session: [6:7587241392188886949:2048]
2025-12-24T02:11:36.523110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241392188886949:2048] [node 1] ICS09 handshake done sender: [6:7587241392188886942:2118] self: [6:7587241392188886944:0] peer: [1:7587241392230509743:0] socket: 321 qp: -1
2025-12-24T02:11:36.523117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241392188886949:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.523126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.524359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241392230509671:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.524378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.524385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241392230509671:2049] [node 3] ICP22 created new session: [1:7587241392230509767:2049]
2025-12-24T02:11:36.524391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241392230509767:2049] [node 3] ICS09 handshake done sender: [1:7587241392230509744:2114] self: [1:7587241392230509732:0] peer: [3:7587241388335885842:0] socket: 299 qp: -1
2025-12-24T02:11:36.524397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241392230509767:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:36.524403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.522989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241391866114003:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.523135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114016:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.523145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241391866113995:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.523156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.523163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241391866113995:2098] [node 6] ICP22 created new session: [4:7587241391866114027:2098]
2025-12-24T02:11:36.523169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241391866114027:2098] [node 6] ICS09 handshake done sender: [4:7587241391866114003:2108] self: [4:7587241391866113997:0] peer: [6:7587241392188886945:0] socket: 297 qp: -1
2025-12-24T02:11:36.523175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241391866114027:2098] [node 6] ICS10 traffic start
2025-12-24T02:11:36.523183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.524163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114030:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.527203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389049727931:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.527224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.527233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389049727931:2049] [node 3] ICP22 created new session: [2:7587241389049728028:2049]
2025-12-24T02:11:36.527241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389049728028:2049] [node 3] ICS09 handshake done sender: [2:7587241389049728005:2113] self: [2:7587241389049727992:0] peer: [3:7587241388335885844:0] socket: 300 qp: -1
2025-12-24T02:11:36.527246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389049728028:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:36.527272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.527414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241392230509756:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.527428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241392230509756:2118] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.531288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241388335885840:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.531300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241388335885840:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.532309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241388335885838:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.530735Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241391866113938:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51001206ns) {Configure:104(50999875ns) {ProcessConfigured:135(50712424ns) {StartInitialHandshake:157(50711642ns) {DropHandshakes:470(400ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(576ns)}}}}}
2025-12-24T02:11:36.530847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241391866113938:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.534725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114032:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.534915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241388335885840:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.535322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389049728016:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:36.535530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241388892045542:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.535550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.535556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241388892045542:2048] [node 1] ICP22 created new session: [5:7587241388892045641:2048]
2025-12-24T02:11:36.535563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241388892045641:2048] [node 1] ICS09 handshake done sender: [5:7587241388892045621:2116] self: [5:7587241388892045626:0] peer: [1:7587241392230509738:0] socket: 310 qp: -1
2025-12-24T02:11:36.535569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241388892045641:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.535576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.535740Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241388892045542:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.535183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241391866114016:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.535296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114033:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.535441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241391866114023:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.535526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241391866113937:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.535539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.535545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241391866113937:2049] [node 2] ICP22 created new session: [4:7587241391866114034:2049]
2025-12-24T02:11:36.535551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241391866114034:2049] [node 2] ICS09 handshake done sender: [4:7587241391866114023:2119] self: [4:7587241391866114025:0] peer: [2:7587241389049728011:0] socket: 336 qp: -1
2025-12-24T02:11:36.535557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241391866114034:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.535576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:36.536338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241391866113938:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.536348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:36.536353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241391866113938:2050] [node 3] ICP22 created new session: [4:7587241391866114036:2050]
2025-12-24T02:11:36.536359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241391866114036:2050] [node 3] ICS09 handshake done sender: [4:7587241391866114016:2116] self: [4:7587241391866114031:0] peer: [3:7587241388335885834:0] socket: 317 qp: -1
2025-12-24T02:11:36.536365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241391866114036:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:36.536372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:36.543191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241388335885821:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.543213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.543224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241388335885821:2108] [node 4] ICP22 created new session: [3:7587241388335885854:2108]
2025-12-24T02:11:36.543231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241388335885854:2108] [node 4] ICS09 handshake done sender: [3:7587241388335885840:2117] self: [3:7587241388335885834:0] peer: [4:7587241391866114031:0] socket: 316 qp: -1
2025-12-24T02:11:36.543239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241388335885854:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:36.543251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.543584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241388335885754:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.543592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.543596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241388335885754:2048] [node 1] ICP22 created new session: [3:7587241388335885856:2048]
2025-12-24T02:11:36.543602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241388335885856:2048] [node 1] ICS09 handshake done sender: [3:7587241388335885838:2115] self: [3:7587241388335885842:0] peer: [1:7587241392230509732:0] socket: 301 qp: -1
2025-12-24T02:11:36.543607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241388335885856:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.543612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.550883Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241388335885754:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.555158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241392230509756:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.555279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241392230509734:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.555305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.555313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241392230509734:2109] [node 4] ICP22 created new session: [1:7587241392230509770:2109]
2025-12-24T02:11:36.555319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241392230509770:2109] [node 4] ICS09 handshake done sender: [1:7587241392230509756:2118] self: [1:7587241392230509751:0] peer: [4:7587241391866114024:0] socket: 323 qp: -1
2025-12-24T02:11:36.555326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241392230509770:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:36.555333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.555307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886952:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.555464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886953:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.555499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886954:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.556283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241392188886954:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.556311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241392188886941:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:36.556384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241392188886886:2063] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.556490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241388892045619:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.556501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241388892045619:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.557351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241388892045619:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.557615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241388892045598:2083] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.557638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.557646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241388892045598:2083] [node 6] ICP22 created new session: [5:7587241388892045645:2083]
2025-12-24T02:11:36.557653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241388892045645:2083] [node 6] ICS09 handshake done sender: [5:7587241388892045619:2114] self: [5:7587241388892045599:0] peer: [6:7587241392188886955:0] socket: 279 qp: -1
2025-12-24T02:11:36.557659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241388892045645:2083] [node 6] ICS10 traffic start
2025-12-24T02:11:36.557666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.558229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241392188886956:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63510
2025-12-24T02:11:36.555811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241391866114030:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.555872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241391866113995:2098] [node 6] ICP18 handshake (actor [4:7587241391866114030:2120]) is too old
2025-12-24T02:11:36.556092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241391866114038:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36201
2025-12-24T02:11:36.557508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241391866114022:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:36.558318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241392188886905:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.558333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.558338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241392188886905:2070] [node 4] ICP22 created new session: [6:7587241392188886957:2070]
2025-12-24T02:11:36.558343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241392188886957:2070] [node 4] ICS09 handshake done sender: [6:7587241392188886941:2117] self: [6:7587241392188886945:0] peer: [4:7587241391866113997:0] socket: 298 qp: -1
2025-12-24T02:11:36.558349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241392188886957:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:36.558356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.558769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389049728000:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.558809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:36.558818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389049728000:2110] [node 4] ICP22 created new session: [2:7587241389049728031:2110]
2025-12-24T02:11:36.558824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389049728031:2110] [node 4] ICS09 handshake done sender: [2:7587241389049728016:2116] self: [2:7587241389049728011:0] peer: [4:7587241391866114025:0] socket: 330 qp: -1
2025-12-24T02:11:36.558831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389049728031:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:36.558845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:36.559349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241392188886954:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.559063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241391866113936:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.559082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:36.559087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241391866113936:2048] [node 1] ICP22 created new session: [4:7587241391866114039:2048]
2025-12-24T02:11:36.559092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241391866114039:2048] [node 1] ICS09 handshake done sender: [4:7587241391866114022:2118] self: [4:7587241391866114024:0] peer: [1:7587241392230509751:0] socket: 325 qp: -1
2025-12-24T02:11:36.559097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241391866114039:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:36.559102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:36.562727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241392188886906:2090] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.563495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241392188886886:2063] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.563510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:36.563518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241392188886886:2063] [node 5] ICP22 created new session: [6:7587241392188886959:2063]
2025-12-24T02:11:36.563525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241392188886959:2063] [node 5] ICS09 handshake done sender: [6:7587241392188886954:2123] self: [6:7587241392188886955:0] peer: [5:7587241388892045599:0] socket: 276 qp: -1
2025-12-24T02:11:36.563532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241392188886959:2063] [node 5] ICS10 traffic start
2025-12-24T02:11:36.563539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:36.563921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241392188886859:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.563927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.563933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241392188886859:2049] [node 2] ICP22 created new session: [6:7587241392188886961:2049]
2025-12-24T02:11:36.563937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241392188886961:2049] [node 2] ICS09 handshake done sender: [6:7587241392188886906:2090] self: [6:7587241392188886938:0] peer: [2:7587241389049727996:0] socket: 290 qp: -1
2025-12-24T02:11:36.563941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241392188886961:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.563947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.566869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241388335885858:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34801
2025-12-24T02:11:36.566977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241388335885821:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.565517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241392188886886:2063] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.571182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241389049728000:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(35637690ns) {HandleHandshakeStatus:329(35636898ns) {DropIncomingHandshake:442(15426ns) DropOutgoingHandshake:457(149ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(12175326ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 12157067ns max# 12077934ns}}}
2025-12-24T02:11:36.575133Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241389049727931:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(67988121ns) {HandleHandshakeStatus:329(67987513ns) {DropIncomingHandshake:442(385ns) DropOutgoingHandshake:457(387ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(27718571ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27693287ns max# 27595999ns}}}
2025-12-24T02:11:36.579589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241391866113936:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20415543ns) {HandleHandshakeStatus:329(20414903ns) {DropIncomingHandshake:442(164ns) DropOutgoingHandshake:457(1349ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(20340301ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 20321963ns max# 20247479ns}}}
2025-12-24T02:11:36.603728Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241389049727931:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:36.608434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:36.608533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241389049728009:2048] [node 1] ICS01 socket: 280 reason# EndOfStream
2025-12-24T02:11:36.608544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241389049727930:2048] [node 1] ICP30 unregister session Session# [2:7587241389049728009:2048] VirtualId# [2:7587241389049727991:0]
2025-12-24T02:11:36.608552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241389049728009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.608591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.608614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.609313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:36.609385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241388335885856:2048] [node 1] ICS01 socket: 301 reason# EndOfStream
2025-12-24T02:11:36.609391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241388335885754:2048] [node 1] ICP30 unregister session Session# [3:7587241388335885856:2048] VirtualId# [3:7587241388335885842:0]
2025-12-24T02:11:36.609396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241388335885856:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.609422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.609438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.618770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:36.618752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:36.618852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241392188886949:2048] [node 1] ICS01 socket: 321 reason# EndOfStream
2025-12-24T02:11:36.618860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241392188886858:2048] [node 1] ICP30 unregister session Session# [6:7587241392188886949:2048] VirtualId# [6:7587241392188886944:0]
2025-12-24T02:11:36.618867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241392188886949:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.618914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.618941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.622295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241391866114039:2048] [node 1] ICS07 socket disconnect 325 reason# EndOfStream
2025-12-24T02:11:36.622309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241391866114039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.622358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.622402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241391866114039:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:36.618817Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:36.618905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241388892045641:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-12-24T02:11:36.618913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241388892045542:2048] [node 1] ICP30 unregister session Session# [5:7587241388892045641:2048] VirtualId# [5:7587241388892045626:0]
2025-12-24T02:11:36.618920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241388892045641:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.618981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:36.619008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.622964Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241391866113936:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2460) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:36.622976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241391866114039:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:36.622981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241391866113936:2048] [node 1] ICP30 unregister session Session# [4:7587241391866114039:2048] VirtualId# [4:7587241391866114024:0]
2025-12-24T02:11:36.623036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:36.623042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241391866113936:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.623046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2460) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.631376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:36.631470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241388335885850:2049] [node 2] ICS01 socket: 315 reason# EndOfStream
2025-12-24T02:11:36.631478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241388335885755:2049] [node 2] ICP30 unregister session Session# [3:7587241388335885850:2049] VirtualId# [3:7587241388335885844:0]
2025-12-24T02:11:36.631484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241388335885850:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.631520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.631538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.634762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241391866113936:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:36.641787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:36.641877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241388892045631:2049] [node 2] ICS01 socket: 312 reason# EndOfStream
2025-12-24T02:11:36.641886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241388892045543:2049] [node 2] ICP30 unregister session Session# [5:7587241388892045631:2049] VirtualId# [5:7587241388892045625:0]
2025-12-24T02:11:36.641892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241388892045631:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.641942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.641966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.642737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:36.642815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241392188886961:2049] [node 2] ICS01 socket: 290 reason# EndOfStream
2025-12-24T02:11:36.642822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241392188886859:2049] [node 2] ICP30 unregister session Session# [6:7587241392188886961:2049] VirtualId# [6:7587241392188886938:0]
2025-12-24T02:11:36.642827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241392188886961:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.642879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.642902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.642931Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:36.643006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241391866114034:2049] [node 2] ICS07 socket disconnect 336 reason# EndOfStream
2025-12-24T02:11:36.643012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241391866114034:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.643060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:36.643081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241391866114034:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:36.643517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241391866113937:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1160) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:36.643528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241391866114034:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:36.643533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241391866113937:2049] [node 2] ICP30 unregister session Session# [4:7587241391866114034:2049] VirtualId# [4:7587241391866114025:0]
2025-12-24T02:11:36.643558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:36.643564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241391866113937:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1160) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.643567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1160) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.654826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241391866113937:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:36.662785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:36.662894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241392188886946:2050] [node 3] ICS01 socket: 291 reason# EndOfStream
2025-12-24T02:11:36.662904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241392188886860:2050] [node 3] ICP30 unregister session Session# [6:7587241392188886946:2050] VirtualId# [6:7587241392188886935:0]
2025-12-24T02:11:36.662913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241392188886946:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.662963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:36.662988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:36.662854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:36.662936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241388892045639:2050] [node 3] ICS07 socket disconnect 288 reason# EndOfStream
2025-12-24T02:11:36.662943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241388892045639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.662989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:36.663010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241388892045639:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:36.666941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241388892045544:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61831) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:36.666961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241388892045639:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:36.666967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241388892045544:2050] [node 3] ICP30 unregister session Session# [5:7587241388892045639:2050] VirtualId# [5:7587241388892045624:0]
2025-12-24T02:11:36.667027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:36.667034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241388892045544:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.667037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61831) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:36.678755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241388892045544:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:36.682766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:36.682856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241392188886957:2070] [node 4] ICS01 socket: 298 reason# EndOfStream
2025-12-24T02:11:36.682865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241392188886905:2070] [node 4] ICP30 unregister session Session# [6:7587241392188886957:2070] VirtualId# [6:7587241392188886945:0]
2025-12-24T02:11:36.682872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241392188886957:2070] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:36.682911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:36.682935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:36.770941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241390609086128:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:36.770958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241390609086129:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:36.772118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241391242343068:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.772129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241391242343069:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.786878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241391504770416:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.786896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241391504770417:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.786902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241391504770418:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:36.786894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389894450893:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.786922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389894450894:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:36.802811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241389902856170:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.802831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241389902856171:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.802837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241389902856172:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:36.806856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241388736502461:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:36.806873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241388736502462:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:36.806880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241388736502463:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.891722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389894450953:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51539
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.895858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241391504770477:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:36.895056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241388736502516:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:36.902928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241391242343129:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:36.907597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241390609086194:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:36.911026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389894450955:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:36.927023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389894450961:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:36.927128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241389894450962:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:36.930847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241390609086194:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23045173ns) {Configure:104(23044439ns) {ProcessConfigured:135(22560341ns) {StartInitialHandshake:157(22559760ns) {DropHandshakes:470(367ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(166ns) PrepareNewSessionHandshake:477(1657ns) {SwitchToState:179(1094ns)}}}}}
2025-12-24T02:11:36.931215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241390609086200:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:36.931298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241390609086201:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.944107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241388736502527:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.944212Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241391242343129:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41178894ns) {Configure:104(41177483ns) {ProcessConfigured:135(40710774ns) {StartInitialHandshake:157(40710000ns) {DropHandshakes:470(690ns) {DropIncomingHandshake:442(39ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(1636ns) {SwitchToState:179(799ns)}}}}}
2025-12-24T02:11:36.944815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241391242343136:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:36.944935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241391242343137:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:36.945623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241391242343138:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45070
2025-12-24T02:11:36.946956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241389902856231:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:36.951067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389894450953:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.952170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241389894450893:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.956027Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241388736502527:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.956441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241388736502462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.956577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389894450957:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.956585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389894450957:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.957110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241388736502529:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.959067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241390609086207:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54117
2025-12-24T02:11:36.959123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086189:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.959127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241390609086189:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.960812Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389894450957:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:36.960873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241390609086189:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:36.970900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241390609086128:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.970928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.970940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241390609086128:2048] [node 2] ICP22 created new session: [1:7587241390609086209:2048]
2025-12-24T02:11:36.970949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241390609086209:2048] [node 2] ICS09 handshake done sender: [1:7587241390609086189:2107] self: [1:7587241390609086188:0] peer: [2:7587241389894450971:0] socket: 276 qp: -1
2025-12-24T02:11:36.970959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241390609086209:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:36.970971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.974865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389894450955:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:36.974890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:36.974900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389894450955:2108] [node 6] ICP22 created new session: [2:7587241389894450976:2108]
2025-12-24T02:11:36.974908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241389894450976:2108] [node 6] ICS09 handshake done sender: [2:7587241389894450957:2109] self: [2:7587241389894450956:0] peer: [6:7587241388736502528:0] socket: 282 qp: -1
2025-12-24T02:11:36.974916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241389894450976:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:36.974927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:36.976030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086207:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.974917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241391242343138:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.974973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241391242343068:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.975078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241391504770477:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.975142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241391504770478:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:36.975202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241391504770479:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:36.975425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241391504770481:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:36.975916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241391504770481:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.976220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241391504770478:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.976475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241391504770418:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.986784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241391242343140:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.986807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241391242343140:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.986830Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086192:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.986847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241390609086192:2108] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.991022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241389902856232:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:36.991069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241389902856233:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:36.994873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241388736502518:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.994887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241388736502518:2102] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:36.995319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241388736502527:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.996152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241389894450955:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(21218811ns) {HandleHandshakeStatus:329(21217831ns) {DropIncomingHandshake:442(310ns) DropOutgoingHandshake:457(265ns) SwitchToState:179(64ns) ProcessPendingSessionEvents:495(19832455ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 19813816ns max# 19777883ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:36.996674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241391504770479:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.997028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241391504770417:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.996753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241388736502462:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:36.996772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:36.996777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241388736502462:2049] [node 2] ICP22 created new session: [6:7587241388736502532:2049]
2025-12-24T02:11:36.996784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241388736502532:2049] [node 2] ICS09 handshake done sender: [6:7587241388736502527:2110] self: [6:7587241388736502528:0] peer: [2:7587241389894450956:0] socket: 284 qp: -1
2025-12-24T02:11:36.996791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241388736502532:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:36.996798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:36.997062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241388736502530:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:36.997193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241389902856233:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:36.997610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241389902856171:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:36.998201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241391242343140:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:36.998347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241390609086192:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:36.998398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241389894450975:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51539
2025-12-24T02:11:36.998483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241388736502518:2102] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.003001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241389902856231:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.003035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241389902856232:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.003084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241389902856170:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.003439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241389902856172:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.003844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737781:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.003926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241391242343136:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.003943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.003948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241391242343136:2108] [node 5] ICP22 created new session: [3:7587241395537310447:2108]
2025-12-24T02:11:37.003954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395537310447:2108] [node 5] ICS09 handshake done sender: [3:7587241391242343140:2113] self: [3:7587241391242343139:0] peer: [5:7587241391504770483:0] socket: 289 qp: -1
2025-12-24T02:11:37.003959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395537310447:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:37.003967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.004182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241391242343133:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.004185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241391242343133:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.004757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395537310446:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45070
2025-12-24T02:11:37.003370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389894450964:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.003379Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389894450964:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.007128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241391504770477:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.007259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241391504770481:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.007272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.007276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241391504770481:2111] [node 6] ICP22 created new session: [5:7587241395799737783:2111]
2025-12-24T02:11:37.007281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395799737783:2111] [node 6] ICS09 handshake done sender: [5:7587241391504770477:2108] self: [5:7587241391504770482:0] peer: [6:7587241388736502517:0] socket: 280 qp: -1
2025-12-24T02:11:37.007286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395799737783:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:37.007306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.006833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241390609086129:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.006858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.006893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241390609086129:2049] [node 3] ICP22 created new session: [1:7587241394904053510:2049]
2025-12-24T02:11:37.006900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394904053510:2049] [node 3] ICS09 handshake done sender: [1:7587241390609086192:2108] self: [1:7587241390609086191:0] peer: [3:7587241391242343148:0] socket: 285 qp: -1
2025-12-24T02:11:37.006907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394904053510:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.006916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.007216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086199:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.007219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241390609086199:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.008201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241388736502530:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.008218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241388736502516:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.008229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.008236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241388736502516:2099] [node 5] ICP22 created new session: [6:7587241393031469830:2099]
2025-12-24T02:11:37.008240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393031469830:2099] [node 5] ICS09 handshake done sender: [6:7587241388736502518:2102] self: [6:7587241388736502517:0] peer: [5:7587241391504770482:0] socket: 279 qp: -1
2025-12-24T02:11:37.008246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393031469830:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:37.008253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.010714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389894450966:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.010730Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389894450966:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.014764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737782:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.015260Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241391504770478:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.015469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241391504770418:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.015485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.015490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241391504770418:2050] [node 3] ICP22 created new session: [5:7587241395799737785:2050]
2025-12-24T02:11:37.015495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395799737785:2050] [node 3] ICS09 handshake done sender: [5:7587241391504770478:2109] self: [5:7587241391504770483:0] peer: [3:7587241391242343139:0] socket: 291 qp: -1
2025-12-24T02:11:37.015500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395799737785:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.015506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.016003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241388736502461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.018633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394904053513:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54117
2025-12-24T02:11:37.018875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394904053512:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54117
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.020399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086203:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.020407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241390609086203:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.020967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241390609086199:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.021100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241390609086194:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.021114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.021119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241390609086194:2109] [node 4] ICP22 created new session: [1:7587241394904053517:2109]
2025-12-24T02:11:37.021125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394904053517:2109] [node 4] ICS09 handshake done sender: [1:7587241390609086199:2112] self: [1:7587241390609086196:0] peer: [4:7587241394197823533:0] socket: 294 qp: -1
2025-12-24T02:11:37.021131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394904053517:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:37.021138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.022657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241390609086203:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.019218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393031469834:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:37.019846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389894450953:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.020304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241394197823536:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:37.020324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241394197823537:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:37.022670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241389902856232:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.027026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394904053513:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.027041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241390609086200:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.027052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.027060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241390609086200:2110] [node 6] ICP22 created new session: [1:7587241394904053519:2110]
2025-12-24T02:11:37.027067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394904053519:2110] [node 6] ICS09 handshake done sender: [1:7587241390609086203:2113] self: [1:7587241390609086202:0] peer: [6:7587241393031469832:0] socket: 302 qp: -1
2025-12-24T02:11:37.027073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394904053519:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:37.027081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.027338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241390609086194:2109] [node 4] ICP18 handshake (actor [1:7587241394904053513:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.035946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241391504770416:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17152559ns) {Configure:104(17151817ns) {ProcessConfigured:135(16800513ns) {StartInitialHandshake:157(16799862ns) {DropHandshakes:470(615ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(2437ns) {SwitchToState:179(1701ns)}}}}}
2025-12-24T02:11:37.036132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395799737790:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:37.037137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737791:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.039032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394904053521:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54117
2025-12-24T02:11:37.038968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241389902856170:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.038992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.039000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241389902856170:2048] [node 1] ICP22 created new session: [4:7587241394197823540:2048]
2025-12-24T02:11:37.039007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241394197823540:2048] [node 1] ICS09 handshake done sender: [4:7587241389902856232:2109] self: [4:7587241394197823533:0] peer: [1:7587241390609086196:0] socket: 295 qp: -1
2025-12-24T02:11:37.039015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241394197823540:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.039024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.039509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241394197823542:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:37.039595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241394197823543:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:37.040539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241389894450893:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.040562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.040567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389894450893:2048] [node 1] ICP22 created new session: [2:7587241394189418278:2048]
2025-12-24T02:11:37.040573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241394189418278:2048] [node 1] ICS09 handshake done sender: [2:7587241389894450953:2107] self: [2:7587241389894450971:0] peer: [1:7587241390609086188:0] socket: 277 qp: -1
2025-12-24T02:11:37.040578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241394189418278:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.040587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.043073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241389902856170:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:37.043480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241394197823544:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:37.043699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241394197823545:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:37.047865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395799737791:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.050138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241391504770416:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.050243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737796:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.050260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737797:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.053128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241394197823544:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.053236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395799737797:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.053258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241395799737790:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.053343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241394197823549:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.053348Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241394197823549:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.053813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241394197823549:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.058570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241390609086205:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.058589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241390609086205:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.058131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241393031469834:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38840558ns) {Configure:104(38839976ns) {ProcessConfigured:135(38576975ns) {StartInitialHandshake:157(38576316ns) {DropHandshakes:470(125ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(46ns) PrepareNewSessionHandshake:477(1837ns) {SwitchToState:179(1130ns)}}}}}
2025-12-24T02:11:37.058474Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241389894450893:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17832797ns) {HandleHandshakeStatus:329(17832018ns) {DropIncomingHandshake:442(166ns) DropOutgoingHandshake:457(1570ns) SwitchToState:179(219ns) ProcessPendingSessionEvents:495(17741874ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17698443ns max# 17614214ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.062880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241394197823543:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.062910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.062920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241394197823543:2115] [node 5] ICP22 created new session: [4:7587241394197823554:2115]
2025-12-24T02:11:37.062927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241394197823554:2115] [node 5] ICS09 handshake done sender: [4:7587241394197823549:2119] self: [4:7587241394197823548:0] peer: [5:7587241395799737799:0] socket: 332 qp: -1
2025-12-24T02:11:37.062935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241394197823554:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:37.062946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.063190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737800:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.063594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395799737797:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.063645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241395799737790:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.063660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.063665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395799737790:2114] [node 4] ICP22 created new session: [5:7587241395799737801:2114]
2025-12-24T02:11:37.063681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395799737801:2114] [node 4] ICS09 handshake done sender: [5:7587241395799737797:2119] self: [5:7587241395799737799:0] peer: [4:7587241394197823548:0] socket: 334 qp: -1
2025-12-24T02:11:37.063689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395799737801:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:37.063695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.067115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241391242343133:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.076653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394904053521:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.074890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389894450964:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.076006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393031469837:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:37.089427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241390609086205:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.086821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241391242343138:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.090790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241390609086201:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.090820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.090832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241390609086201:2111] [node 5] ICP22 created new session: [1:7587241394904053524:2111]
2025-12-24T02:11:37.090839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394904053524:2111] [node 5] ICS09 handshake done sender: [1:7587241390609086205:2114] self: [1:7587241390609086204:0] peer: [5:7587241395799737798:0] socket: 319 qp: -1
2025-12-24T02:11:37.090849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394904053524:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:37.090860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.094127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241394189418277:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51539
2025-12-24T02:11:37.094216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389894450966:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.094432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241389902856231:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.094866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241391504770479:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.094770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389894450961:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.094787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.094796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389894450961:2110] [node 5] ICP22 created new session: [2:7587241394189418280:2110]
2025-12-24T02:11:37.094802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241394189418280:2110] [node 5] ICS09 handshake done sender: [2:7587241389894450964:2112] self: [2:7587241389894450963:0] peer: [5:7587241391504770484:0] socket: 296 qp: -1
2025-12-24T02:11:37.094808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241394189418280:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:37.094816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.095037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389894450962:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.095045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.095049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389894450962:2111] [node 4] ICP22 created new session: [2:7587241394189418282:2111]
2025-12-24T02:11:37.095053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241394189418282:2111] [node 4] ICS09 handshake done sender: [2:7587241389894450966:2113] self: [2:7587241389894450965:0] peer: [4:7587241389902856234:0] socket: 297 qp: -1
2025-12-24T02:11:37.095057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241394189418282:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:37.095061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.094956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393031469838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:37.095153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241388736502530:2112] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.098396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241389902856172:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.098424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.098430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241389902856172:2050] [node 3] ICP22 created new session: [4:7587241394197823557:2050]
2025-12-24T02:11:37.098435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241394197823557:2050] [node 3] ICS09 handshake done sender: [4:7587241389902856231:2108] self: [4:7587241394197823535:0] peer: [3:7587241391242343130:0] socket: 290 qp: -1
2025-12-24T02:11:37.098440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241394197823557:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.098446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.098905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241394197823556:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58482
2025-12-24T02:11:37.098914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241389902856233:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.095821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241391242343068:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241391242343138:2112] held: no
2025-12-24T02:11:37.095932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241391242343068:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.095941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.095947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241391242343068:2048] [node 1] ICP22 created new session: [3:7587241395537310451:2048]
2025-12-24T02:11:37.095954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395537310451:2048] [node 1] ICS09 handshake done sender: [3:7587241391242343138:2112] self: [3:7587241391242343148:0] peer: [1:7587241390609086191:0] socket: 286 qp: -1
2025-12-24T02:11:37.095959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395537310451:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.095965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.096283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241391242343129:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.096289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.096293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241391242343129:2107] [node 4] ICP22 created new session: [3:7587241395537310453:2107]
2025-12-24T02:11:37.096298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395537310453:2107] [node 4] ICS09 handshake done sender: [3:7587241391242343133:2110] self: [3:7587241391242343130:0] peer: [4:7587241394197823535:0] socket: 287 qp: -1
2025-12-24T02:11:37.096302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395537310453:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:37.096306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.098770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241391504770417:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.098786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.098792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241391504770417:2049] [node 2] ICP22 created new session: [5:7587241395799737804:2049]
2025-12-24T02:11:37.098798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395799737804:2049] [node 2] ICS09 handshake done sender: [5:7587241391504770479:2110] self: [5:7587241391504770484:0] peer: [2:7587241389894450963:0] socket: 301 qp: -1
2025-12-24T02:11:37.098803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395799737804:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.098811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.099170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395799737803:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48416
2025-12-24T02:11:37.099025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241389902856171:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.099038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.099050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241389902856171:2049] [node 2] ICP22 created new session: [4:7587241394197823559:2049]
2025-12-24T02:11:37.099054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241394197823559:2049] [node 2] ICS09 handshake done sender: [4:7587241389902856233:2110] self: [4:7587241389902856234:0] peer: [2:7587241389894450965:0] socket: 304 qp: -1
2025-12-24T02:11:37.099058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241394197823559:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.099063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.101781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241388736502461:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.101800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.101807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241388736502461:2048] [node 1] ICP22 created new session: [6:7587241393031469843:2048]
2025-12-24T02:11:37.101815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393031469843:2048] [node 1] ICS09 handshake done sender: [6:7587241388736502530:2112] self: [6:7587241393031469832:0] peer: [1:7587241390609086202:0] socket: 305 qp: -1
2025-12-24T02:11:37.101821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393031469843:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.101829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.103964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393031469838:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.105085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393031469834:2113] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.105160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393031469845:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:37.106667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395799737791:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.108493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241394197823547:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.108506Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241394197823547:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.108325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395537310456:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45070
2025-12-24T02:11:37.108379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395537310457:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45070
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.111081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395537310457:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.111313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241391242343144:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.111319Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241391242343144:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.111746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395537310456:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.111802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241391242343144:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.111791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241394197823556:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.111869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241394197823547:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.111887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241391504770416:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241395799737791:2116] held: no
2025-12-24T02:11:37.111994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241391504770416:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.112005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.112012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241391504770416:2048] [node 1] ICP22 created new session: [5:7587241395799737806:2048]
2025-12-24T02:11:37.112019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395799737806:2048] [node 1] ICS09 handshake done sender: [5:7587241395799737791:2116] self: [5:7587241395799737798:0] peer: [1:7587241390609086204:0] socket: 318 qp: -1
2025-12-24T02:11:37.112026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395799737806:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.112033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.111985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241391242343069:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.112000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241391242343137:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.112012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.112025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241391242343137:2109] [node 6] ICP22 created new session: [3:7587241395537310461:2109]
2025-12-24T02:11:37.112030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395537310461:2109] [node 6] ICS09 handshake done sender: [3:7587241391242343144:2115] self: [3:7587241391242343143:0] peer: [6:7587241393031469848:0] socket: 338 qp: -1
2025-12-24T02:11:37.112036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395537310461:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:37.112043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.112607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241394197823542:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.112623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.112627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241394197823542:2114] [node 6] ICP22 created new session: [4:7587241394197823563:2114]
2025-12-24T02:11:37.112632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241394197823563:2114] [node 6] ICS09 handshake done sender: [4:7587241394197823547:2118] self: [4:7587241394197823546:0] peer: [6:7587241393031469846:0] socket: 331 qp: -1
2025-12-24T02:11:37.112638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241394197823563:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:37.112646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.110999Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393031469847:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:37.111204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393031469845:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.111223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241388736502463:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.112541Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393031469838:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.111563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241394189418277:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.112480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241389894450970:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.112483Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241389894450970:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.113360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241389894450970:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.113420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241389894450894:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.113434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.113441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241389894450894:2049] [node 3] ICP22 created new session: [2:7587241394189418287:2049]
2025-12-24T02:11:37.113447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241394189418287:2049] [node 3] ICS09 handshake done sender: [2:7587241389894450970:2115] self: [2:7587241389894450969:0] peer: [3:7587241395537310460:0] socket: 337 qp: -1
2025-12-24T02:11:37.113452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241394189418287:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.113457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.118882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241391242343069:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21449) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241395537310456:2117] held: no
2025-12-24T02:11:37.119094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395537310463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45070
2025-12-24T02:11:37.119713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395537310456:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.119787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241391242343069:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.119802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.119808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241391242343069:2049] [node 2] ICP22 created new session: [3:7587241395537310464:2049]
2025-12-24T02:11:37.119814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395537310464:2049] [node 2] ICS09 handshake done sender: [3:7587241395537310456:2117] self: [3:7587241395537310460:0] peer: [2:7587241389894450969:0] socket: 340 qp: -1
2025-12-24T02:11:37.119819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395537310464:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.119826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.120398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393031469834:2113] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.120413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.120419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393031469834:2113] [node 4] ICP22 created new session: [6:7587241393031469849:2113]
2025-12-24T02:11:37.120424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393031469849:2113] [node 4] ICS09 handshake done sender: [6:7587241393031469838:2116] self: [6:7587241393031469846:0] peer: [4:7587241394197823546:0] socket: 310 qp: -1
2025-12-24T02:11:37.120429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393031469849:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:37.120435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.120751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241388736502463:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3760) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241393031469845:2118] held: no
2025-12-24T02:11:37.121963Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241393031469834:2113] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:37.122036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393031469851:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42089
2025-12-24T02:11:37.122173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393031469845:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.122210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241388736502463:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.122218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.122223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241388736502463:2050] [node 3] ICP22 created new session: [6:7587241393031469852:2050]
2025-12-24T02:11:37.122227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393031469852:2050] [node 3] ICS09 handshake done sender: [6:7587241393031469845:2118] self: [6:7587241393031469848:0] peer: [3:7587241391242343143:0] socket: 326 qp: -1
2025-12-24T02:11:37.122232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393031469852:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.122238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.129174Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:37.129244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241395537310451:2048] [node 1] ICS01 socket: 286 reason# EndOfStream
2025-12-24T02:11:37.129250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241391242343068:2048] [node 1] ICP30 unregister session Session# [3:7587241395537310451:2048] VirtualId# [3:7587241391242343148:0]
2025-12-24T02:11:37.129257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241395537310451:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.129281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.129300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.128511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:37.128603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241394189418278:2048] [node 1] ICS07 socket disconnect 277 reason# EndOfStream
2025-12-24T02:11:37.128610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241394189418278:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.128652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.128668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241394189418278:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:37.134599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241389894450893:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:37.134616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241394189418278:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:37.134901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241389894450893:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.134916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241394189418278:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.134921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241389894450893:2048] [node 1] ICP30 unregister session Session# [2:7587241394189418278:2048] VirtualId# [2:7587241389894450971:0]
2025-12-24T02:11:37.134975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.134981Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241389894450893:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.134984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.142778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:37.142877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395799737806:2048] [node 1] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:37.142886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241391504770416:2048] [node 1] ICP30 unregister session Session# [5:7587241395799737806:2048] VirtualId# [5:7587241395799737798:0]
2025-12-24T02:11:37.142893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395799737806:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.142946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.142971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.143010Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:37.143077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241394197823540:2048] [node 1] ICS07 socket disconnect 295 reason# EndOfStream
2025-12-24T02:11:37.143081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241394197823540:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.143106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.143126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241394197823540:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:37.143779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241389902856170:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.143789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241394197823540:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.143794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241389902856170:2048] [node 1] ICP30 unregister session Session# [4:7587241394197823540:2048] VirtualId# [4:7587241394197823533:0]
2025-12-24T02:11:37.143838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.143845Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241389902856170:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.143848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26067) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.143312Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:37.143371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393031469843:2048] [node 1] ICS01 socket: 305 reason# EndOfStream
2025-12-24T02:11:37.143376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241388736502461:2048] [node 1] ICP30 unregister session Session# [6:7587241393031469843:2048] VirtualId# [6:7587241393031469832:0]
2025-12-24T02:11:37.143381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393031469843:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.143405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.143425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.150888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:37.150988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241395537310464:2049] [node 2] ICS01 socket: 340 reason# EndOfStream
2025-12-24T02:11:37.150997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241391242343069:2049] [node 2] ICP30 unregister session Session# [3:7587241395537310464:2049] VirtualId# [3:7587241395537310460:0]
2025-12-24T02:11:37.151006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241395537310464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.151049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.151068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.154918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241389902856170:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:37.162775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:37.162871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241388736502532:2049] [node 2] ICS07 socket disconnect 284 reason# EndOfStream
2025-12-24T02:11:37.162878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241388736502532:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.162923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.162944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241388736502532:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:37.163130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:37.163189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241394197823559:2049] [node 2] ICS01 socket: 304 reason# EndOfStream
2025-12-24T02:11:37.163194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241389902856171:2049] [node 2] ICP30 unregister session Session# [4:7587241394197823559:2049] VirtualId# [4:7587241389902856234:0]
2025-12-24T02:11:37.163199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241394197823559:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.163221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.163241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.163549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241388736502462:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21449) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.163558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241388736502532:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.163562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241388736502462:2049] [node 2] ICP30 unregister session Session# [6:7587241388736502532:2049] VirtualId# [6:7587241388736502528:0]
2025-12-24T02:11:37.163605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.163610Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241388736502462:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21449) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.163613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21449) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.166916Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:37.167011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395799737804:2049] [node 2] ICS01 socket: 301 reason# EndOfStream
2025-12-24T02:11:37.167024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241391504770417:2049] [node 2] ICP30 unregister session Session# [5:7587241395799737804:2049] VirtualId# [5:7587241391504770484:0]
2025-12-24T02:11:37.167031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395799737804:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.167082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.167105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.178052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241388736502462:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:37.195105Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:37.195215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393031469852:2050] [node 3] ICS01 socket: 326 reason# EndOfStream
2025-12-24T02:11:37.195229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241388736502463:2050] [node 3] ICP30 unregister session Session# [6:7587241393031469852:2050] VirtualId# [6:7587241393031469848:0]
2025-12-24T02:11:37.195238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393031469852:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.195304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:37.195333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:37.198542Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:37.198639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241395799737785:2050] [node 3] ICS07 socket disconnect 291 reason# EndOfStream
2025-12-24T02:11:37.198646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395799737785:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.198708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:37.198735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241395799737785:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:37.199095Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241391504770418:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3760) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.199110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395799737785:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.199115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241391504770418:2050] [node 3] ICP30 unregister session Session# [5:7587241395799737785:2050] VirtualId# [5:7587241391504770483:0]
2025-12-24T02:11:37.199152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:37.199159Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241391504770418:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.199162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3760) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.213333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:37.213439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393031469849:2113] [node 4] ICS01 socket: 310 reason# EndOfStream
2025-12-24T02:11:37.213448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241393031469834:2113] [node 4] ICP30 unregister session Session# [6:7587241393031469849:2113] VirtualId# [6:7587241393031469846:0]
2025-12-24T02:11:37.213456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393031469849:2113] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.213510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:37.213534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:37.290926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393148286463:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.290945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393148286464:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:37.290934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241396280727807:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:37.290943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241396280727808:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:37.297240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395216655092:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.297249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395216655093:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.297254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395216655094:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.302043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241396536001627:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.302055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241396536001628:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.302791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395783472317:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.302801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395783472318:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.302806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395783472319:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.314849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241395027817290:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.314860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241395027817291:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.314865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241395027817292:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.363050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241396280727869:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:37.363159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241396280727870:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:37.367009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395783472359:2086] [node 6] ICP01 ready to work
2025-12-24T02:11:37.368070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395216655147:2099] [node 6] ICP01 ready to work
2025-12-24T02:11:37.373474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241395783472376:2098] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.382431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241396280727869:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19173199ns) {Configure:104(19172487ns) {ProcessConfigured:135(18770279ns) {StartInitialHandshake:157(18769661ns) {DropHandshakes:470(428ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(2260ns) {SwitchToState:179(1046ns)}}}}}
2025-12-24T02:11:37.383551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241396280727878:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48377
2025-12-24T02:11:37.386848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241395027817334:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:37.387076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241395027817356:2094] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.391258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655162:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.394898Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241395783472318:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21340089ns) {Configure:104(21339200ns) {ProcessConfigured:135(20957684ns) {StartInitialHandshake:157(20957248ns) {DropHandshakes:470(1084ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(371ns) PrepareNewSessionHandshake:477(1032ns) {SwitchToState:179(215ns)}}}}}
2025-12-24T02:11:37.399911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393148286524:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:37.398777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395216655156:2102] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.406408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472395:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.401201Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241395027817290:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(38275327ns) {Configure:104(38274289ns) {ProcessConfigured:135(37851276ns) {StartInitialHandshake:157(37850468ns) {DropHandshakes:470(757ns) {DropIncomingHandshake:442(85ns) DropOutgoingHandshake:457(126ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1970ns) {SwitchToState:179(1058ns)}}}}}
2025-12-24T02:11:37.405906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241396280727883:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.410965Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241396280727870:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47273054ns) {Configure:104(47270417ns) {ProcessConfigured:135(47038218ns) {StartInitialHandshake:157(47037536ns) {DropHandshakes:470(268ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(958ns) {SwitchToState:179(264ns)}}}}}
2025-12-24T02:11:37.413497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727878:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.414130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241396280727885:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48377
2025-12-24T02:11:37.415378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655169:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.418787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241396536001690:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:37.418986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241396536001691:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:37.419221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241396536001692:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.422892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727885:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.423030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241396280727889:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48377
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.438310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817369:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.435827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393148286535:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:37.435981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393148286536:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:37.436077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286528:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
No RDMA context found for GID: 2025-12-24T02:11:37.436146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286529:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
::ffff:127.1.0.0
2025-12-24T02:11:37.436180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286530:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
2025-12-24T02:11:37.438562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001706:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.439000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286528:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.442934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655162:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.443085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395216655092:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.442764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817367:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.447499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001708:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.447670Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001706:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.450961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001707:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.447099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241393148286524:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47053002ns) {Configure:104(47051662ns) {ProcessConfigured:135(46578686ns) {StartInitialHandshake:157(46577629ns) {DropHandshakes:470(1369ns) {DropIncomingHandshake:442(247ns) DropOutgoingHandshake:457(253ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(1656ns) {SwitchToState:179(831ns)}}}}}
2025-12-24T02:11:37.447144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241393148286463:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.447207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286529:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.451071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241395027817367:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.451138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241395027817356:2094] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.454734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817368:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.455079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241395027817368:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.455127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241395027817334:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.455267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655169:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.456662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472395:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.454723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727873:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.454755Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241396280727873:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.455154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727875:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.455159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241396280727875:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.455631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241396280727891:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48377
2025-12-24T02:11:37.457421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241396280727875:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.458640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817370:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.458782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655149:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.458810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241395216655149:2100] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.458736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001709:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.458874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001708:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.458934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001707:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.458965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241396536001627:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.461281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286547:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
2025-12-24T02:11:37.461978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241395027817369:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.462050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241395027817292:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.468488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472361:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.468516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241395783472361:2088] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.468479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001694:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.468513Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241396536001694:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.471353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393148286528:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.472599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472397:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.472582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241393148286463:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.472598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.472604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393148286463:2048] [node 1] ICP22 created new session: [2:7587241393148286548:2048]
2025-12-24T02:11:37.472609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393148286548:2048] [node 1] ICS09 handshake done sender: [2:7587241393148286528:2110] self: [2:7587241393148286545:0] peer: [1:7587241396280727874:0] socket: 294 qp: -1
2025-12-24T02:11:37.472614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393148286548:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.472621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.471454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727889:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.471493Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727877:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.471498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241396280727877:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.472032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241396280727807:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.472049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.472058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241396280727807:2048] [node 2] ICP22 created new session: [1:7587241396280727895:2048]
2025-12-24T02:11:37.472065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241396280727895:2048] [node 2] ICS09 handshake done sender: [1:7587241396280727875:2110] self: [1:7587241396280727874:0] peer: [2:7587241393148286545:0] socket: 291 qp: -1
2025-12-24T02:11:37.472073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241396280727895:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:37.472083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.474791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241395027817291:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16498602ns) {Configure:104(16497191ns) {ProcessConfigured:135(16086051ns) {StartInitialHandshake:157(16085229ns) {DropHandshakes:470(1002ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:477(1757ns) {SwitchToState:179(1205ns)}}}}}
2025-12-24T02:11:37.474869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241395027817370:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.474932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241395027817291:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.478815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001709:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.484853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395783472361:2088] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.487386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241396280727807:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:37.487446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241396280727873:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.486724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241395783472359:2086] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.486758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.486770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395783472359:2086] [node 6] ICP22 created new session: [5:7587241395783472399:2086]
2025-12-24T02:11:37.486784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395783472399:2086] [node 6] ICS09 handshake done sender: [5:7587241395783472361:2088] self: [5:7587241395783472360:0] peer: [6:7587241395027817371:0] socket: 283 qp: -1
2025-12-24T02:11:37.486795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395783472399:2086] [node 6] ICS10 traffic start
2025-12-24T02:11:37.486808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.493556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.493584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241396536001628:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.492511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286538:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.492529Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393148286538:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.493103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286530:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.497938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395216655149:2100] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.498010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241396536001694:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.497848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393148286538:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.498946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241395216655147:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.498973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.498982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395216655147:2099] [node 6] ICP22 created new session: [4:7587241395216655176:2099]
2025-12-24T02:11:37.498991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395216655176:2099] [node 6] ICS09 handshake done sender: [4:7587241395216655149:2100] self: [4:7587241395216655148:0] peer: [6:7587241395027817372:0] socket: 297 qp: -1
2025-12-24T02:11:37.498999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395216655176:2099] [node 6] ICS10 traffic start
2025-12-24T02:11:37.499024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.502543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655175:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.502705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395216655162:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.502771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395216655092:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.502969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472401:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.502834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241396280727869:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.502857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.502864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241396280727869:2107] [node 4] ICP22 created new session: [1:7587241396280727899:2107]
2025-12-24T02:11:37.502871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241396280727899:2107] [node 4] ICS09 handshake done sender: [1:7587241396280727873:2109] self: [1:7587241396280727871:0] peer: [4:7587241395216655172:0] socket: 290 qp: -1
2025-12-24T02:11:37.502877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241396280727899:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:37.502886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.503377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241396280727877:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.503190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241396536001691:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.503211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.503217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241396536001691:2108] [node 6] ICP22 created new session: [3:7587241396536001716:2108]
2025-12-24T02:11:37.503223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241396536001716:2108] [node 6] ICS09 handshake done sender: [3:7587241396536001694:2110] self: [3:7587241396536001693:0] peer: [6:7587241395027817374:0] socket: 306 qp: -1
2025-12-24T02:11:37.503228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241396536001716:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:37.503248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.503589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241396536001708:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.503637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241396536001627:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.503647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.503651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241396536001627:2048] [node 1] ICP22 created new session: [3:7587241396536001719:2048]
2025-12-24T02:11:37.503656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241396536001719:2048] [node 1] ICS09 handshake done sender: [3:7587241396536001708:2117] self: [3:7587241396536001710:0] peer: [1:7587241396280727876:0] socket: 310 qp: -1
2025-12-24T02:11:37.503661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241396536001719:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.503666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.502784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.502803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395216655092:2048] [node 1] ICP22 created new session: [4:7587241395216655179:2048]
2025-12-24T02:11:37.502812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395216655179:2048] [node 1] ICS09 handshake done sender: [4:7587241395216655162:2112] self: [4:7587241395216655172:0] peer: [1:7587241396280727871:0] socket: 296 qp: -1
2025-12-24T02:11:37.502819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395216655179:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.502831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.505055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241396280727901:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48377
2025-12-24T02:11:37.505132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241396280727808:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.505146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.505152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241396280727808:2049] [node 3] ICP22 created new session: [1:7587241396280727902:2049]
2025-12-24T02:11:37.505158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241396280727902:2049] [node 3] ICS09 handshake done sender: [1:7587241396280727877:2111] self: [1:7587241396280727876:0] peer: [3:7587241396536001710:0] socket: 292 qp: -1
2025-12-24T02:11:37.505162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241396280727902:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.505167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.505018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472403:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.505580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655181:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.505615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655182:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.506889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286534:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.506906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393148286534:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.505975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472397:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.506095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472404:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.506112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241395783472317:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.506243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472403:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.506257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241395783472319:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.506322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001702:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.506331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241396536001702:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.506753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241396536001702:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.510723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241396536001690:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.510737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.510744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241396536001690:2107] [node 5] ICP22 created new session: [3:7587241396536001723:2107]
2025-12-24T02:11:37.510751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241396536001723:2107] [node 5] ICS09 handshake done sender: [3:7587241396536001702:2114] self: [3:7587241396536001701:0] peer: [5:7587241395783472406:0] socket: 307 qp: -1
2025-12-24T02:11:37.510757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241396536001723:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:37.510765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.511512Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241395783472359:2086] [node 6] ICP35 event processing took too much time PendingConnection:279(24699984ns) {HandleHandshakeStatus:329(24699001ns) {DropIncomingHandshake:442(6213ns) DropOutgoingHandshake:457(341ns) SwitchToState:179(354ns) ProcessPendingSessionEvents:495(14082781ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 14068088ns max# 14014094ns}}}
2025-12-24T02:11:37.515070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655182:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.515102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655181:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.515133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395216655094:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.514794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393148286535:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.514819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.514831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393148286535:2108] [node 6] ICP22 created new session: [2:7587241393148286553:2108]
2025-12-24T02:11:37.514838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393148286553:2108] [node 6] ICS09 handshake done sender: [2:7587241393148286538:2115] self: [2:7587241393148286537:0] peer: [6:7587241395027817376:0] socket: 317 qp: -1
2025-12-24T02:11:37.514848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393148286553:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:37.514859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.515776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.515918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727884:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.515933Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241396280727884:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.519015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395783472403:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.520044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393148286534:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.520126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393148286464:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.520144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.520153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393148286464:2049] [node 3] ICP22 created new session: [2:7587241393148286557:2049]
2025-12-24T02:11:37.520161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393148286557:2049] [node 3] ICS09 handshake done sender: [2:7587241393148286534:2114] self: [2:7587241393148286533:0] peer: [3:7587241396536001714:0] socket: 316 qp: -1
2025-12-24T02:11:37.520166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393148286557:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.520173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.530472Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241395027817334:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26130) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241395027817368:2116] held: no
2025-12-24T02:11:37.532296Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241396280727884:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.532414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241396280727870:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.532436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.532445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241396280727870:2108] [node 5] ICP22 created new session: [1:7587241396280727906:2108]
2025-12-24T02:11:37.532452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241396280727906:2108] [node 5] ICS09 handshake done sender: [1:7587241396280727884:2114] self: [1:7587241396280727872:0] peer: [5:7587241395783472405:0] socket: 327 qp: -1
2025-12-24T02:11:37.532458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241396280727906:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:37.532469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.534780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241395783472319:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.534807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.534818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395783472319:2050] [node 3] ICP22 created new session: [5:7587241395783472408:2050]
2025-12-24T02:11:37.534825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395783472408:2050] [node 3] ICS09 handshake done sender: [5:7587241395783472403:2118] self: [5:7587241395783472406:0] peer: [3:7587241396536001701:0] socket: 339 qp: -1
2025-12-24T02:11:37.534833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395783472408:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.534841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.538539Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241396536001690:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(27698104ns) {HandleHandshakeStatus:329(27697381ns) {DropIncomingHandshake:442(3897ns) DropOutgoingHandshake:457(113ns) SwitchToState:179(52ns) ProcessPendingSessionEvents:495(27617122ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27592608ns max# 27504773ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.539775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655186:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.542859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655186:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.545261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241396536001700:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.545276Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241396536001700:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.542899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395216655093:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.543805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241395027817356:2094] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29657) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241395027817367:2115] held: no
2025-12-24T02:11:37.544236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.546011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817380:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.544885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241395783472317:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241395783472397:2116] held: no
2025-12-24T02:11:37.545107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472404:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.551804Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241396536001628:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241396536001709:2118] held: no
2025-12-24T02:11:37.552173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001725:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.550785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241395783472319:2050] [node 3] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.553789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472410:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.554181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241396536001700:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.552829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286544:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.552845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393148286544:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.551181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.552671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817381:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.552720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.554065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241395027817368:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.554114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241395027817369:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.554239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241395027817370:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.554348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241395027817334:2089] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.554361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.554367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241395027817334:2089] [node 4] ICP22 created new session: [6:7587241395027817384:2089]
2025-12-24T02:11:37.554372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241395027817384:2089] [node 4] ICS09 handshake done sender: [6:7587241395027817368:2116] self: [6:7587241395027817372:0] peer: [4:7587241395216655148:0] socket: 318 qp: -1
2025-12-24T02:11:37.554378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241395027817384:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:37.554385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.554575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241395027817292:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.554583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.554588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241395027817292:2050] [node 3] ICP22 created new session: [6:7587241395027817386:2050]
2025-12-24T02:11:37.554592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241395027817386:2050] [node 3] ICS09 handshake done sender: [6:7587241395027817369:2117] self: [6:7587241395027817374:0] peer: [3:7587241396536001693:0] socket: 319 qp: -1
2025-12-24T02:11:37.554596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241395027817386:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.554600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.554873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655189:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.554827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393148286544:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.554997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393148286524:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.555017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.555022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393148286524:2107] [node 4] ICP22 created new session: [2:7587241393148286562:2107]
2025-12-24T02:11:37.555028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393148286562:2107] [node 4] ICS09 handshake done sender: [2:7587241393148286544:2117] self: [2:7587241393148286525:0] peer: [4:7587241395216655187:0] socket: 338 qp: -1
2025-12-24T02:11:37.555033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393148286562:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:37.555039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.555615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286560:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
2025-12-24T02:11:37.555634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393148286561:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15973
2025-12-24T02:11:37.558918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241395783472376:2098] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.558968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241395783472401:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.559027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395783472397:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.559067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241396536001692:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.559085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.559091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241396536001692:2109] [node 4] ICP22 created new session: [3:7587241396536001727:2109]
2025-12-24T02:11:37.559097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241396536001727:2109] [node 4] ICS09 handshake done sender: [3:7587241396536001700:2113] self: [3:7587241396536001699:0] peer: [4:7587241395216655184:0] socket: 337 qp: -1
2025-12-24T02:11:37.559102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241396536001727:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:37.559109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.558801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241395027817381:2124] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.558820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241395027817367:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.555068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395216655186:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.555155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395216655093:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.555164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.555169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395216655093:2049] [node 2] ICP22 created new session: [4:7587241395216655190:2049]
2025-12-24T02:11:37.555174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395216655190:2049] [node 2] ICS09 handshake done sender: [4:7587241395216655186:2120] self: [4:7587241395216655187:0] peer: [2:7587241393148286525:0] socket: 314 qp: -1
2025-12-24T02:11:37.555180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395216655190:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.555187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.555547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395216655188:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4710
2025-12-24T02:11:37.564981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241396536001709:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.564183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395216655171:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.564197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241395216655171:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.564807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395216655182:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.565071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241396536001628:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.565085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.565090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241396536001628:2049] [node 2] ICP22 created new session: [3:7587241396536001729:2049]
2025-12-24T02:11:37.565096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241396536001729:2049] [node 2] ICS09 handshake done sender: [3:7587241396536001709:2118] self: [3:7587241396536001714:0] peer: [2:7587241393148286533:0] socket: 323 qp: -1
2025-12-24T02:11:37.565102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241396536001729:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.565110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.565647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241396536001731:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15976
2025-12-24T02:11:37.565952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241395783472317:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.565969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.565975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395783472317:2048] [node 1] ICP22 created new session: [5:7587241395783472413:2048]
2025-12-24T02:11:37.565981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395783472413:2048] [node 1] ICS09 handshake done sender: [5:7587241395783472397:2116] self: [5:7587241395783472405:0] peer: [1:7587241396280727872:0] socket: 328 qp: -1
2025-12-24T02:11:37.565987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395783472413:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.565994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.566174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241395783472318:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.567022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286540:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.567035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393148286540:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.567889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393148286540:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.568197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393148286536:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.568209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.568215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393148286536:2109] [node 5] ICP22 created new session: [2:7587241393148286565:2109]
2025-12-24T02:11:37.568221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393148286565:2109] [node 5] ICS09 handshake done sender: [2:7587241393148286540:2116] self: [2:7587241393148286539:0] peer: [5:7587241395783472415:0] socket: 334 qp: -1
2025-12-24T02:11:37.568226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393148286565:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:37.568233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.567980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241395027817291:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.568003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.568010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241395027817291:2049] [node 2] ICP22 created new session: [6:7587241395027817388:2049]
2025-12-24T02:11:37.568016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241395027817388:2049] [node 2] ICS09 handshake done sender: [6:7587241395027817370:2118] self: [6:7587241395027817376:0] peer: [2:7587241393148286537:0] socket: 320 qp: -1
2025-12-24T02:11:37.568022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241395027817388:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.568029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.568800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241395027817356:2094] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.568808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.568813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241395027817356:2094] [node 5] ICP22 created new session: [6:7587241395027817390:2094]
2025-12-24T02:11:37.568819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241395027817390:2094] [node 5] ICS09 handshake done sender: [6:7587241395027817367:2115] self: [6:7587241395027817371:0] peer: [5:7587241395783472360:0] socket: 295 qp: -1
2025-12-24T02:11:37.568824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241395027817390:2094] [node 5] ICS10 traffic start
2025-12-24T02:11:37.568829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.568977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241395027817290:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.569477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241396280727887:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.569491Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241396280727887:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.570394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241396280727887:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.570503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241396280727883:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.570520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.570526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241396280727883:2113] [node 6] ICP22 created new session: [1:7587241396280727909:2113]
2025-12-24T02:11:37.570532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241396280727909:2113] [node 6] ICS09 handshake done sender: [1:7587241396280727887:2116] self: [1:7587241396280727886:0] peer: [6:7587241395027817392:0] socket: 345 qp: -1
2025-12-24T02:11:37.570536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241396280727909:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:37.570543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.573233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472416:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.573411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241393148286535:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:37.573016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241395216655094:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26686) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241395216655182:2119] held: no
2025-12-24T02:11:37.573119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395216655094:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.573129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.573136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395216655094:2050] [node 3] ICP22 created new session: [4:7587241395216655193:2050]
2025-12-24T02:11:37.573141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395216655193:2050] [node 3] ICS09 handshake done sender: [4:7587241395216655182:2119] self: [4:7587241395216655184:0] peer: [3:7587241396536001699:0] socket: 340 qp: -1
2025-12-24T02:11:37.573148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395216655193:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.573154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.573583Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395216655171:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.573706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241395216655156:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.573721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.573727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395216655156:2102] [node 5] ICP22 created new session: [4:7587241395216655195:2102]
2025-12-24T02:11:37.573732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395216655195:2102] [node 5] ICS09 handshake done sender: [4:7587241395216655171:2116] self: [4:7587241395216655170:0] peer: [5:7587241395783472412:0] socket: 341 qp: -1
2025-12-24T02:11:37.573740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395216655195:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:37.573745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.574758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241395783472417:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55667
2025-12-24T02:11:37.574936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395783472401:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.575009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393148286560:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.575040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241393148286536:2109] [node 5] ICP18 handshake (actor [2:7587241393148286560:2119]) is too old
2025-12-24T02:11:37.575314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241395783472404:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.575322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241395027817290:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241395027817381:2124] held: no
2025-12-24T02:11:37.575492Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241395027817393:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13140
2025-12-24T02:11:37.575652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241395027817381:2124] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.575727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241395027817290:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.575740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.575745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241395027817290:2048] [node 1] ICP22 created new session: [6:7587241395027817394:2048]
2025-12-24T02:11:37.575750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241395027817394:2048] [node 1] ICS09 handshake done sender: [6:7587241395027817381:2124] self: [6:7587241395027817392:0] peer: [1:7587241396280727886:0] socket: 347 qp: -1
2025-12-24T02:11:37.575756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241395027817394:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.575761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.575377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241395783472318:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.575391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.575399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395783472318:2049] [node 2] ICP22 created new session: [5:7587241395783472418:2049]
2025-12-24T02:11:37.575404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395783472418:2049] [node 2] ICS09 handshake done sender: [5:7587241395783472401:2117] self: [5:7587241395783472415:0] peer: [2:7587241393148286539:0] socket: 336 qp: -1
2025-12-24T02:11:37.575409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395783472418:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.575416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.576831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241395783472376:2098] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.576843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.576848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241395783472376:2098] [node 4] ICP22 created new session: [5:7587241395783472420:2098]
2025-12-24T02:11:37.576853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241395783472420:2098] [node 4] ICS09 handshake done sender: [5:7587241395783472404:2119] self: [5:7587241395783472412:0] peer: [4:7587241395216655170:0] socket: 342 qp: -1
2025-12-24T02:11:37.576858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241395783472420:2098] [node 4] ICS10 traffic start
2025-12-24T02:11:37.576862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.594442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:37.594552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241393148286548:2048] [node 1] ICS01 socket: 294 reason# EndOfStream
2025-12-24T02:11:37.594563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241393148286463:2048] [node 1] ICP30 unregister session Session# [2:7587241393148286548:2048] VirtualId# [2:7587241393148286545:0]
2025-12-24T02:11:37.594571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241393148286548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.594614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.594636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.599035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:37.599129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241396536001719:2048] [node 1] ICS01 socket: 310 reason# EndOfStream
2025-12-24T02:11:37.599138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241396536001627:2048] [node 1] ICP30 unregister session Session# [3:7587241396536001719:2048] VirtualId# [3:7587241396536001710:0]
2025-12-24T02:11:37.599146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241396536001719:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.599178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.599198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.605666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:37.605773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241395216655179:2048] [node 1] ICS01 socket: 296 reason# EndOfStream
2025-12-24T02:11:37.605788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241395216655092:2048] [node 1] ICP30 unregister session Session# [4:7587241395216655179:2048] VirtualId# [4:7587241395216655172:0]
2025-12-24T02:11:37.605796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241395216655179:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.605840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.605863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.606062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:37.606139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241395027817394:2048] [node 1] ICS01 socket: 347 reason# EndOfStream
2025-12-24T02:11:37.606143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241395027817290:2048] [node 1] ICP30 unregister session Session# [6:7587241395027817394:2048] VirtualId# [6:7587241395027817392:0]
2025-12-24T02:11:37.606150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241395027817394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.606174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.606194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.605899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:37.605968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395783472413:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-12-24T02:11:37.605973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241395783472317:2048] [node 1] ICP30 unregister session Session# [5:7587241395783472413:2048] VirtualId# [5:7587241395783472405:0]
2025-12-24T02:11:37.605979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395783472413:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.606013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:37.606033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:37.622829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:37.622942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241396536001729:2049] [node 2] ICS01 socket: 323 reason# EndOfStream
2025-12-24T02:11:37.622952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241396536001628:2049] [node 2] ICP30 unregister session Session# [3:7587241396536001729:2049] VirtualId# [3:7587241396536001714:0]
2025-12-24T02:11:37.622963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241396536001729:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.623012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.623036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.626950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:37.627038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241395216655190:2049] [node 2] ICS07 socket disconnect 314 reason# EndOfStream
2025-12-24T02:11:37.627044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241395216655190:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.627084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.627105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241395216655190:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:37.627335Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:37.627401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241395027817388:2049] [node 2] ICS07 socket disconnect 320 reason# EndOfStream
2025-12-24T02:11:37.627406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241395027817388:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.627432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.627453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241395027817388:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:37.628226Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241395027817291:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.628237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241395027817388:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.628242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241395027817291:2049] [node 2] ICP30 unregister session Session# [6:7587241395027817388:2049] VirtualId# [6:7587241395027817376:0]
2025-12-24T02:11:37.628288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.628294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241395027817291:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.628297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.628469Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241395216655093:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.628479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241395216655190:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.628483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241395216655093:2049] [node 2] ICP30 unregister session Session# [4:7587241395216655190:2049] VirtualId# [4:7587241395216655187:0]
2025-12-24T02:11:37.628528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.628531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241395216655093:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.628533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25918) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.627834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:37.627904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395783472418:2049] [node 2] ICS01 socket: 336 reason# EndOfStream
2025-12-24T02:11:37.627909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241395783472318:2049] [node 2] ICP30 unregister session Session# [5:7587241395783472418:2049] VirtualId# [5:7587241395783472415:0]
2025-12-24T02:11:37.627914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395783472418:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.627943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:37.627965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:37.638866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241395027817291:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:37.638918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241395216655093:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:37.654846Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:37.654959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241395027817386:2050] [node 3] ICS01 socket: 319 reason# EndOfStream
2025-12-24T02:11:37.654970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241395027817292:2050] [node 3] ICP30 unregister session Session# [6:7587241395027817386:2050] VirtualId# [6:7587241395027817374:0]
2025-12-24T02:11:37.654978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241395027817386:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.655033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:37.655062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:37.655160Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:37.655242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241395783472408:2050] [node 3] ICS07 socket disconnect 339 reason# EndOfStream
2025-12-24T02:11:37.655246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241395783472408:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.655275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:37.655294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241395783472408:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:37.655783Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241395783472319:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26686) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:37.655790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241395783472408:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:37.655795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241395783472319:2050] [node 3] ICP30 unregister session Session# [5:7587241395783472408:2050] VirtualId# [5:7587241395783472406:0]
2025-12-24T02:11:37.655840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:37.655846Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241395783472319:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.655850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26686) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:37.666801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241395783472319:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:37.684403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:37.684505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241395027817384:2089] [node 4] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:37.684516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241395027817334:2089] [node 4] ICP30 unregister session Session# [6:7587241395027817384:2089] VirtualId# [6:7587241395027817372:0]
2025-12-24T02:11:37.684524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241395027817384:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.684574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:37.684597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:37.698786Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:37.698894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241395027817390:2094] [node 5] ICS01 socket: 295 reason# EndOfStream
2025-12-24T02:11:37.698907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241395027817356:2094] [node 5] ICP30 unregister session Session# [6:7587241395027817390:2094] VirtualId# [6:7587241395027817371:0]
2025-12-24T02:11:37.698915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241395027817390:2094] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:37.698967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:37.698989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:37.735234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241395153318273:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.735245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241395153318274:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.736161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395119770549:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.736172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395119770550:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.736178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395119770551:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.730650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241394239395774:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:37.730660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241394239395775:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:37.728889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393639891399:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.728909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393639891400:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:37.742944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241393389691704:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.742953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241393389691705:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.743054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241393389691703:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.780260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395119770602:2098] [node 5] ICP01 ready to work
2025-12-24T02:11:37.780224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241393389691745:2088] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.781304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770603:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.783072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241395119770613:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:37.781876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691749:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
2025-12-24T02:11:37.782269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241393389691752:2092] [node 6] ICP01 ready to work
2025-12-24T02:11:37.787162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393639891462:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:37.787242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393639891457:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:37.782898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241394239395829:2100] [node 6] ICP01 ready to work
2025-12-24T02:11:37.790763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241394239395830:2097] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.791636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241395153318336:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:37.793452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318340:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.800129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241393389691705:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16390164ns) {Configure:104(16389203ns) {ProcessConfigured:135(15942175ns) {StartInitialHandshake:157(15941846ns) {DropHandshakes:470(484ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(479ns) PrepareNewSessionHandshake:477(940ns) {SwitchToState:179(567ns)}}}}}
2025-12-24T02:11:37.801781Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241395119770613:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18601642ns) {Configure:104(18600840ns) {ProcessConfigured:135(18217164ns) {StartInitialHandshake:157(18216602ns) {DropHandshakes:470(1006ns) {DropIncomingHandshake:442(166ns) DropOutgoingHandshake:457(177ns)} GenerateSessionVirtualId:988(302ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(342ns)}}}}}
2025-12-24T02:11:37.773317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393090955624:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:37.773330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393090955625:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:37.773336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393090955626:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:37.793234Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955662:2083] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.793256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955663:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.802894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770603:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.806744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241393639891461:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:37.807451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393090955696:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:37.807955Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241393639891400:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20642327ns) {Configure:104(20640549ns) {ProcessConfigured:135(19607925ns) {StartInitialHandshake:157(19607195ns) {DropHandshakes:470(807ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(192ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(1586ns) {SwitchToState:179(834ns)}}}}}
2025-12-24T02:11:37.806900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241394239395840:2102] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.808152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891471:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.808226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891472:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:37.808783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891469:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:37.808801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891470:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:37.808969Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241393639891462:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21733663ns) {Configure:104(21732754ns) {ProcessConfigured:135(20497142ns) {StartInitialHandshake:157(20496267ns) {DropHandshakes:470(140ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(557ns) {SwitchToState:179(247ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.810297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770629:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.810373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241395153318347:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:37.810479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241395153318348:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:37.807701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241393389691703:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24792799ns) {Configure:104(24791293ns) {ProcessConfigured:135(20499197ns) {StartInitialHandshake:157(20498415ns) {DropHandshakes:470(683ns) {DropIncomingHandshake:442(35ns) DropOutgoingHandshake:457(167ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(1383ns) {SwitchToState:179(800ns)}}}}}
2025-12-24T02:11:37.811223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891471:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.812920Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241394239395774:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29952136ns) {Configure:104(29950951ns) {ProcessConfigured:135(9412217ns) {StartInitialHandshake:157(9412090ns) {DropHandshakes:470(728ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(544ns) PrepareNewSessionHandshake:477(825ns) {SwitchToState:179(311ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.813751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394239395843:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65009
2025-12-24T02:11:37.813770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394239395844:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65009
2025-12-24T02:11:37.812793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318350:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.812823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318351:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.814713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241393090955691:2065] [node 4] ICP01 ready to work
2025-12-24T02:11:37.814800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393090955662:2083] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.817575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393090955696:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.823302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691783:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
2025-12-24T02:11:37.823012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891472:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.826706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691782:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
2025-12-24T02:11:37.826783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241393389691754:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.826798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241393389691754:2093] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.827034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891470:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.827169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241393389691782:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.827213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241393389691704:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.833192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318354:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.834666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318340:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.838142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241394239395829:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(44874389ns) {Configure:104(44873013ns) {ProcessConfigured:135(28142208ns) {StartInitialHandshake:157(28141130ns) {DropHandshakes:470(618ns) {DropIncomingHandshake:442(106ns) DropOutgoingHandshake:457(166ns)} GenerateSessionVirtualId:988(211ns) PrepareNewSessionHandshake:477(1674ns) {SwitchToState:179(791ns)}}}}}
2025-12-24T02:11:37.837421Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241393090955691:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22496977ns) {Configure:104(22496268ns) {ProcessConfigured:135(21967034ns) {StartInitialHandshake:157(21966316ns) {DropHandshakes:470(462ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(393ns) PrepareNewSessionHandshake:477(1419ns) {SwitchToState:179(852ns)}}}}}
2025-12-24T02:11:37.838301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393090955663:2084] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.837839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318350:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.838152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891483:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:37.840154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891474:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.840166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393639891474:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.840693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241393389691783:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.840756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241393389691703:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.842575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241393389691754:2093] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.842775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318351:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.842854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318354:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.842970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241393389691752:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.842991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.842999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241393389691752:2092] [node 6] ICP22 created new session: [5:7587241393389691787:2092]
2025-12-24T02:11:37.843008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241393389691787:2092] [node 6] ICS09 handshake done sender: [5:7587241393389691754:2093] self: [5:7587241393389691753:0] peer: [6:7587241393090955702:0] socket: 294 qp: -1
2025-12-24T02:11:37.843015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241393389691787:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:37.843025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.842972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891483:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.843078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891469:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.838248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395844:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.838391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395843:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.840926Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241394239395830:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(33912709ns) {Configure:104(33911803ns) {ProcessConfigured:135(33567550ns) {StartInitialHandshake:157(33566452ns) {DropHandshakes:470(1068ns) {DropIncomingHandshake:442(268ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1023ns) {SwitchToState:179(36ns)}}}}}
2025-12-24T02:11:37.841986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395848:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.841993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241394239395848:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.843220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241395153318274:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.843233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241395153318273:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.843848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691789:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.843503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241394239395848:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.843987Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395846:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.843991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241394239395846:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.843757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770629:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.843818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395119770550:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.844044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241394239395840:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.844055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.844064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241394239395840:2102] [node 5] ICP22 created new session: [1:7587241394239395858:2102]
2025-12-24T02:11:37.844071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394239395858:2102] [node 5] ICS09 handshake done sender: [1:7587241394239395848:2113] self: [1:7587241394239395847:0] peer: [5:7587241393389691786:0] socket: 325 qp: -1
2025-12-24T02:11:37.844079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394239395858:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:37.844099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.844783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241394239395846:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.844637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318358:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.845257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395153318354:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.845307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241395153318273:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.845322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.845330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241395153318273:2048] [node 1] ICP22 created new session: [3:7587241395153318359:2048]
2025-12-24T02:11:37.845336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395153318359:2048] [node 1] ICS09 handshake done sender: [3:7587241395153318354:2117] self: [3:7587241395153318357:0] peer: [1:7587241394239395845:0] socket: 326 qp: -1
2025-12-24T02:11:37.845342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395153318359:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.845361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.846860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241395153318347:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36249975ns) {Configure:104(36248976ns) {ProcessConfigured:135(35852329ns) {StartInitialHandshake:157(35851581ns) {DropHandshakes:470(959ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(670ns) {SwitchToState:179(165ns)}}}}}
2025-12-24T02:11:37.847430Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891468:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.847439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393639891468:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.847317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318338:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.847327Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241395153318338:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.851368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241393389691749:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.851369Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891477:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.851390Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393639891477:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.854849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393090955626:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.854895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241393090955626:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16519127ns) {IncomingHandshake:261(16518298ns) {DropIncomingHandshake:442(419ns) PrepareNewSessionHandshake:477(813ns) {SwitchToState:179(339ns)} GenerateSessionVirtualId:988(966ns) IssueIncomingHandshakeReply:193(16508749ns) {SwitchToState:179(1673ns)}}}
2025-12-24T02:11:37.855042Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393639891474:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.854997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241393389691783:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.852668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770631:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.858889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241394239395775:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.858933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.858947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241394239395775:2049] [node 3] ICP22 created new session: [1:7587241394239395862:2049]
2025-12-24T02:11:37.858957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394239395862:2049] [node 3] ICS09 handshake done sender: [1:7587241394239395846:2112] self: [1:7587241394239395845:0] peer: [3:7587241395153318357:0] socket: 324 qp: -1
2025-12-24T02:11:37.858967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394239395862:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.858978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.862564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395856:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.862571Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241394239395856:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.860752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395153318338:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.860893Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770632:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.862408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770632:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.862458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395119770549:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.863284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393639891477:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.863537Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395119770629:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.863648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395119770550:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.863678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.863694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395119770550:2049] [node 2] ICP22 created new session: [4:7587241395119770634:2049]
2025-12-24T02:11:37.863702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395119770634:2049] [node 2] ICS09 handshake done sender: [4:7587241395119770629:2116] self: [4:7587241395119770630:0] peer: [2:7587241393639891464:0] socket: 317 qp: -1
2025-12-24T02:11:37.863711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395119770634:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.863722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.867623Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241393639891399:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29231024ns) {Configure:104(29226197ns) {ProcessConfigured:135(28890992ns) {StartInitialHandshake:157(28890277ns) {DropHandshakes:470(937ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(137ns)} GenerateSessionVirtualId:988(321ns) PrepareNewSessionHandshake:477(1313ns) {SwitchToState:179(750ns)}}}}}
2025-12-24T02:11:37.867652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393639891461:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.867666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.867673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393639891461:2105] [node 5] ICP22 created new session: [2:7587241393639891489:2105]
2025-12-24T02:11:37.867680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393639891489:2105] [node 5] ICS09 handshake done sender: [2:7587241393639891474:2115] self: [2:7587241393639891473:0] peer: [5:7587241393389691784:0] socket: 314 qp: -1
2025-12-24T02:11:37.867056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241395153318336:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.867095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.867096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318364:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.867107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241395153318336:2105] [node 6] ICP22 created new session: [3:7587241395153318366:2105]
2025-12-24T02:11:37.867114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395153318366:2105] [node 6] ICS09 handshake done sender: [3:7587241395153318338:2108] self: [3:7587241395153318337:0] peer: [6:7587241393090955705:0] socket: 297 qp: -1
2025-12-24T02:11:37.867740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241393389691703:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.867123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395153318366:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:37.867146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.867762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.867771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241393389691703:2048] [node 1] ICP22 created new session: [5:7587241393389691793:2048]
2025-12-24T02:11:37.867778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241393389691793:2048] [node 1] ICS09 handshake done sender: [5:7587241393389691783:2117] self: [5:7587241393389691786:0] peer: [1:7587241394239395847:0] socket: 329 qp: -1
2025-12-24T02:11:37.867785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241393389691793:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.867793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.867938Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770636:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.867701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393639891489:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:37.867716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.867960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393639891462:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.867995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.868002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393639891462:2106] [node 4] ICP22 created new session: [2:7587241393639891492:2106]
2025-12-24T02:11:37.868007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393639891492:2106] [node 4] ICS09 handshake done sender: [2:7587241393639891477:2116] self: [2:7587241393639891464:0] peer: [4:7587241395119770630:0] socket: 315 qp: -1
2025-12-24T02:11:37.868012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393639891492:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:37.868018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.870154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241393639891399:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.868630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394239395865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65009
2025-12-24T02:11:37.870927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393639891468:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.870795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241393090955626:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241393090955663:2084] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.871290Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241393389691752:2092] [node 6] ICP35 event processing took too much time PendingConnection:279(28212214ns) {HandleHandshakeStatus:329(28211227ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(161ns) SwitchToState:179(173ns) ProcessPendingSessionEvents:495(23759637ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 23739816ns max# 23676750ns}}}
2025-12-24T02:11:37.871167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395842:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.871179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241394239395842:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.871222Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241394239395840:2102] [node 5] ICP35 event processing took too much time PendingConnection:279(27105884ns) {HandleHandshakeStatus:329(27104922ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(169ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(24268246ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 24258007ns max# 24223350ns}}}
2025-12-24T02:11:37.874772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770636:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.874833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241395119770551:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.874847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393639891400:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.874894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.874903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393639891400:2049] [node 3] ICP22 created new session: [2:7587241393639891497:2049]
2025-12-24T02:11:37.874912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393639891497:2049] [node 3] ICS09 handshake done sender: [2:7587241393639891468:2110] self: [2:7587241393639891463:0] peer: [3:7587241395153318356:0] socket: 313 qp: -1
2025-12-24T02:11:37.874923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393639891497:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:37.874934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.875747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395865:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.881052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241395119770549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241395119770632:2118] held: no
2025-12-24T02:11:37.881108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770639:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.876495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241394239395856:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.876626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241394239395830:2097] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.876648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.876657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241394239395830:2097] [node 4] ICP22 created new session: [1:7587241394239395868:2097]
2025-12-24T02:11:37.876663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394239395868:2097] [node 4] ICS09 handshake done sender: [1:7587241394239395856:2116] self: [1:7587241394239395841:0] peer: [4:7587241395119770633:0] socket: 343 qp: -1
2025-12-24T02:11:37.876671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394239395868:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:37.876681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.882170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955706:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.882186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318361:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.882217Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241395153318361:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.882297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395119770632:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.888637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241394239395842:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.888762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241394239395774:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.888783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.888795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241394239395774:2048] [node 2] ICP22 created new session: [1:7587241394239395872:2048]
2025-12-24T02:11:37.888805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394239395872:2048] [node 2] ICS09 handshake done sender: [1:7587241394239395842:2109] self: [1:7587241394239395838:0] peer: [2:7587241393639891494:0] socket: 323 qp: -1
2025-12-24T02:11:37.886795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241393389691745:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.889610Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241393389691703:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21788158ns) {HandleHandshakeStatus:329(21787480ns) {DropIncomingHandshake:442(180ns) DropOutgoingHandshake:457(1288ns) SwitchToState:179(129ns) ProcessPendingSessionEvents:495(18107146ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 18089849ns max# 18036471ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.891416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241395153318274:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241395153318351:2115] held: no
2025-12-24T02:11:37.888813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394239395872:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:37.888824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.888866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241394239395871:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:65009
2025-12-24T02:11:37.890937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395871:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.890980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241394239395774:2048] [node 2] ICP18 handshake (actor [1:7587241394239395871:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.891748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395119770549:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.891769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.891776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395119770549:2048] [node 1] ICP22 created new session: [4:7587241395119770640:2048]
2025-12-24T02:11:37.891783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395119770640:2048] [node 1] ICS09 handshake done sender: [4:7587241395119770632:2118] self: [4:7587241395119770633:0] peer: [1:7587241394239395841:0] socket: 345 qp: -1
2025-12-24T02:11:37.891790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395119770640:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.891798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.892050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770605:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.892056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241395119770605:2101] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.892536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770642:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.893933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395119770605:2101] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.894789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395153318361:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.891031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691792:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
2025-12-24T02:11:37.894122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241393389691704:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241393389691782:2116] held: no
2025-12-24T02:11:37.894482Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241393389691782:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.894648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241393389691704:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.894660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.894665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241393389691704:2049] [node 2] ICP22 created new session: [5:7587241393389691797:2049]
2025-12-24T02:11:37.894671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241393389691797:2049] [node 2] ICS09 handshake done sender: [5:7587241393389691782:2116] self: [5:7587241393389691784:0] peer: [2:7587241393639891473:0] socket: 319 qp: -1
2025-12-24T02:11:37.894676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241393389691797:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.894680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.891194Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241393639891399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241393639891483:2118] held: no
2025-12-24T02:11:37.895072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691791:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.898910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393090955662:2083] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.899042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393090955696:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.899055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.899066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393090955696:2077] [node 5] ICP22 created new session: [6:7587241393090955708:2077]
2025-12-24T02:11:37.899073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393090955708:2077] [node 5] ICS09 handshake done sender: [6:7587241393090955662:2083] self: [6:7587241393090955702:0] peer: [5:7587241393389691753:0] socket: 298 qp: -1
2025-12-24T02:11:37.899081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393090955708:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:37.899091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.899490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955707:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.898980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241395119770602:2098] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.899019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.899030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395119770602:2098] [node 5] ICP22 created new session: [4:7587241395119770645:2098]
2025-12-24T02:11:37.899038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395119770645:2098] [node 5] ICS09 handshake done sender: [4:7587241395119770605:2101] self: [4:7587241395119770604:0] peer: [5:7587241393389691795:0] socket: 291 qp: -1
2025-12-24T02:11:37.899047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395119770645:2098] [node 5] ICS10 traffic start
2025-12-24T02:11:37.899057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.899244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395119770636:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.902913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241395119770551:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.902937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.902945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395119770551:2050] [node 3] ICP22 created new session: [4:7587241395119770647:2050]
2025-12-24T02:11:37.902954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395119770647:2050] [node 3] ICS09 handshake done sender: [4:7587241395119770636:2119] self: [4:7587241395119770638:0] peer: [3:7587241395153318349:0] socket: 306 qp: -1
2025-12-24T02:11:37.902962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395119770647:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.902971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.903610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393090955707:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.903676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393090955624:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.908111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691796:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
2025-12-24T02:11:37.907044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241394239395853:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.907059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241394239395853:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.906829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241393639891499:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39416
2025-12-24T02:11:37.910763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393639891483:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.910915Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241393389691749:2090] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.910824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241395153318347:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.910860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.910871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241395153318347:2112] [node 4] ICP22 created new session: [3:7587241395153318370:2112]
2025-12-24T02:11:37.910878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395153318370:2112] [node 4] ICS09 handshake done sender: [3:7587241395153318361:2119] self: [3:7587241395153318349:0] peer: [4:7587241395119770638:0] socket: 322 qp: -1
2025-12-24T02:11:37.910884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395153318370:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:37.910894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.911097Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241395153318347:2112] [node 4] ICP35 event processing took too much time PendingConnection:279(16084730ns) {HandleHandshakeStatus:329(16084082ns) {DropIncomingHandshake:442(7114ns) DropOutgoingHandshake:457(173ns) SwitchToState:179(205ns) ProcessPendingSessionEvents:495(91341ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 80692ns max# 36537ns}}}
2025-12-24T02:11:37.911259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241395153318371:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23392
2025-12-24T02:11:37.915001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395153318351:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:37.915172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241395153318274:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.915209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:37.915221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241395153318274:2049] [node 2] ICP22 created new session: [3:7587241395153318373:2049]
2025-12-24T02:11:37.915230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395153318373:2049] [node 2] ICS09 handshake done sender: [3:7587241395153318351:2115] self: [3:7587241395153318356:0] peer: [2:7587241393639891463:0] socket: 320 qp: -1
2025-12-24T02:11:37.915242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395153318373:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:37.915254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:37.915766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241393090955624:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241393090955707:2118] held: no
2025-12-24T02:11:37.914783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241394239395853:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.915191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241394239395829:2100] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.915212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.915219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241394239395829:2100] [node 6] ICP22 created new session: [1:7587241394239395875:2100]
2025-12-24T02:11:37.915225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241394239395875:2100] [node 6] ICS09 handshake done sender: [1:7587241394239395853:2115] self: [1:7587241394239395839:0] peer: [6:7587241393090955712:0] socket: 335 qp: -1
2025-12-24T02:11:37.915231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241394239395875:2100] [node 6] ICS10 traffic start
2025-12-24T02:11:37.915240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.915190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241393389691791:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.915234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241393389691705:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.919068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955713:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.919985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955714:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.920068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955715:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.920106Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955716:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.922416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318353:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.922435Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241395153318353:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.922848Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241395119770551:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(19878714ns) {HandleHandshakeStatus:329(19877660ns) {DropIncomingHandshake:442(418ns) DropOutgoingHandshake:457(2535ns) SwitchToState:179(402ns) ProcessPendingSessionEvents:495(19772534ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 19759662ns max# 19726461ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.923887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241395119770649:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20248
2025-12-24T02:11:37.923506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241393389691745:2088] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6223) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241393389691749:2090] held: no
2025-12-24T02:11:37.923565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241393389691745:2088] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.923576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.923581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241393389691745:2088] [node 4] ICP22 created new session: [5:7587241393389691800:2088]
2025-12-24T02:11:37.923587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241393389691800:2088] [node 4] ICS09 handshake done sender: [5:7587241393389691749:2090] self: [5:7587241393389691795:0] peer: [4:7587241395119770604:0] socket: 292 qp: -1
2025-12-24T02:11:37.923594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241393389691800:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:37.923602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.926787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241393639891399:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.926809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.926817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393639891399:2048] [node 1] ICP22 created new session: [2:7587241393639891500:2048]
2025-12-24T02:11:37.926824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393639891500:2048] [node 1] ICS09 handshake done sender: [2:7587241393639891483:2118] self: [2:7587241393639891494:0] peer: [1:7587241394239395838:0] socket: 328 qp: -1
2025-12-24T02:11:37.926831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393639891500:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.926839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.926882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241395153318364:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.943110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241395153318353:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:37.944093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241395153318348:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.944114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:37.944125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241395153318348:2113] [node 5] ICP22 created new session: [3:7587241395153318377:2113]
2025-12-24T02:11:37.944133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241395153318377:2113] [node 5] ICS09 handshake done sender: [3:7587241395153318353:2116] self: [3:7587241395153318352:0] peer: [5:7587241393389691799:0] socket: 337 qp: -1
2025-12-24T02:11:37.944142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241395153318377:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:37.944152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:37.950820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393090955715:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.950921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241393090955714:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.950934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393090955663:2084] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:37.951069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393090955707:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:37.954819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770649:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.954812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393090955624:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.954813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393090955626:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.954827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:37.954827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.954838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393090955624:2048] [node 1] ICP22 created new session: [6:7587241393090955718:2048]
2025-12-24T02:11:37.954838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393090955626:2050] [node 3] ICP22 created new session: [6:7587241393090955717:2050]
2025-12-24T02:11:37.954844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393090955717:2050] [node 3] ICS09 handshake done sender: [6:7587241393090955663:2084] self: [6:7587241393090955705:0] peer: [3:7587241395153318337:0] socket: 299 qp: -1
2025-12-24T02:11:37.954844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393090955718:2048] [node 1] ICS09 handshake done sender: [6:7587241393090955707:2118] self: [6:7587241393090955712:0] peer: [1:7587241394239395839:0] socket: 349 qp: -1
2025-12-24T02:11:37.954851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393090955717:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.954851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393090955718:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:37.954858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.954859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:37.955011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393090955691:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.955024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241393090955625:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:37.966826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241393389691705:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24667) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241393389691791:2119] held: no
2025-12-24T02:11:37.966957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241395153318348:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(22753830ns) {HandleHandshakeStatus:329(22752539ns) {DropIncomingHandshake:442(4032ns) DropOutgoingHandshake:457(651ns) SwitchToState:179(323ns) ProcessPendingSessionEvents:495(22640212ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22617717ns max# 22530643ns}}}
2025-12-24T02:11:37.970745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241395119770627:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.970767Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241395119770627:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.970719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241393639891480:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:37.970745Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241393639891480:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:37.979450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241395119770627:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.979588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241395119770613:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.979612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.979622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241395119770613:2103] [node 6] ICP22 created new session: [4:7587241395119770652:2103]
2025-12-24T02:11:37.979631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241395119770652:2103] [node 6] ICS09 handshake done sender: [4:7587241395119770627:2115] self: [4:7587241395119770614:0] peer: [6:7587241393090955721:0] socket: 350 qp: -1
2025-12-24T02:11:37.979639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241395119770652:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:37.979650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.983287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241393389691802:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40550
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.990779Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241393090955691:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6223) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241393090955715:2121] held: no
2025-12-24T02:11:37.990989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955723:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:37.991255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393090955715:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:37.991591Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241393639891480:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:37.991333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393090955691:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.991347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:37.991356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393090955691:2065] [node 4] ICP22 created new session: [6:7587241393090955724:2065]
2025-12-24T02:11:37.991364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241393090955724:2065] [node 4] ICS09 handshake done sender: [6:7587241393090955715:2121] self: [6:7587241393090955721:0] peer: [4:7587241395119770614:0] socket: 308 qp: -1
2025-12-24T02:11:37.991372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241393090955724:2065] [node 4] ICS10 traffic start
2025-12-24T02:11:37.991381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:37.993127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241393639891457:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:37.993152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:37.993159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241393639891457:2104] [node 6] ICP22 created new session: [2:7587241393639891503:2104]
2025-12-24T02:11:37.993165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241393639891503:2104] [node 6] ICS09 handshake done sender: [2:7587241393639891480:2117] self: [2:7587241393639891467:0] peer: [6:7587241393090955722:0] socket: 311 qp: -1
2025-12-24T02:11:37.993172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241393639891503:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:37.993180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:37.994957Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241393389691791:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:37.998921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241393389691705:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:37.998940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:37.998951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241393389691705:2050] [node 3] ICP22 created new session: [5:7587241393389691803:2050]
2025-12-24T02:11:37.998958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241393389691803:2050] [node 3] ICS09 handshake done sender: [5:7587241393389691791:2119] self: [5:7587241393389691799:0] peer: [3:7587241395153318352:0] socket: 316 qp: -1
2025-12-24T02:11:37.998965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241393389691803:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:37.998972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:37.994914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241393090955625:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241393090955714:2120] held: no
2025-12-24T02:11:38.006741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241393090955726:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42516
2025-12-24T02:11:38.007036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241393090955714:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.007181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241393090955625:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.007192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.007200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241393090955625:2049] [node 2] ICP22 created new session: [6:7587241397385923023:2049]
2025-12-24T02:11:38.007206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241397385923023:2049] [node 2] ICS09 handshake done sender: [6:7587241393090955714:2120] self: [6:7587241393090955722:0] peer: [2:7587241393639891467:0] socket: 289 qp: -1
2025-12-24T02:11:38.007213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241397385923023:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.007221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.055196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:38.055297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241393639891500:2048] [node 1] ICS07 socket disconnect 328 reason# EndOfStream
2025-12-24T02:11:38.055305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241393639891500:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.055338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.055354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241393639891500:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:38.057604Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241393639891399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:38.057620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241393639891500:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:38.057875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241393639891399:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.057887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241393639891500:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.057892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241393639891399:2048] [node 1] ICP30 unregister session Session# [2:7587241393639891500:2048] VirtualId# [2:7587241393639891494:0]
2025-12-24T02:11:38.057924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.057930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241393639891399:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.057934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.058781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:38.058882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241395153318359:2048] [node 1] ICS01 socket: 326 reason# EndOfStream
2025-12-24T02:11:38.058890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241395153318273:2048] [node 1] ICP30 unregister session Session# [3:7587241395153318359:2048] VirtualId# [3:7587241395153318357:0]
2025-12-24T02:11:38.058897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241395153318359:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.058937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.058962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.060922Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:38.061003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393090955718:2048] [node 1] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:38.061010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241393090955624:2048] [node 1] ICP30 unregister session Session# [6:7587241393090955718:2048] VirtualId# [6:7587241393090955712:0]
2025-12-24T02:11:38.061016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393090955718:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.061055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.061083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.062797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:38.062880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241395119770640:2048] [node 1] ICS07 socket disconnect 345 reason# EndOfStream
2025-12-24T02:11:38.062885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241395119770640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.062925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.062948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241395119770640:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:38.063866Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241395119770549:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.063876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241395119770640:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.063881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241395119770549:2048] [node 1] ICP30 unregister session Session# [4:7587241395119770640:2048] VirtualId# [4:7587241395119770633:0]
2025-12-24T02:11:38.063905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.063912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241395119770549:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.063915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61949) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.063180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:38.063239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241393389691793:2048] [node 1] ICS01 socket: 329 reason# EndOfStream
2025-12-24T02:11:38.063244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241393389691703:2048] [node 1] ICP30 unregister session Session# [5:7587241393389691793:2048] VirtualId# [5:7587241393389691786:0]
2025-12-24T02:11:38.063249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241393389691793:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.063273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.063294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.074488Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:38.074583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241395153318373:2049] [node 2] ICS07 socket disconnect 320 reason# EndOfStream
2025-12-24T02:11:38.074590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241395153318373:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.074623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.074641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241395153318373:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:38.077701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241395119770549:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:38.087343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241395153318274:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:38.087362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241395153318373:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:38.093383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:38.093506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241397385923023:2049] [node 2] ICS07 socket disconnect 289 reason# EndOfStream
2025-12-24T02:11:38.093513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241397385923023:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.093558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.093578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241397385923023:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:38.093785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:38.093845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241395119770634:2049] [node 2] ICS07 socket disconnect 317 reason# EndOfStream
2025-12-24T02:11:38.093851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241395119770634:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.093878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.093898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241395119770634:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:38.094394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241395119770550:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.094405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241395119770634:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.094410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241395119770550:2049] [node 2] ICP30 unregister session Session# [4:7587241395119770634:2049] VirtualId# [4:7587241395119770630:0]
2025-12-24T02:11:38.094437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.094444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241395119770550:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.094447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.094751Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241393090955625:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.094762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241397385923023:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.094767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241393090955625:2049] [node 2] ICP30 unregister session Session# [6:7587241397385923023:2049] VirtualId# [6:7587241393090955722:0]
2025-12-24T02:11:38.094779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.094784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241393090955625:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.094786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.099026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:38.099123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241393389691797:2049] [node 2] ICS07 socket disconnect 319 reason# EndOfStream
2025-12-24T02:11:38.099130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241393389691797:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.099176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.099198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241393389691797:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:38.102829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241393389691704:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.102855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241393389691797:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.102864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241393389691704:2049] [node 2] ICP30 unregister session Session# [5:7587241393389691797:2049] VirtualId# [5:7587241393389691784:0]
2025-12-24T02:11:38.102912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.102920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241393389691704:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.102922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63641) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.106730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241395119770550:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:38.106780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241393090955625:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:38.114865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241393389691704:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:38.126057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:38.126156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241393389691803:2050] [node 3] ICS07 socket disconnect 316 reason# EndOfStream
2025-12-24T02:11:38.126164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241393389691803:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.126216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:38.126238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241393389691803:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:38.126614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241393389691705:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24667) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.126627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241393389691803:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.126632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241393389691705:2050] [node 3] ICP30 unregister session Session# [5:7587241393389691803:2050] VirtualId# [5:7587241393389691799:0]
2025-12-24T02:11:38.126660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:38.126667Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241393389691705:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.126670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24667) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.134871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:38.134967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393090955717:2050] [node 3] ICS01 socket: 299 reason# EndOfStream
2025-12-24T02:11:38.134979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241393090955626:2050] [node 3] ICP30 unregister session Session# [6:7587241393090955717:2050] VirtualId# [6:7587241393090955705:0]
2025-12-24T02:11:38.134987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393090955717:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.135046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:38.135080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:38.136214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241393389691705:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:38.150856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:38.150958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241393090955724:2065] [node 4] ICS01 socket: 308 reason# EndOfStream
2025-12-24T02:11:38.150967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241393090955691:2065] [node 4] ICP30 unregister session Session# [6:7587241393090955724:2065] VirtualId# [6:7587241393090955721:0]
2025-12-24T02:11:38.150974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241393090955724:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.151021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:38.151045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:38.199461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241397088570743:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:38.199470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241397088570744:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:38.210791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241400803808963:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.210807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241400803808964:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:38.206853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399961282639:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.206864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399961282640:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.222854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241399678713926:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.222870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241399678713927:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.222876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241399678713928:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:38.218850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397081430907:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.218864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397081430908:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.218869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397081430909:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:38.251128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241399678713988:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.251323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241397088570785:2085] [node 6] ICP01 ready to work
2025-12-24T02:11:38.251795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241397088570789:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:38.252250Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241399678713988:2108] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.252277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241399678713988:2108] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.252281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.251019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399961282668:2069] [node 5] ICP01 ready to work
2025-12-24T02:11:38.251066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399961282671:2072] [node 6] ICP01 ready to work
2025-12-24T02:11:38.250874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241400803808986:2066] [node 4] ICP01 ready to work
2025-12-24T02:11:38.251967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803808995:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
2025-12-24T02:11:38.254107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241400803809000:2076] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.257230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241399678713999:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:38.255053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430971:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.258119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397081430974:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:38.258257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397081430975:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:38.258949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241400803809009:2079] [node 5] ICP01 ready to work
2025-12-24T02:11:38.262758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241399678713988:2108] [node 6] ICP33 wake up from error state
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.263116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241397088570798:2092] [node 4] ICP01 ready to work
2025-12-24T02:11:38.263731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241397088570799:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42513
2025-12-24T02:11:38.263841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430971:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.270779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241399961282668:2069] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19656475ns) {Configure:104(19655961ns) {ProcessConfigured:135(11267467ns) {StartInitialHandshake:157(11266022ns) {DropHandshakes:470(698ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(183ns) PrepareNewSessionHandshake:477(805ns) {SwitchToState:179(250ns)}}}}}
2025-12-24T02:11:38.271383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282698:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.271460Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282699:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.271503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282700:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.271651Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282697:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.272538Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241397088570800:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42513
2025-12-24T02:11:38.272654Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241397088570785:2085] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.272685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7587241397088570785:2085] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.272688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29116) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.257968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241400767210156:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.257980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241400767210157:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.257986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241400767210158:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:38.258116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241400767210217:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:38.274870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803808995:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.274857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282699:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.291239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241400803809009:2079] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(30826797ns) {Configure:104(30825499ns) {ProcessConfigured:135(30373355ns) {StartInitialHandshake:157(30372418ns) {DropHandshakes:470(339ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(180ns) PrepareNewSessionHandshake:477(1700ns) {SwitchToState:179(892ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.295578Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241400767210217:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(37284500ns) {Configure:104(37283775ns) {ProcessConfigured:135(36863609ns) {StartInitialHandshake:157(36862720ns) {DropHandshakes:470(550ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(159ns) PrepareNewSessionHandshake:477(2305ns) {SwitchToState:179(1303ns)}}}}}
2025-12-24T02:11:38.295254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430986:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.299466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803809045:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
2025-12-24T02:11:38.300149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803808981:2064] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.300153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241400803808981:2064] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.300834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803809046:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
2025-12-24T02:11:38.300999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241400803808981:2064] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.301088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241400803808964:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.301109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.301115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241400803808964:2049] [node 3] ICP22 created new session: [2:7587241400803809048:2049]
2025-12-24T02:11:38.301122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241400803809048:2049] [node 3] ICS09 handshake done sender: [2:7587241400803808981:2064] self: [2:7587241400803808980:0] peer: [3:7587241399961282720:0] socket: 291 qp: -1
2025-12-24T02:11:38.301130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241400803809048:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:38.301152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.299260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241397081430974:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(40384425ns) {Configure:104(40382873ns) {ProcessConfigured:135(39978832ns) {StartInitialHandshake:157(39977997ns) {DropHandshakes:470(508ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(2034ns) {SwitchToState:179(767ns)}}}}}
2025-12-24T02:11:38.299553Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241399961282640:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(48468774ns) {Configure:104(48467965ns) {ProcessConfigured:135(40296956ns) {StartInitialHandshake:157(40295654ns) {DropHandshakes:470(509ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(153ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(220ns)}}}}}
2025-12-24T02:11:38.299612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282698:2099] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.299798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399961282640:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.299966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282718:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.301667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282721:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.301814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399961282698:2099] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.301862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399961282640:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.301872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.301878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399961282640:2049] [node 2] ICP22 created new session: [3:7587241399961282722:2049]
2025-12-24T02:11:38.301884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399961282722:2049] [node 2] ICS09 handshake done sender: [3:7587241399961282698:2099] self: [3:7587241399961282720:0] peer: [2:7587241400803808980:0] socket: 317 qp: -1
2025-12-24T02:11:38.301889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399961282722:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.301905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.302877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714005:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.303727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803809010:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
2025-12-24T02:11:38.306718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399961282686:2080] [node 4] ICP01 ready to work
2025-12-24T02:11:38.306767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430987:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.306837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397081430907:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.306995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241397088570812:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42513
2025-12-24T02:11:38.306903Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241397081430907:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(42967199ns) {IncomingHandshake:261(42966521ns) {DropIncomingHandshake:442(42ns) PrepareNewSessionHandshake:477(353ns) {SwitchToState:179(284ns)} GenerateSessionVirtualId:988(232ns) IssueIncomingHandshakeReply:193(42961712ns) {SwitchToState:179(1423ns)}}}
2025-12-24T02:11:38.307015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714005:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.307096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241399678713927:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.307128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282700:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.310953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570800:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.310983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570812:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.310799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803809010:2084] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.310960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803808990:2070] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.310966Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241400803808990:2070] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.314786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241397088570743:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(63939208ns) {Configure:104(63938130ns) {ProcessConfigured:135(63600383ns) {StartInitialHandshake:157(63599568ns) {DropHandshakes:470(399ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(561ns) {SwitchToState:179(246ns)}}}}}
2025-12-24T02:11:38.314867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570799:2093] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.315423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714007:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.319710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570793:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.319737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241397088570793:2090] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.320080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714007:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.320150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241399678713926:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.322722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241400767210221:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.323485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210232:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.323504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210233:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.329028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282697:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.330817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241399961282686:2080] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23632675ns) {Configure:104(23631961ns) {ProcessConfigured:135(23266193ns) {StartInitialHandshake:157(23247237ns) {DropHandshakes:470(1285ns) {DropIncomingHandshake:442(196ns) DropOutgoingHandshake:457(225ns)} GenerateSessionVirtualId:988(541ns) PrepareNewSessionHandshake:477(1443ns) {SwitchToState:179(696ns)}} IncomingHandshake:261(7662ns) {DropIncomingHandshake:442(44ns) PrepareNewSessionHandshake:477(124ns) {SwitchToState:179(74ns)} GenerateSessionVirtualId:988(382ns) IssueIncomingHandshakeReply:193(311ns)}}}}
2025-12-24T02:11:38.330855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241400767210233:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.330908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241400767210157:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.331090Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241400767210156:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23886770ns) {Configure:104(23886047ns) {ProcessConfigured:135(23511154ns) {StartInitialHandshake:157(23510247ns) {DropHandshakes:470(601ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(117ns)} GenerateSessionVirtualId:988(415ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(56ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.334960Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714009:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.336129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803809002:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.336144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241400803809002:2077] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.336982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241400803809002:2077] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.337220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241400803808990:2070] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.337304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241400803808986:2066] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.337326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.337334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241400803808986:2066] [node 4] ICP22 created new session: [2:7587241400803809054:2066]
2025-12-24T02:11:38.337343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241400803809054:2066] [node 4] ICS09 handshake done sender: [2:7587241400803808990:2070] self: [2:7587241400803808989:0] peer: [4:7587241399678714006:0] socket: 296 qp: -1
2025-12-24T02:11:38.337349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241400803809054:2066] [node 4] ICS10 traffic start
2025-12-24T02:11:38.337358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.337591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241400767210232:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.337727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241400767210158:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.337807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210236:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.338013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241400767210233:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.338091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241400767210157:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.338111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.338117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241400767210157:2049] [node 2] ICP22 created new session: [6:7587241400767210238:2049]
2025-12-24T02:11:38.338124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241400767210238:2049] [node 2] ICS09 handshake done sender: [6:7587241400767210233:2114] self: [6:7587241400767210234:0] peer: [2:7587241400803809001:0] socket: 338 qp: -1
2025-12-24T02:11:38.338132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241400767210238:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.338152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.338816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570814:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.338829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241397088570814:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.339029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714009:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.339052Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241399678713927:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241399678714005:2115] held: no
2025-12-24T02:11:38.339195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241399678713928:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.339032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7587241397088570785:2085] [node 6] ICP33 wake up from error state
2025-12-24T02:11:38.339062Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:309: Proxy [1:7587241397088570785:2085] [node 6] ICP35 event processing took too much time HoldByError:309(27923473ns) {WakeupFromErrorState:799(27922515ns) {SwitchToInitialState:202(639ns) {SwitchToState:179(437ns)}}}
2025-12-24T02:11:38.339147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241397088570829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42513
2025-12-24T02:11:38.342916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241400803809000:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.342941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.342950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241400803809000:2076] [node 6] ICP22 created new session: [2:7587241400803809057:2076]
2025-12-24T02:11:38.342959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241400803809057:2076] [node 6] ICS09 handshake done sender: [2:7587241400803809002:2077] self: [2:7587241400803809001:0] peer: [6:7587241400767210234:0] socket: 334 qp: -1
2025-12-24T02:11:38.342968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241400803809057:2076] [node 6] ICS10 traffic start
2025-12-24T02:11:38.342978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.342979Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241399961282639:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(88199599ns) {Configure:104(88197345ns) {ProcessConfigured:135(79849333ns) {StartInitialHandshake:157(79848490ns) {DropHandshakes:470(362ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1289ns) {SwitchToState:179(716ns)}}}}}
2025-12-24T02:11:38.343075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282726:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.343079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241399961282726:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.343829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399961282639:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.346328Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241400767210221:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23425774ns) {Configure:104(23423921ns) {ProcessConfigured:135(23026621ns) {StartInitialHandshake:157(23026114ns) {DropHandshakes:470(1172ns) {DropIncomingHandshake:442(136ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(347ns) PrepareNewSessionHandshake:477(1445ns) {SwitchToState:179(790ns)}}}}}
2025-12-24T02:11:38.346801Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399961282726:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.346921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241399961282686:2080] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.346945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.346954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399961282686:2080] [node 4] ICP22 created new session: [3:7587241399961282732:2080]
2025-12-24T02:11:38.346963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399961282732:2080] [node 4] ICS09 handshake done sender: [3:7587241399961282726:2119] self: [3:7587241399961282724:0] peer: [4:7587241399678714010:0] socket: 332 qp: -1
2025-12-24T02:11:38.346970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399961282732:2080] [node 4] ICS10 traffic start
2025-12-24T02:11:38.346993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.351112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282696:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.351128Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241399961282696:2097] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.354715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570776:2079] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.354732Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241397088570776:2079] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.355409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241397088570832:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42513
2025-12-24T02:11:38.356625Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241397088570785:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17489756ns) {Configure:104(17488972ns) {ProcessConfigured:135(17481723ns) {StartInitialHandshake:157(17480996ns) {DropHandshakes:470(434ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(425ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(266ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.358952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714011:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.359098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714012:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.359047Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241397088570776:2079] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.359290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241397088570814:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.359404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241397088570793:2090] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.359475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241397088570744:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.359490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.359496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241397088570744:2049] [node 3] ICP22 created new session: [1:7587241397088570836:2049]
2025-12-24T02:11:38.359502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241397088570836:2049] [node 3] ICS09 handshake done sender: [1:7587241397088570776:2079] self: [1:7587241397088570775:0] peer: [3:7587241399961282731:0] socket: 290 qp: -1
2025-12-24T02:11:38.359507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241397088570836:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:38.359526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:38.359903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241397088570798:2092] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.359915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.359920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241397088570798:2092] [node 4] ICP22 created new session: [1:7587241397088570838:2092]
2025-12-24T02:11:38.359924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241397088570838:2092] [node 4] ICS09 handshake done sender: [1:7587241397088570814:2107] self: [1:7587241397088570813:0] peer: [4:7587241399678714008:0] socket: 331 qp: -1
2025-12-24T02:11:38.359927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241397088570838:2092] [node 4] ICS10 traffic start
2025-12-24T02:11:38.359933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.360394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430986:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.360450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397081430909:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.358843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399961282735:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43298
2025-12-24T02:11:38.359210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399961282697:2098] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.359272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399961282639:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.359289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.359297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399961282639:2048] [node 1] ICP22 created new session: [3:7587241399961282736:2048]
2025-12-24T02:11:38.359304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399961282736:2048] [node 1] ICS09 handshake done sender: [3:7587241399961282697:2098] self: [3:7587241399961282731:0] peer: [1:7587241397088570775:0] socket: 316 qp: -1
2025-12-24T02:11:38.359312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399961282736:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.359321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.362912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714013:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.362383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241397088570789:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.362407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.362416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241397088570789:2086] [node 5] ICP22 created new session: [1:7587241397088570840:2086]
2025-12-24T02:11:38.362423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241397088570840:2086] [node 5] ICS09 handshake done sender: [1:7587241397088570793:2090] self: [1:7587241397088570792:0] peer: [5:7587241397081430985:0] socket: 299 qp: -1
2025-12-24T02:11:38.362431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241397088570840:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:38.362439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.370793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399961282691:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.374897Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397081430907:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4518) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241397081430971:2109] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.382877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803809059:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.393576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803809043:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.393593Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241400803809043:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.394441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241400803809043:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.394539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241400803809009:2079] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.394565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.394572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241400803809009:2079] [node 5] ICP22 created new session: [2:7587241400803809061:2079]
2025-12-24T02:11:38.392610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430990:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.394578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241400803809061:2079] [node 5] ICS09 handshake done sender: [2:7587241400803809043:2116] self: [2:7587241400803809016:0] peer: [5:7587241397081430995:0] socket: 343 qp: -1
2025-12-24T02:11:38.394582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241400803809061:2079] [node 5] ICS10 traffic start
2025-12-24T02:11:38.392660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430991:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.394590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.392994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397081430971:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.393053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397081430907:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.393069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.393077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397081430907:2048] [node 1] ICP22 created new session: [5:7587241397081430993:2048]
2025-12-24T02:11:38.393084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397081430993:2048] [node 1] ICS09 handshake done sender: [5:7587241397081430971:2109] self: [5:7587241397081430985:0] peer: [1:7587241397088570792:0] socket: 300 qp: -1
2025-12-24T02:11:38.393091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397081430993:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.393100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.393452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430990:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.393475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397081430908:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.370810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241399961282691:2093] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.392239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210242:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.392311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210243:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.370896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399961282696:2097] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.375661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241399961282671:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.375677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.375686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399961282671:2072] [node 6] ICP22 created new session: [3:7587241399961282739:2072]
2025-12-24T02:11:38.375703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399961282739:2072] [node 6] ICS09 handshake done sender: [3:7587241399961282696:2097] self: [3:7587241399961282695:0] peer: [6:7587241400767210237:0] socket: 321 qp: -1
2025-12-24T02:11:38.375712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399961282739:2072] [node 6] ICS10 traffic start
2025-12-24T02:11:38.375726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.395730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241399678713928:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241399678714009:2117] held: no
2025-12-24T02:11:38.395883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241399678714009:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.396254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241399678714007:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.395023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241397088570798:2092] [node 4] ICP35 event processing took too much time PendingConnection:279(34996178ns) {HandleHandshakeStatus:329(34995219ns) {DropIncomingHandshake:442(3844ns) DropOutgoingHandshake:457(78ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(34945842ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15716615ns max# 15614584ns}}}
2025-12-24T02:11:38.396260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241399961282671:2072] [node 6] ICP35 event processing took too much time PendingConnection:279(20485139ns) {HandleHandshakeStatus:329(20483956ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(231ns) SwitchToState:179(223ns) ProcessPendingSessionEvents:495(20355291ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 111113ns max# 13687ns}}}
2025-12-24T02:11:38.396462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397081430908:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24329) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241397081430990:2118] held: no
2025-12-24T02:11:38.396433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241399678713926:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.396452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.396459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241399678713926:2048] [node 1] ICP22 created new session: [4:7587241399678714014:2048]
2025-12-24T02:11:38.396465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241399678714014:2048] [node 1] ICS09 handshake done sender: [4:7587241399678714007:2116] self: [4:7587241399678714008:0] peer: [1:7587241397088570813:0] socket: 337 qp: -1
2025-12-24T02:11:38.396472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241399678714014:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.396480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.397974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241399678714005:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.398138Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241399678713926:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:38.398198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241399678713928:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.398209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.398216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241399678713928:2050] [node 3] ICP22 created new session: [4:7587241399678714016:2050]
2025-12-24T02:11:38.398222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241399678714016:2050] [node 3] ICS09 handshake done sender: [4:7587241399678714009:2117] self: [4:7587241399678714010:0] peer: [3:7587241399961282724:0] socket: 339 qp: -1
2025-12-24T02:11:38.398229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241399678714016:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.398245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:38.398765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241399678713927:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.398785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.398793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241399678713927:2049] [node 2] ICP22 created new session: [4:7587241399678714019:2049]
2025-12-24T02:11:38.398798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241399678714019:2049] [node 2] ICS09 handshake done sender: [4:7587241399678714005:2115] self: [4:7587241399678714006:0] peer: [2:7587241400803808989:0] socket: 298 qp: -1
2025-12-24T02:11:38.398804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241399678714019:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.398810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.399370Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241397088570744:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:38.400010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399961282691:2093] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.400231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241399961282668:2069] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.400250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.400269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399961282668:2069] [node 5] ICP22 created new session: [3:7587241399961282742:2069]
2025-12-24T02:11:38.400274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399961282742:2069] [node 5] ICS09 handshake done sender: [3:7587241399961282691:2093] self: [3:7587241399961282681:0] peer: [5:7587241397081430989:0] socket: 320 qp: -1
2025-12-24T02:11:38.400278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399961282742:2069] [node 5] ICS10 traffic start
2025-12-24T02:11:38.400285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.397698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210244:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.398862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241400767210232:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.402912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430987:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.402953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397081430909:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8873) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241397081430986:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.406655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430996:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.406948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430997:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.406875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241400767210158:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.406897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.406910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241400767210158:2050] [node 3] ICP22 created new session: [6:7587241400767210245:2050]
2025-12-24T02:11:38.406918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241400767210245:2050] [node 3] ICS09 handshake done sender: [6:7587241400767210232:2113] self: [6:7587241400767210237:0] peer: [3:7587241399961282695:0] socket: 323 qp: -1
2025-12-24T02:11:38.406924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241400767210245:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.406933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.411274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714021:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.410880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081430998:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.411085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397081430986:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.414944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397081430990:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.415500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241400767210242:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.416228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241400767210156:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.418881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570835:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.418894Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241397088570835:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.419608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241400803809059:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.419662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241400803808963:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.419803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241397088570835:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.419936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241400767210244:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.419930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570826:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.419937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241397088570826:2117] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.420140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241397088570785:2085] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.420149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.420157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241397088570785:2085] [node 6] ICP22 created new session: [1:7587241397088570845:2085]
2025-12-24T02:11:38.420163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241397088570845:2085] [node 6] ICS09 handshake done sender: [1:7587241397088570835:2120] self: [1:7587241397088570830:0] peer: [6:7587241400767210247:0] socket: 353 qp: -1
2025-12-24T02:11:38.420169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241397088570845:2085] [node 6] ICS10 traffic start
2025-12-24T02:11:38.420176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.420273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241397088570829:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.418846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430998:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.418919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397081430909:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.418933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.418942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397081430909:2050] [node 3] ICP22 created new session: [5:7587241397081431001:2050]
2025-12-24T02:11:38.418949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397081431001:2050] [node 3] ICS09 handshake done sender: [5:7587241397081430986:2115] self: [5:7587241397081430989:0] peer: [3:7587241399961282681:0] socket: 324 qp: -1
2025-12-24T02:11:38.418957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397081431001:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.418965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:38.422607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241399961282671:2072] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:38.422348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241400767210217:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.431717Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714022:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.432733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241397088570826:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.432875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241397088570743:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.432898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.432906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241397088570743:2048] [node 2] ICP22 created new session: [1:7587241397088570848:2048]
2025-12-24T02:11:38.432913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241397088570848:2048] [node 2] ICS09 handshake done sender: [1:7587241397088570826:2117] self: [1:7587241397088570784:0] peer: [2:7587241400803809063:0] socket: 354 qp: -1
2025-12-24T02:11:38.432922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241397088570848:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:38.432932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.431891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210250:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.434970Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241400767210158:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(27975841ns) {HandleHandshakeStatus:329(27974927ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(1429ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(27876244ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27849467ns max# 27766306ns}}}
2025-12-24T02:11:38.435362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397081430982:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.435376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241397081430982:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.435750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397081430908:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.435765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.435771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397081430908:2049] [node 2] ICP22 created new session: [5:7587241397081431003:2049]
2025-12-24T02:11:38.435780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397081431003:2049] [node 2] ICS09 handshake done sender: [5:7587241397081430990:2118] self: [5:7587241397081430995:0] peer: [2:7587241400803809016:0] socket: 346 qp: -1
2025-12-24T02:11:38.435788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397081431003:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.435797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.436150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241397088570785:2085] [node 6] ICP18 handshake (actor [1:7587241397088570829:2118]) is too old
2025-12-24T02:11:38.438907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241400767210242:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.439428Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241400767210241:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.439437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241400767210241:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.440286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241400767210241:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.440481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241400767210221:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.440498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.440508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241400767210221:2109] [node 4] ICP22 created new session: [6:7587241400767210252:2109]
2025-12-24T02:11:38.440514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241400767210252:2109] [node 4] ICS09 handshake done sender: [6:7587241400767210241:2117] self: [6:7587241400767210231:0] peer: [4:7587241399678714023:0] socket: 359 qp: -1
2025-12-24T02:11:38.440521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241400767210252:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:38.440545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.439090Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714022:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.439167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241399678713988:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.446736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397081430974:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.447123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241400803809064:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2399
2025-12-24T02:11:38.450359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397081430982:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.450534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241397081430975:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.450562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.450572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397081430975:2111] [node 6] ICP22 created new session: [5:7587241397081431007:2111]
2025-12-24T02:11:38.450577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397081431007:2111] [node 6] ICS09 handshake done sender: [5:7587241397081430982:2114] self: [5:7587241397081430981:0] peer: [6:7587241400767210249:0] socket: 307 qp: -1
2025-12-24T02:11:38.450585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397081431007:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:38.450593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.451789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241400803809059:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.454721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241400803808963:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.454735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.454741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241400803808963:2048] [node 1] ICP22 created new session: [2:7587241400803809065:2048]
2025-12-24T02:11:38.454746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241400803809065:2048] [node 1] ICS09 handshake done sender: [2:7587241400803809059:2119] self: [2:7587241400803809063:0] peer: [1:7587241397088570784:0] socket: 355 qp: -1
2025-12-24T02:11:38.454751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241400803809065:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.452114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241397088570743:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19122494ns) {HandleHandshakeStatus:329(19121209ns) {DropIncomingHandshake:442(5196ns) DropOutgoingHandshake:457(208ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(19020505ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18995655ns max# 18908899ns}}}
2025-12-24T02:11:38.454759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.450935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714003:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.450949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241399678714003:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.454224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241399678714003:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.454304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241399678713999:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.454317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.454378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241399678713999:2112] [node 5] ICP22 created new session: [4:7587241399678714025:2112]
2025-12-24T02:11:38.454385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241399678714025:2112] [node 5] ICS09 handshake done sender: [4:7587241399678714003:2114] self: [4:7587241399678714002:0] peer: [5:7587241397081431006:0] socket: 309 qp: -1
2025-12-24T02:11:38.454392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241399678714025:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:38.454405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.455050Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241400767210217:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14475) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241400767210244:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.457576Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241400803808963:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:38.457134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241399678714027:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8816
2025-12-24T02:11:38.457943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241399678714021:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.457973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241399678713999:2112] [node 5] ICP18 handshake (actor [4:7587241399678714021:2121]) is too old
2025-12-24T02:11:38.458420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241399678714022:2122] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.457417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397081431009:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35640
2025-12-24T02:11:38.458216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397081430998:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.458776Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397081430974:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28706) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241397081430998:2122] held: no
2025-12-24T02:11:38.458908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241399678713988:2108] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.458923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.458929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241399678713988:2108] [node 6] ICP22 created new session: [4:7587241399678714028:2108]
2025-12-24T02:11:38.458936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241399678714028:2108] [node 6] ICS09 handshake done sender: [4:7587241399678714022:2122] self: [4:7587241399678714023:0] peer: [6:7587241400767210231:0] socket: 360 qp: -1
2025-12-24T02:11:38.458942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241399678714028:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:38.458954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.466758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397081430974:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.466786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.466801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397081430974:2110] [node 4] ICP22 created new session: [5:7587241397081431010:2110]
2025-12-24T02:11:38.466809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397081431010:2110] [node 4] ICS09 handshake done sender: [5:7587241397081430998:2122] self: [5:7587241397081431006:0] peer: [4:7587241399678714002:0] socket: 347 qp: -1
2025-12-24T02:11:38.466819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397081431010:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:38.466833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.467687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241400767210156:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.469223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.469702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241400767210156:2048] [node 1] ICP22 created new session: [6:7587241400767210254:2048]
2025-12-24T02:11:38.469710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241400767210254:2048] [node 1] ICS09 handshake done sender: [6:7587241400767210242:2118] self: [6:7587241400767210247:0] peer: [1:7587241397088570830:0] socket: 356 qp: -1
2025-12-24T02:11:38.469720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241400767210254:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.469729Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.470684Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241400767210156:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:38.471016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241400767210256:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48241
2025-12-24T02:11:38.471404Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241400767210244:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.471507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241400767210217:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.471521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.471526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241400767210217:2107] [node 5] ICP22 created new session: [6:7587241400767210257:2107]
2025-12-24T02:11:38.471531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241400767210257:2107] [node 5] ICS09 handshake done sender: [6:7587241400767210244:2120] self: [6:7587241400767210249:0] peer: [5:7587241397081430981:0] socket: 318 qp: -1
2025-12-24T02:11:38.471536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241400767210257:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:38.471543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.486177Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:38.486283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241400803809065:2048] [node 1] ICS07 socket disconnect 355 reason# EndOfStream
2025-12-24T02:11:38.486290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241400803809065:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.486331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.486348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241400803809065:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:38.486961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:38.487033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241399961282736:2048] [node 1] ICS01 socket: 316 reason# EndOfStream
2025-12-24T02:11:38.487040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241399961282639:2048] [node 1] ICP30 unregister session Session# [3:7587241399961282736:2048] VirtualId# [3:7587241399961282731:0]
2025-12-24T02:11:38.487046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241399961282736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.487073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.487092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.488312Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241400803808963:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4518) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:38.488326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241400803809065:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:38.490257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241400803808963:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4518) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.490270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241400803809065:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.490274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241400803808963:2048] [node 1] ICP30 unregister session Session# [2:7587241400803809065:2048] VirtualId# [2:7587241400803809063:0]
2025-12-24T02:11:38.490329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.490337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241400803808963:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.490340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4518) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.497421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:38.497519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241400767210254:2048] [node 1] ICS01 socket: 356 reason# EndOfStream
2025-12-24T02:11:38.497530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241400767210156:2048] [node 1] ICP30 unregister session Session# [6:7587241400767210254:2048] VirtualId# [6:7587241400767210247:0]
2025-12-24T02:11:38.497538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241400767210254:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.497591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.497615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.497659Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:38.497727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397081430993:2048] [node 1] ICS01 socket: 300 reason# EndOfStream
2025-12-24T02:11:38.497733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397081430907:2048] [node 1] ICP30 unregister session Session# [5:7587241397081430993:2048] VirtualId# [5:7587241397081430985:0]
2025-12-24T02:11:38.497739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397081430993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.497772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.497796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.498971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:38.499053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241399678714014:2048] [node 1] ICS01 socket: 337 reason# EndOfStream
2025-12-24T02:11:38.499059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241399678713926:2048] [node 1] ICP30 unregister session Session# [4:7587241399678714014:2048] VirtualId# [4:7587241399678714008:0]
2025-12-24T02:11:38.499066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241399678714014:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.499110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:38.499144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:38.510213Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:38.510309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241399961282722:2049] [node 2] ICS01 socket: 317 reason# EndOfStream
2025-12-24T02:11:38.510320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241399961282640:2049] [node 2] ICP30 unregister session Session# [3:7587241399961282722:2049] VirtualId# [3:7587241399961282720:0]
2025-12-24T02:11:38.510327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241399961282722:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.510368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.510388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.517766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:38.517874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397081431003:2049] [node 2] ICS01 socket: 346 reason# EndOfStream
2025-12-24T02:11:38.517884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397081430908:2049] [node 2] ICP30 unregister session Session# [5:7587241397081431003:2049] VirtualId# [5:7587241397081430995:0]
2025-12-24T02:11:38.517892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397081431003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.517941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.517964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.518060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:38.518133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241400767210238:2049] [node 2] ICS01 socket: 338 reason# EndOfStream
2025-12-24T02:11:38.518138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241400767210157:2049] [node 2] ICP30 unregister session Session# [6:7587241400767210238:2049] VirtualId# [6:7587241400767210234:0]
2025-12-24T02:11:38.518143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241400767210238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.518170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.518192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.519008Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:38.519098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241399678714019:2049] [node 2] ICS01 socket: 298 reason# EndOfStream
2025-12-24T02:11:38.519105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241399678713927:2049] [node 2] ICP30 unregister session Session# [4:7587241399678714019:2049] VirtualId# [4:7587241399678714006:0]
2025-12-24T02:11:38.519111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241399678714019:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.519155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:38.519180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:38.558580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:38.558707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241400767210245:2050] [node 3] ICS07 socket disconnect 323 reason# EndOfStream
2025-12-24T02:11:38.558715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241400767210245:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.558769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:38.558793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241400767210245:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:38.559158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:38.559254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397081431001:2050] [node 3] ICS01 socket: 324 reason# EndOfStream
2025-12-24T02:11:38.559260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397081430909:2050] [node 3] ICP30 unregister session Session# [5:7587241397081431001:2050] VirtualId# [5:7587241397081430989:0]
2025-12-24T02:11:38.559267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397081431001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.559314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:38.559340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:38.559739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241400767210158:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8873) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.559755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241400767210245:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.559760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241400767210158:2050] [node 3] ICP30 unregister session Session# [6:7587241400767210245:2050] VirtualId# [6:7587241400767210237:0]
2025-12-24T02:11:38.559818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:38.559826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241400767210158:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.559829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8873) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.570767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:38.570879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241400767210252:2109] [node 4] ICS07 socket disconnect 359 reason# EndOfStream
2025-12-24T02:11:38.570886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241400767210252:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:38.570928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:38.570954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241400767210252:2109] [node 4] ICS15 start handshake
2025-12-24T02:11:38.571023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241400767210158:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:38.573276Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241400767210221:2109] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28706) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:38.573292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241400767210252:2109] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:38.573297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241400767210221:2109] [node 4] ICP30 unregister session Session# [6:7587241400767210252:2109] VirtualId# [6:7587241400767210231:0]
2025-12-24T02:11:38.573311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:38.573318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241400767210221:2109] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.573321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28706) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:38.585210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241400767210221:2109] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:38.634886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241399909185321:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:38.638861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241399909185320:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:38.650857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399150777749:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.650877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399150777750:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.650838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241397253766496:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.650847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241397253766497:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.646876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241398178248346:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.650852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241397253766498:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:38.646890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241398178248347:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:38.647327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397176886518:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.647337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397176886519:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.647342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397176886520:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:38.662827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241401104496344:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:38.662837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241401104496345:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:38.662842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241401104496346:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.740036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241398178248406:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38951
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.739152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241399909185381:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:38.741285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777810:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.741321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777809:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.743454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399150777809:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.750489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241401104496391:2089] [node 5] ICP01 ready to work
2025-12-24T02:11:38.749532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399150777812:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:38.750211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399150777812:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.751754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248406:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.751897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241398178248408:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:38.752248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241398178248408:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.760291Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241399909185320:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21249601ns) {Configure:104(21247315ns) {ProcessConfigured:135(15677296ns) {StartInitialHandshake:157(15675785ns) {DropHandshakes:470(1033ns) {DropIncomingHandshake:442(159ns) DropOutgoingHandshake:457(143ns)} GenerateSessionVirtualId:988(373ns) PrepareNewSessionHandshake:477(1542ns) {SwitchToState:179(991ns)}}}}}
2025-12-24T02:11:38.761262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241399909185381:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21909251ns) {Configure:104(21907516ns) {ProcessConfigured:135(16449625ns) {StartInitialHandshake:157(16448680ns) {DropHandshakes:470(601ns) {DropIncomingHandshake:442(32ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(158ns) PrepareNewSessionHandshake:477(1635ns) {SwitchToState:179(1035ns)}}}}}
2025-12-24T02:11:38.761988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241399909185321:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22763862ns) {Configure:104(22763121ns) {ProcessConfigured:135(17077991ns) {StartInitialHandshake:157(17076796ns) {DropHandshakes:470(294ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(164ns) PrepareNewSessionHandshake:477(736ns) {SwitchToState:179(429ns)}}}}}
2025-12-24T02:11:38.762193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241401104496389:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:38.762247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886579:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.762274Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241397176886579:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.762532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399150777810:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.762739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399150777815:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:38.763148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399150777815:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.762828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766558:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.762860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241397253766558:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.767297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397176886583:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:38.766913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241401104496391:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16315019ns) {Configure:104(16314543ns) {ProcessConfigured:135(15960178ns) {StartInitialHandshake:157(15959622ns) {DropHandshakes:470(633ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(359ns) PrepareNewSessionHandshake:477(1172ns) {SwitchToState:179(633ns)}}}}}
2025-12-24T02:11:38.770761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241401104496380:2082] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.770790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241401104496380:2082] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.775106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241399909185392:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:38.774959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766560:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.777192Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766560:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.780058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777817:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.780143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777818:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.781253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399150777809:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.782246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397176886579:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.782506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241397176886519:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.782525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.782535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397176886519:2049] [node 2] ICP22 created new session: [5:7587241397176886586:2049]
2025-12-24T02:11:38.782543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397176886586:2049] [node 2] ICS09 handshake done sender: [5:7587241397176886579:2107] self: [5:7587241397176886578:0] peer: [2:7587241398178248409:0] socket: 300 qp: -1
2025-12-24T02:11:38.782550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397176886586:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.782562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.780837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241397253766558:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.781052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241397253766498:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.781071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.781079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241397253766498:2050] [node 3] ICP22 created new session: [4:7587241397253766563:2050]
2025-12-24T02:11:38.781087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241397253766563:2050] [node 3] ICS09 handshake done sender: [4:7587241397253766558:2108] self: [4:7587241397253766557:0] peer: [3:7587241399150777813:0] socket: 301 qp: -1
2025-12-24T02:11:38.781095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241397253766563:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.781119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.785237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766565:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.786804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399150777817:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.787235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399150777749:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.787938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241401104496380:2082] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.788129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241401104496346:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.788151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.788159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241401104496346:2050] [node 3] ICP22 created new session: [6:7587241401104496419:2050]
2025-12-24T02:11:38.788167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241401104496419:2050] [node 3] ICS09 handshake done sender: [6:7587241401104496380:2082] self: [6:7587241401104496379:0] peer: [3:7587241399150777816:0] socket: 306 qp: -1
2025-12-24T02:11:38.788173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241401104496419:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.788199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:38.787678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241397253766496:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.788264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766565:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.790849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185387:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.790864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241399909185387:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.794462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185389:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.794475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241399909185389:2110] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.794819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241397176886583:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27420348ns) {Configure:104(27419338ns) {ProcessConfigured:135(26995907ns) {StartInitialHandshake:157(26994960ns) {DropHandshakes:470(1051ns) {DropIncomingHandshake:442(232ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(278ns) PrepareNewSessionHandshake:477(1454ns) {SwitchToState:179(726ns)}}}}}
2025-12-24T02:11:38.794683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241398178248417:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38951
2025-12-24T02:11:38.796910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241398178248416:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38951
2025-12-24T02:11:38.799015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241398178248411:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:38.799129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248416:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.799168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241398178248346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.800200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496421:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
2025-12-24T02:11:38.798790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886588:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.799949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241397176886519:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(17407440ns) {HandleHandshakeStatus:329(17406498ns) {DropIncomingHandshake:442(146ns) DropOutgoingHandshake:457(287ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(11888378ns) {ForwardSessionEventToSession:612(11873047ns) ForwardSessionEventToSession:612(1956ns) ForwardSessionEventToSession:612(1155ns) ForwardSessionEventToSession:612(956ns) ForwardSessionEventToSession:612(689ns) ForwardSessionEventToSession:612(1897ns) ForwardSessionEventToSession:612(786ns) ForwardSessionEventToSession:612(830ns) ForwardSessionEventToSession:612(664ns) ForwardSessionEventToSession:612(1592ns)}}}
2025-12-24T02:11:38.800496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886588:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.802283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777821:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.802856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241399909185396:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:38.803664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241398178248406:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.806874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399150777810:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.810866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399150777815:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.810885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.810894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399150777815:2110] [node 6] ICP22 created new session: [3:7587241399150777822:2110]
2025-12-24T02:11:38.810902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399150777822:2110] [node 6] ICS09 handshake done sender: [3:7587241399150777810:2108] self: [3:7587241399150777816:0] peer: [6:7587241401104496379:0] socket: 307 qp: -1
2025-12-24T02:11:38.810910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399150777822:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:38.810939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.806874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241399909185387:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.807256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241399909185381:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.807272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.807280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241399909185381:2107] [node 4] ICP22 created new session: [1:7587241399909185401:2107]
2025-12-24T02:11:38.807286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241399909185401:2107] [node 4] ICS09 handshake done sender: [1:7587241399909185387:2109] self: [1:7587241399909185383:0] peer: [4:7587241397253766562:0] socket: 310 qp: -1
2025-12-24T02:11:38.807293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241399909185401:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:38.807302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.807540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241399909185400:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63595
2025-12-24T02:11:38.812648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241397253766567:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:38.813093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241397253766567:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.811631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241398178248408:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.811649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.814748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185385:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.814766Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241399909185385:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.814737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399150777812:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.814758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.814768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399150777812:2109] [node 4] ICP22 created new session: [3:7587241399150777824:2109]
2025-12-24T02:11:38.814775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399150777824:2109] [node 4] ICS09 handshake done sender: [3:7587241399150777809:2107] self: [3:7587241399150777813:0] peer: [4:7587241397253766557:0] socket: 303 qp: -1
2025-12-24T02:11:38.814781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399150777824:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:38.814806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.811654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241398178248408:2108] [node 5] ICP22 created new session: [2:7587241398178248421:2108]
2025-12-24T02:11:38.811660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241398178248421:2108] [node 5] ICS09 handshake done sender: [2:7587241398178248406:2107] self: [2:7587241398178248409:0] peer: [5:7587241397176886578:0] socket: 302 qp: -1
2025-12-24T02:11:38.811666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241398178248421:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:38.811673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.815112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241401104496416:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.815120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241401104496416:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.814926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241399909185389:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.815093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185400:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.815316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241399909185321:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.815332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.815341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241399909185321:2049] [node 3] ICP22 created new session: [1:7587241399909185406:2049]
2025-12-24T02:11:38.815347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241399909185406:2049] [node 3] ICS09 handshake done sender: [1:7587241399909185389:2110] self: [1:7587241399909185384:0] peer: [3:7587241399150777819:0] socket: 312 qp: -1
2025-12-24T02:11:38.815355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241399909185406:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:38.815365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.818819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886592:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.818779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241399150777750:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19949031ns) {Configure:104(19948385ns) {ProcessConfigured:135(19544823ns) {StartInitialHandshake:157(19544143ns) {DropHandshakes:470(1212ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(722ns) {SwitchToState:179(497ns)}}}}}
2025-12-24T02:11:38.819078Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241398178248411:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19776420ns) {Configure:104(19775889ns) {ProcessConfigured:135(19426372ns) {StartInitialHandshake:157(19425751ns) {DropHandshakes:470(849ns) {DropIncomingHandshake:442(37ns) DropOutgoingHandshake:457(127ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(879ns) {SwitchToState:179(398ns)}}}}}
2025-12-24T02:11:38.822046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241401104496421:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.822087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241401104496391:2089] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.821796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241399150777812:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(40454319ns) {HandleHandshakeStatus:329(40454089ns) {DropIncomingHandshake:442(891ns) DropOutgoingHandshake:457(59ns) SwitchToState:179(59ns) ProcessPendingSessionEvents:495(27ns)}}
2025-12-24T02:11:38.823334Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241401104496346:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(35136538ns) {HandleHandshakeStatus:329(35136216ns) {DropIncomingHandshake:442(83ns) DropOutgoingHandshake:457(136ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(26032184ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 26018673ns max# 25966148ns}}}
2025-12-24T02:11:38.826642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241399909185385:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.822710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241398178248418:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:38.826784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241399909185320:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.826814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.826821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241399909185320:2048] [node 2] ICP22 created new session: [1:7587241399909185410:2048]
2025-12-24T02:11:38.826828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241399909185410:2048] [node 2] ICS09 handshake done sender: [1:7587241399909185385:2108] self: [1:7587241399909185382:0] peer: [2:7587241398178248419:0] socket: 309 qp: -1
2025-12-24T02:11:38.826834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241399909185410:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:38.826851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.827009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766569:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.827344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777829:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.827395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777828:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.827660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241397253766560:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.827762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241397253766496:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.827776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.827787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241397253766496:2048] [node 1] ICP22 created new session: [4:7587241397253766570:2048]
2025-12-24T02:11:38.827792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241397253766570:2048] [node 1] ICS09 handshake done sender: [4:7587241397253766560:2109] self: [4:7587241397253766562:0] peer: [1:7587241399909185383:0] socket: 313 qp: -1
2025-12-24T02:11:38.827797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241397253766570:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.827807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.831172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886589:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.831184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241397176886589:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.830604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241401104496416:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.830900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241401104496389:2084] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.830910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.830916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241401104496389:2084] [node 4] ICP22 created new session: [6:7587241401104496424:2084]
2025-12-24T02:11:38.830922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241401104496424:2084] [node 4] ICS09 handshake done sender: [6:7587241401104496416:2112] self: [6:7587241401104496409:0] peer: [4:7587241397253766568:0] socket: 318 qp: -1
2025-12-24T02:11:38.830928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241401104496424:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:38.830934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.830725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399150777829:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.830761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241399150777750:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.831556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241398178248346:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241398178248416:2112] held: no
2025-12-24T02:11:38.834941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241399150777832:2117] [node 5] ICP01 ready to work
2025-12-24T02:11:38.834944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399150777817:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.835590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248424:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.835601Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241398178248424:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.836083Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241398178248430:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38951
2025-12-24T02:11:38.836345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241399909185321:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(20918669ns) {HandleHandshakeStatus:329(20918032ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(286ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(13390947ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13370457ns max# 13291837ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.842139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241398178248416:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.839071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886593:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.842891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241398178248424:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.843092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241398178248347:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.843109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.843116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241398178248347:2049] [node 3] ICP22 created new session: [2:7587241398178248433:2049]
2025-12-24T02:11:38.843124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241398178248433:2049] [node 3] ICS09 handshake done sender: [2:7587241398178248424:2114] self: [2:7587241398178248423:0] peer: [3:7587241399150777830:0] socket: 336 qp: -1
2025-12-24T02:11:38.843130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241398178248433:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:38.843138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.843610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241398178248429:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38951
2025-12-24T02:11:38.843422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496426:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
2025-12-24T02:11:38.843466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496427:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
2025-12-24T02:11:38.846261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886592:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.846883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766572:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.847749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241397253766565:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.847847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241397253766567:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.847860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.847866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241397253766567:2111] [node 6] ICP22 created new session: [4:7587241397253766573:2111]
2025-12-24T02:11:38.847871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241397253766573:2111] [node 6] ICS09 handshake done sender: [4:7587241397253766565:2110] self: [4:7587241397253766568:0] peer: [6:7587241401104496409:0] socket: 319 qp: -1
2025-12-24T02:11:38.847877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241397253766573:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:38.847889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.848026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241401104496426:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.848512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241401104496344:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.846942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399150777749:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.846963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.846970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399150777749:2048] [node 1] ICP22 created new session: [3:7587241399150777837:2048]
2025-12-24T02:11:38.846979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399150777837:2048] [node 1] ICS09 handshake done sender: [3:7587241399150777817:2111] self: [3:7587241399150777819:0] peer: [1:7587241399909185384:0] socket: 314 qp: -1
2025-12-24T02:11:38.846985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399150777837:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.846993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.854713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241399150777836:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5476
2025-12-24T02:11:38.851308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241398178248346:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.851328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.851336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241398178248346:2048] [node 1] ICP22 created new session: [2:7587241398178248436:2048]
2025-12-24T02:11:38.851344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241398178248436:2048] [node 1] ICS09 handshake done sender: [2:7587241398178248416:2112] self: [2:7587241398178248419:0] peer: [1:7587241399909185382:0] socket: 315 qp: -1
2025-12-24T02:11:38.851349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241398178248436:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.851366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.851621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248429:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.854745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185399:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.854761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241399909185399:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.854811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241399909185394:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.854815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241399909185394:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.858831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399150777829:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.859214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241401104496427:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.859643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241401104496345:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.859932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496430:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.861823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241399150777750:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.861846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.861854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399150777750:2049] [node 2] ICP22 created new session: [3:7587241399150777839:2049]
2025-12-24T02:11:38.861861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399150777839:2049] [node 2] ICS09 handshake done sender: [3:7587241399150777829:2116] self: [3:7587241399150777830:0] peer: [2:7587241398178248423:0] socket: 345 qp: -1
2025-12-24T02:11:38.861869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399150777839:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.861877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.867012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241401104496421:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.867142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241401104496391:2089] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.867163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.867193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241401104496391:2089] [node 5] ICP22 created new session: [6:7587241401104496431:2089]
2025-12-24T02:11:38.867200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241401104496431:2089] [node 5] ICS09 handshake done sender: [6:7587241401104496421:2113] self: [6:7587241401104496422:0] peer: [5:7587241397176886584:0] socket: 330 qp: -1
2025-12-24T02:11:38.867207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241401104496431:2089] [node 5] ICS10 traffic start
2025-12-24T02:11:38.867220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.866775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397176886589:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.870833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241398178248347:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27696057ns) {HandleHandshakeStatus:329(27695435ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(207ns) SwitchToState:179(253ns) ProcessPendingSessionEvents:495(24001489ns) {unordered ForwardSessionEventToSession:612 num# 30 sum# 23993167ns max# 23968780ns}}}
2025-12-24T02:11:38.870973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241398178248347:2049] [node 3] ICP18 handshake (actor [2:7587241398178248429:2116]) is too old
2025-12-24T02:11:38.871005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241397176886583:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.871030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.871037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397176886583:2109] [node 6] ICP22 created new session: [5:7587241397176886596:2109]
2025-12-24T02:11:38.871044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397176886596:2109] [node 6] ICS09 handshake done sender: [5:7587241397176886589:2111] self: [5:7587241397176886584:0] peer: [6:7587241401104496422:0] socket: 329 qp: -1
2025-12-24T02:11:38.871050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397176886596:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:38.871058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.878157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248426:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.878180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241398178248426:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.878052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766575:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.878515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766575:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.878716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397176886518:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.878790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241397176886518:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32438973ns) {Configure:104(32438177ns) {ProcessConfigured:135(31989036ns) {IncomingHandshake:261(31972671ns) {DropIncomingHandshake:442(64ns) PrepareNewSessionHandshake:477(237ns) {SwitchToState:179(106ns)} GenerateSessionVirtualId:988(295ns) IssueIncomingHandshakeReply:193(31966747ns) {SwitchToState:179(1285ns)}}}}}
2025-12-24T02:11:38.879102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886593:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.879583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397176886520:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.879758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241399150777834:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.879770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241399150777834:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.879076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241397253766497:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.882938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241399909185399:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.883067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241399909185396:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.883084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.883125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241399909185396:2113] [node 5] ICP22 created new session: [1:7587241399909185413:2113]
2025-12-24T02:11:38.883133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241399909185413:2113] [node 5] ICS09 handshake done sender: [1:7587241399909185399:2114] self: [1:7587241399909185398:0] peer: [5:7587241397176886594:0] socket: 338 qp: -1
2025-12-24T02:11:38.883140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241399909185413:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:38.883163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:38.887104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241399909185394:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.887414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241399909185392:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.887429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.887437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241399909185392:2111] [node 6] ICP22 created new session: [1:7587241399909185416:2111]
2025-12-24T02:11:38.887445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241399909185416:2111] [node 6] ICS09 handshake done sender: [1:7587241399909185394:2112] self: [1:7587241399909185393:0] peer: [6:7587241401104496428:0] socket: 333 qp: -1
2025-12-24T02:11:38.887450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241399909185416:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:38.887457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.889997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241398178248431:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.890016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241398178248431:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.898952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241398178248426:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:38.899166Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241399150777834:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.899099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241398178248411:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.899120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:38.899131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241398178248411:2109] [node 6] ICP22 created new session: [2:7587241398178248439:2109]
2025-12-24T02:11:38.899139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241398178248439:2109] [node 6] ICS09 handshake done sender: [2:7587241398178248426:2115] self: [2:7587241398178248420:0] peer: [6:7587241401104496429:0] socket: 341 qp: -1
2025-12-24T02:11:38.899147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241398178248439:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:38.899155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:38.899274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241399150777832:2117] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.899288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.899294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241399150777832:2117] [node 5] ICP22 created new session: [3:7587241399150777842:2117]
2025-12-24T02:11:38.899300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241399150777842:2117] [node 5] ICS09 handshake done sender: [3:7587241399150777834:2118] self: [3:7587241399150777833:0] peer: [5:7587241397176886598:0] socket: 347 qp: -1
2025-12-24T02:11:38.899306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241399150777842:2117] [node 5] ICS10 traffic start
2025-12-24T02:11:38.899313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.903058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241399909185396:2113] [node 5] ICP35 event processing took too much time PendingConnection:279(19929104ns) {HandleHandshakeStatus:329(19927522ns) {DropIncomingHandshake:442(120ns) DropOutgoingHandshake:457(425ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(19772710ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 19765081ns max# 19745371ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.903385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496433:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
2025-12-24T02:11:38.903574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241401104496434:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32551
2025-12-24T02:11:38.906988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241401104496427:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.907813Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241399909185392:2111] [node 6] ICP35 event processing took too much time PendingConnection:279(20295117ns) {HandleHandshakeStatus:329(20294339ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(144ns) SwitchToState:179(206ns) ProcessPendingSessionEvents:495(20212963ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 20178593ns max# 20114555ns}}}
2025-12-24T02:11:38.910745Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241401104496426:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.914825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241401104496344:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.914851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.914862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241401104496344:2048] [node 1] ICP22 created new session: [6:7587241401104496435:2048]
2025-12-24T02:11:38.914871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241401104496435:2048] [node 1] ICS09 handshake done sender: [6:7587241401104496426:2114] self: [6:7587241401104496428:0] peer: [1:7587241399909185393:0] socket: 334 qp: -1
2025-12-24T02:11:38.914880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241401104496435:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.914892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.918716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886599:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.918864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397176886592:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:38.918886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241401104496345:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.918911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.918925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241401104496345:2049] [node 2] ICP22 created new session: [6:7587241401104496437:2049]
2025-12-24T02:11:38.918933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241401104496437:2049] [node 2] ICS09 handshake done sender: [6:7587241401104496427:2115] self: [6:7587241401104496429:0] peer: [2:7587241398178248420:0] socket: 352 qp: -1
2025-12-24T02:11:38.918939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241401104496437:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.918952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.922144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886600:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.918854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241398178248431:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.923017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241398178248418:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.923033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.923040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241398178248418:2111] [node 4] ICP22 created new session: [2:7587241398178248442:2111]
2025-12-24T02:11:38.923047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241398178248442:2111] [node 4] ICS09 handshake done sender: [2:7587241398178248431:2118] self: [2:7587241398178248428:0] peer: [4:7587241397253766576:0] socket: 353 qp: -1
2025-12-24T02:11:38.923055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241398178248442:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:38.923068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.923272Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766577:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.923014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397176886593:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:38.923208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397176886520:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.923221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:38.923229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397176886520:2050] [node 3] ICP22 created new session: [5:7587241397176886601:2050]
2025-12-24T02:11:38.923234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397176886601:2050] [node 3] ICS09 handshake done sender: [5:7587241397176886593:2113] self: [5:7587241397176886598:0] peer: [3:7587241399150777833:0] socket: 350 qp: -1
2025-12-24T02:11:38.923240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397176886601:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:38.923246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:38.923579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397176886518:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.923590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:38.923595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397176886518:2048] [node 1] ICP22 created new session: [5:7587241397176886603:2048]
2025-12-24T02:11:38.923602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397176886603:2048] [node 1] ICS09 handshake done sender: [5:7587241397176886592:2112] self: [5:7587241397176886594:0] peer: [1:7587241399909185398:0] socket: 339 qp: -1
2025-12-24T02:11:38.923607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397176886603:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:38.923617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:38.931037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241397253766575:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:38.935023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241397253766497:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.935046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:38.935057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241397253766497:2049] [node 2] ICP22 created new session: [4:7587241397253766578:2049]
2025-12-24T02:11:38.935064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241397253766578:2049] [node 2] ICS09 handshake done sender: [4:7587241397253766575:2114] self: [4:7587241397253766576:0] peer: [2:7587241398178248428:0] socket: 354 qp: -1
2025-12-24T02:11:38.935072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241397253766578:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:38.935088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:38.947136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241397176886606:2116] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.953998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241397253766580:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2653
2025-12-24T02:11:38.954438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241397253766582:2117] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:38.967003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886610:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.967288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241397176886610:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.967321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241397176886606:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:38.968179Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241397176886612:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37647
2025-12-24T02:11:38.968517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241397176886610:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:38.968605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241397176886606:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:38.968624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:38.968633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241397176886606:2116] [node 4] ICP22 created new session: [5:7587241397176886613:2116]
2025-12-24T02:11:38.968640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241397176886613:2116] [node 4] ICS09 handshake done sender: [5:7587241397176886610:2118] self: [5:7587241397176886611:0] peer: [4:7587241397253766583:0] socket: 365 qp: -1
2025-12-24T02:11:38.968647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241397176886613:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:38.968658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:38.967183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766580:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.967498Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241397253766584:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:38.967501Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241397253766584:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:38.968325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241397253766584:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:38.969595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241397253766582:2117] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:38.969608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:38.969615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241397253766582:2117] [node 5] ICP22 created new session: [4:7587241397253766588:2117]
2025-12-24T02:11:38.969620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241397253766588:2117] [node 5] ICS09 handshake done sender: [4:7587241397253766584:2118] self: [4:7587241397253766583:0] peer: [5:7587241397176886611:0] socket: 364 qp: -1
2025-12-24T02:11:38.969625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241397253766588:2117] [node 5] ICS10 traffic start
2025-12-24T02:11:38.969633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.060065Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:39.060175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241398178248436:2048] [node 1] ICS07 socket disconnect 315 reason# EndOfStream
2025-12-24T02:11:39.060185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241398178248436:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.060231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.060252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241398178248436:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.061212Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241398178248346:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.061226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241398178248436:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.061233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241398178248346:2048] [node 1] ICP30 unregister session Session# [2:7587241398178248436:2048] VirtualId# [2:7587241398178248419:0]
2025-12-24T02:11:39.061270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.061277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241398178248346:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.061281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.063182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:39.063264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241399150777837:2048] [node 1] ICS07 socket disconnect 314 reason# EndOfStream
2025-12-24T02:11:39.063269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241399150777837:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.063313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.063335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241399150777837:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.063763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241399150777749:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.063771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241399150777837:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.063776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241399150777749:2048] [node 1] ICP30 unregister session Session# [3:7587241399150777837:2048] VirtualId# [3:7587241399150777819:0]
2025-12-24T02:11:39.063815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.063820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241399150777749:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.063822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.070267Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:39.070366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241397253766570:2048] [node 1] ICS07 socket disconnect 313 reason# EndOfStream
2025-12-24T02:11:39.070374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241397253766570:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.070436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.070460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241397253766570:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.070514Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:39.070591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241401104496435:2048] [node 1] ICS07 socket disconnect 334 reason# EndOfStream
2025-12-24T02:11:39.070596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241401104496435:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.070634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.070658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241401104496435:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.071633Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241401104496344:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.071644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241401104496435:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.071649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241401104496344:2048] [node 1] ICP30 unregister session Session# [6:7587241401104496435:2048] VirtualId# [6:7587241401104496428:0]
2025-12-24T02:11:39.071695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.071711Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241401104496344:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.071714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.070877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:39.070961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241397176886603:2048] [node 1] ICS07 socket disconnect 339 reason# EndOfStream
2025-12-24T02:11:39.070966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397176886603:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.071005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.071027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241397176886603:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.071403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397176886518:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.071412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397176886603:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.071417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397176886518:2048] [node 1] ICP30 unregister session Session# [5:7587241397176886603:2048] VirtualId# [5:7587241397176886594:0]
2025-12-24T02:11:39.071477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.071484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241397176886518:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.071487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.074821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241399150777749:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.078984Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241397253766496:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.079006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241397253766570:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.079013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241397253766496:2048] [node 1] ICP30 unregister session Session# [4:7587241397253766570:2048] VirtualId# [4:7587241397253766562:0]
2025-12-24T02:11:39.079035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.079043Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241397253766496:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.079047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27640) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.082863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241401104496344:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.082920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241397176886518:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.088525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241397253766496:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.090766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:39.090893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241399150777839:2049] [node 2] ICS01 socket: 345 reason# EndOfStream
2025-12-24T02:11:39.090903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241399150777750:2049] [node 2] ICP30 unregister session Session# [3:7587241399150777839:2049] VirtualId# [3:7587241399150777830:0]
2025-12-24T02:11:39.090911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241399150777839:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.090966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.090993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.097603Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:39.097722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241397253766578:2049] [node 2] ICS07 socket disconnect 354 reason# EndOfStream
2025-12-24T02:11:39.097730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241397253766578:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.097785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.097808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241397253766578:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.098282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241397253766497:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.098295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241397253766578:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.098773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:39.098300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241397253766497:2049] [node 2] ICP30 unregister session Session# [4:7587241397253766578:2049] VirtualId# [4:7587241397253766576:0]
2025-12-24T02:11:39.098353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.098360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241397253766497:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.098363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.098859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241397176886586:2049] [node 2] ICS07 socket disconnect 300 reason# EndOfStream
2025-12-24T02:11:39.098873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397176886586:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.099158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:39.099231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241401104496437:2049] [node 2] ICS07 socket disconnect 352 reason# EndOfStream
2025-12-24T02:11:39.099236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241401104496437:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.099273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.099297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241401104496437:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.098924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.098953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241397176886586:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.103056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397176886519:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.103073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397176886586:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.103080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397176886519:2049] [node 2] ICP30 unregister session Session# [5:7587241397176886586:2049] VirtualId# [5:7587241397176886578:0]
2025-12-24T02:11:39.103136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.103143Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241397176886519:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.103147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.106852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241401104496345:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.106868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241401104496437:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.106872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241401104496345:2049] [node 2] ICP30 unregister session Session# [6:7587241401104496437:2049] VirtualId# [6:7587241401104496429:0]
2025-12-24T02:11:39.106919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.106923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241401104496345:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.106926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10136) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.109846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241397253766497:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.114768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241397176886519:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.118768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:39.118894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241401104496419:2050] [node 3] ICS07 socket disconnect 306 reason# EndOfStream
2025-12-24T02:11:39.118902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241401104496419:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.118963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.118994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241401104496419:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.119052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241401104496345:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.122599Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241401104496346:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.122619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241401104496419:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.122627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241401104496346:2050] [node 3] ICP30 unregister session Session# [6:7587241401104496419:2050] VirtualId# [6:7587241401104496379:0]
2025-12-24T02:11:39.122657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.122663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241401104496346:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.122666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.122748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:39.122834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241397176886601:2050] [node 3] ICS07 socket disconnect 350 reason# EndOfStream
2025-12-24T02:11:39.122841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241397176886601:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.122896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.122918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241397176886601:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.123416Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241397176886520:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.123426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241397176886601:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.123431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241397176886520:2050] [node 3] ICP30 unregister session Session# [5:7587241397176886601:2050] VirtualId# [5:7587241397176886598:0]
2025-12-24T02:11:39.123482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.123487Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241397176886520:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.123490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25621) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.130762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241401104496346:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:39.134743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241397176886520:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:39.142956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:39.143063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241401104496424:2084] [node 4] ICS01 socket: 318 reason# EndOfStream
2025-12-24T02:11:39.143074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241401104496389:2084] [node 4] ICP30 unregister session Session# [6:7587241401104496424:2084] VirtualId# [6:7587241401104496409:0]
2025-12-24T02:11:39.143083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241401104496424:2084] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.143127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:39.143162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:39.210210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241401516432527:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:39.210226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241401516432528:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:39.215637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405004068134:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.215646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405004068135:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.215652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405004068136:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.214778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403619698798:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.214796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403619698799:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:39.226873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241403201971864:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.226884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241403201971865:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.226889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241403201971866:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.216264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241403519290805:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.216274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241403519290806:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.234909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241404315871038:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.234922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241404315871039:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.234927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241404315871040:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.286874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241404315871085:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:39.288393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241401516432593:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:39.288486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241401516432594:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:39.288560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241401516432595:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.291117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403619698862:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:39.291232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403619698863:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:39.291310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403619698864:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:39.292777Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403619698861:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41084
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.294898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241404315871093:2096] [node 5] ICP01 ready to work
2025-12-24T02:11:39.295360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871094:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.295765Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068195:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.298834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971925:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.299194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290867:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
2025-12-24T02:11:39.307079Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068195:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.307325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405004068197:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:39.309567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405004068197:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.311588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241404315871092:2099] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.311600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241404315871092:2099] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.312358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241404315871096:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.312364Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241404315871096:2101] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.312807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241404315871092:2099] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.312952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068199:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.313128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405004068195:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.313180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405004068197:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.313197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.313203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405004068197:2109] [node 6] ICP22 created new session: [4:7587241405004068200:2109]
2025-12-24T02:11:39.313210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405004068200:2109] [node 6] ICS09 handshake done sender: [4:7587241405004068195:2108] self: [4:7587241405004068198:0] peer: [6:7587241404315871086:0] socket: 316 qp: -1
2025-12-24T02:11:39.313217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405004068200:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:39.313241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.313431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241404315871085:2089] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.313444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.313448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241404315871085:2089] [node 4] ICP22 created new session: [6:7587241404315871112:2089]
2025-12-24T02:11:39.313453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241404315871112:2089] [node 4] ICS09 handshake done sender: [6:7587241404315871092:2099] self: [6:7587241404315871086:0] peer: [4:7587241405004068198:0] socket: 314 qp: -1
2025-12-24T02:11:39.313459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241404315871112:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:39.313466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.310804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241403201971925:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.310990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241403201971927:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:39.311467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241403201971927:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.313934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971929:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.314243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241404315871096:2101] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.314433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403619698878:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41084
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.315615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241401516432604:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64239
2025-12-24T02:11:39.315663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241401516432605:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64239
2025-12-24T02:11:39.315675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241401516432606:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64239
2025-12-24T02:11:39.314411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241403201971925:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.314463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241403201971927:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.314470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.314475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241403201971927:2109] [node 6] ICP22 created new session: [5:7587241403201971930:2109]
2025-12-24T02:11:39.314481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241403201971930:2109] [node 6] ICS09 handshake done sender: [5:7587241403201971925:2108] self: [5:7587241403201971928:0] peer: [6:7587241404315871095:0] socket: 318 qp: -1
2025-12-24T02:11:39.314486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241403201971930:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:39.314501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.324828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241404315871094:2100] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.324934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241404315871093:2096] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.324947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.324958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241404315871093:2096] [node 5] ICP22 created new session: [6:7587241404315871115:2096]
2025-12-24T02:11:39.324966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241404315871115:2096] [node 5] ICS09 handshake done sender: [6:7587241404315871096:2101] self: [6:7587241404315871095:0] peer: [5:7587241403201971928:0] socket: 315 qp: -1
2025-12-24T02:11:39.324973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241404315871115:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:39.324983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.325364Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871114:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
2025-12-24T02:11:39.325790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241404315871039:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.325938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698861:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.326087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403619698879:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41084
2025-12-24T02:11:39.326099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241403619698798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.326432Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698868:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.326438Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403619698868:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.326962Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403619698868:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.327042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403619698862:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.327054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.327059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403619698862:2107] [node 6] ICP22 created new session: [2:7587241403619698882:2107]
2025-12-24T02:11:39.327064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403619698882:2107] [node 6] ICS09 handshake done sender: [2:7587241403619698868:2112] self: [2:7587241403619698867:0] peer: [6:7587241404315871117:0] socket: 310 qp: -1
2025-12-24T02:11:39.327070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403619698882:2107] [node 6] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.327076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.327410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871118:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
2025-12-24T02:11:39.327660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241404315871094:2100] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.327728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241404315871039:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.327737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.327741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241404315871039:2049] [node 2] ICP22 created new session: [6:7587241404315871121:2049]
2025-12-24T02:11:39.327747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241404315871121:2049] [node 2] ICS09 handshake done sender: [6:7587241404315871094:2100] self: [6:7587241404315871117:0] peer: [2:7587241403619698867:0] socket: 313 qp: -1
2025-12-24T02:11:39.327753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241404315871121:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.327760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.334072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405004068206:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:39.331245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241403201971936:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:39.333276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971937:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.335340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241403519290875:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:39.335461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290867:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.335534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241403519290876:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:39.335618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241403519290877:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:39.335854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241403201971937:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.352811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068211:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.355035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403619698885:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41084
2025-12-24T02:11:39.358852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698879:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.364158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432604:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.364221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432588:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.364225Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241401516432588:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.365244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241401516432609:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64239
2025-12-24T02:11:39.365297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241401516432610:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64239
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.368429Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403619698887:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41084
2025-12-24T02:11:39.368489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698878:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.370886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432609:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.371040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241401516432588:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.368310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241403519290806:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.374830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290878:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
2025-12-24T02:11:39.374820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241404315871114:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.374864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241404315871038:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.378748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290879:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
2025-12-24T02:11:39.378967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403619698861:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.383072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290879:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.383238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290887:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
2025-12-24T02:11:39.387463Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290887:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.387455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432599:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.387471Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241401516432599:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.388360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241401516432527:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.388382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.388388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241401516432527:2048] [node 2] ICP22 created new session: [1:7587241401516432614:2048]
2025-12-24T02:11:39.388395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241401516432614:2048] [node 2] ICS09 handshake done sender: [1:7587241401516432588:2107] self: [1:7587241401516432587:0] peer: [2:7587241403619698880:0] socket: 306 qp: -1
2025-12-24T02:11:39.388402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241401516432614:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:39.388411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.388721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432605:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.386984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241403201971865:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.394136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698870:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.394148Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403619698870:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.394469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241403619698798:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.394486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.394011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971945:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.394492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403619698798:2048] [node 1] ICP22 created new session: [2:7587241403619698889:2048]
2025-12-24T02:11:39.394499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403619698889:2048] [node 1] ICS09 handshake done sender: [2:7587241403619698861:2110] self: [2:7587241403619698880:0] peer: [1:7587241401516432587:0] socket: 308 qp: -1
2025-12-24T02:11:39.394505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403619698889:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.394514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.395154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698885:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.397618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068211:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.397715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241403201971945:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.397743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241403201971866:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.399182Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432606:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.399224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432610:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.406751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871128:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
2025-12-24T02:11:39.406743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290881:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.406765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241403519290881:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.410896Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068217:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.414481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698872:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.414503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403619698872:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.411357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290878:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.411513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241403519290805:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.414883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241403519290881:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.415006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241403619698798:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:39.416518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403619698872:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.411837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971949:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.415763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432597:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.415782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241401516432597:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.416358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241401516432599:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.418795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241403519290876:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.418819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.418828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241403519290876:2111] [node 5] ICP22 created new session: [3:7587241403519290893:2111]
2025-12-24T02:11:39.418837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241403519290893:2111] [node 5] ICS09 handshake done sender: [3:7587241403519290881:2115] self: [3:7587241403519290880:0] peer: [5:7587241403201971948:0] socket: 358 qp: -1
2025-12-24T02:11:39.418845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241403519290893:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:39.418856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.419281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290896:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
2025-12-24T02:11:39.419293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241403519290895:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19535
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.419529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241403201971945:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.419749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241403201971866:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.419760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.419765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241403201971866:2050] [node 3] ICP22 created new session: [5:7587241403201971951:2050]
2025-12-24T02:11:39.419770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241403201971951:2050] [node 3] ICS09 handshake done sender: [5:7587241403201971945:2116] self: [5:7587241403201971948:0] peer: [3:7587241403519290880:0] socket: 359 qp: -1
2025-12-24T02:11:39.419775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241403201971951:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.419781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.420195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971950:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.421888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241403201971937:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.421940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241403201971865:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.421953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.421959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241403201971865:2049] [node 2] ICP22 created new session: [5:7587241403201971953:2049]
2025-12-24T02:11:39.421965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241403201971953:2049] [node 2] ICS09 handshake done sender: [5:7587241403201971937:2113] self: [5:7587241403201971944:0] peer: [2:7587241403619698871:0] socket: 343 qp: -1
2025-12-24T02:11:39.421973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241403201971953:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.421980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.427113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241404315871114:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.430265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.429838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403619698863:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.429869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.429885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403619698863:2108] [node 5] ICP22 created new session: [2:7587241403619698895:2108]
2025-12-24T02:11:39.429893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403619698895:2108] [node 5] ICS09 handshake done sender: [2:7587241403619698872:2114] self: [2:7587241403619698871:0] peer: [5:7587241403201971944:0] socket: 341 qp: -1
2025-12-24T02:11:39.429901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403619698895:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:39.429913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.429845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241401516432593:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.429880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.429888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241401516432593:2108] [node 6] ICP22 created new session: [1:7587241401516432622:2108]
2025-12-24T02:11:39.429896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241401516432622:2108] [node 6] ICS09 handshake done sender: [1:7587241401516432599:2112] self: [1:7587241401516432598:0] peer: [6:7587241404315871127:0] socket: 336 qp: -1
2025-12-24T02:11:39.429904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241401516432622:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:39.429914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.437641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403619698870:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.438246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403619698799:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.438269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.438275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403619698799:2049] [node 3] ICP22 created new session: [2:7587241403619698897:2049]
2025-12-24T02:11:39.438280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403619698897:2049] [node 3] ICS09 handshake done sender: [2:7587241403619698870:2113] self: [2:7587241403619698869:0] peer: [3:7587241403519290886:0] socket: 311 qp: -1
2025-12-24T02:11:39.438285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403619698897:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:39.438291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.434901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241404315871038:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.434928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.434938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241404315871038:2048] [node 1] ICP22 created new session: [6:7587241404315871130:2048]
2025-12-24T02:11:39.434946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241404315871130:2048] [node 1] ICS09 handshake done sender: [6:7587241404315871114:2113] self: [6:7587241404315871127:0] peer: [1:7587241401516432598:0] socket: 338 qp: -1
2025-12-24T02:11:39.434954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241404315871130:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.434963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.438047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871129:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
2025-12-24T02:11:39.439189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068218:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.439226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405004068135:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.439368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403619698874:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.439377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403619698874:2115] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.440147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068220:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.440242Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403619698874:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.440411Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405004068218:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.440465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405004068135:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.440479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.440484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405004068135:2049] [node 2] ICP22 created new session: [4:7587241405004068221:2049]
2025-12-24T02:11:39.440493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405004068221:2049] [node 2] ICS09 handshake done sender: [4:7587241405004068218:2118] self: [4:7587241405004068219:0] peer: [2:7587241403619698873:0] socket: 368 qp: -1
2025-12-24T02:11:39.440499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405004068221:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.440506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.438980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241404315871129:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.439019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241404315871040:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.441327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403619698864:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.441345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.441351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403619698864:2109] [node 4] ICP22 created new session: [2:7587241403619698900:2109]
2025-12-24T02:11:39.441355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403619698900:2109] [node 4] ICS09 handshake done sender: [2:7587241403619698874:2115] self: [2:7587241403619698873:0] peer: [4:7587241405004068219:0] socket: 366 qp: -1
2025-12-24T02:11:39.441360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403619698900:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:39.441368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.442578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241403519290806:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241403519290867:2108] held: no
2025-12-24T02:11:39.442738Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290883:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.442744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241403519290883:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.443862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241403519290867:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.443918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241403519290806:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.443931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.443937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241403519290806:2049] [node 2] ICP22 created new session: [3:7587241403519290898:2049]
2025-12-24T02:11:39.443942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241403519290898:2049] [node 2] ICS09 handshake done sender: [3:7587241403519290867:2108] self: [3:7587241403519290886:0] peer: [2:7587241403619698869:0] socket: 319 qp: -1
2025-12-24T02:11:39.443948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241403519290898:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.443954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.444389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241403519290883:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.444456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241403519290877:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.444477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.444482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241403519290877:2112] [node 6] ICP22 created new session: [3:7587241403519290902:2112]
2025-12-24T02:11:39.444487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241403519290902:2112] [node 6] ICS09 handshake done sender: [3:7587241403519290883:2116] self: [3:7587241403519290882:0] peer: [6:7587241404315871132:0] socket: 371 qp: -1
2025-12-24T02:11:39.444492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241403519290902:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:39.444498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.446034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971955:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.447243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241401516432597:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.447684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241401516432528:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.447698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.447730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241401516432528:2049] [node 3] ICP22 created new session: [1:7587241401516432625:2049]
2025-12-24T02:11:39.447736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241401516432625:2049] [node 3] ICS09 handshake done sender: [1:7587241401516432597:2111] self: [1:7587241401516432596:0] peer: [3:7587241403519290892:0] socket: 335 qp: -1
2025-12-24T02:11:39.447742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241401516432625:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:39.447748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.451672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241403519290805:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241403519290878:2113] held: no
2025-12-24T02:11:39.452034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241403519290878:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.452090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241403519290805:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.452105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.452112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241403519290805:2048] [node 1] ICP22 created new session: [3:7587241403519290904:2048]
2025-12-24T02:11:39.452119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241403519290904:2048] [node 1] ICS09 handshake done sender: [3:7587241403519290878:2113] self: [3:7587241403519290892:0] peer: [1:7587241401516432596:0] socket: 346 qp: -1
2025-12-24T02:11:39.452126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241403519290904:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.452131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.451219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241404315871040:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241404315871129:2118] held: no
2025-12-24T02:11:39.451558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241404315871133:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51306
2025-12-24T02:11:39.452637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241404315871129:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.452686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241404315871040:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.452696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.452726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241404315871040:2050] [node 3] ICP22 created new session: [6:7587241404315871134:2050]
2025-12-24T02:11:39.452732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241404315871134:2050] [node 3] ICS09 handshake done sender: [6:7587241404315871129:2118] self: [6:7587241404315871132:0] peer: [3:7587241403519290882:0] socket: 344 qp: -1
2025-12-24T02:11:39.452737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241404315871134:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.452744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.470936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068217:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.471080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405004068134:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.475000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432601:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.475014Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241401516432601:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.474880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068224:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.478997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241401516432601:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.482730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971956:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.482894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241403201971955:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.483018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241403201971936:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.494896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241401516432594:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.494932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.494945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241401516432594:2109] [node 4] ICP22 created new session: [1:7587241401516432628:2109]
2025-12-24T02:11:39.494954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241401516432628:2109] [node 4] ICS09 handshake done sender: [1:7587241401516432601:2113] self: [1:7587241401516432600:0] peer: [4:7587241405004068225:0] socket: 360 qp: -1
2025-12-24T02:11:39.494963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241401516432628:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:39.494974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.498940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241403201971956:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.498999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241403201971864:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.499111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068224:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.502863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241401516432603:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.502878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241401516432603:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.503875Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241401516432603:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.506881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971959:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.507055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241403201971956:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.507108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241403201971864:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.507128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.507135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241403201971864:2048] [node 1] ICP22 created new session: [5:7587241403201971960:2048]
2025-12-24T02:11:39.507142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241403201971960:2048] [node 1] ICS09 handshake done sender: [5:7587241403201971956:2121] self: [5:7587241403201971958:0] peer: [1:7587241401516432602:0] socket: 331 qp: -1
2025-12-24T02:11:39.507150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241403201971960:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.507159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.515177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241401516432595:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.515213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.515223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241401516432595:2110] [node 5] ICP22 created new session: [1:7587241401516432631:2110]
2025-12-24T02:11:39.515232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241401516432631:2110] [node 5] ICS09 handshake done sender: [1:7587241401516432603:2114] self: [1:7587241401516432602:0] peer: [5:7587241403201971958:0] socket: 352 qp: -1
2025-12-24T02:11:39.515242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241401516432631:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:39.515254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.516186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405004068136:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.516223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068226:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.516409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405004068213:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.516412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241405004068213:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.516816Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241405004068134:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241405004068217:2117] held: no
2025-12-24T02:11:39.516972Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405004068217:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.517119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405004068134:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.517128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.517133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405004068134:2048] [node 1] ICP22 created new session: [4:7587241405004068228:2048]
2025-12-24T02:11:39.517139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405004068228:2048] [node 1] ICS09 handshake done sender: [4:7587241405004068217:2117] self: [4:7587241405004068225:0] peer: [1:7587241401516432600:0] socket: 361 qp: -1
2025-12-24T02:11:39.517145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405004068228:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.517153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.517589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405004068213:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.517647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241405004068206:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.517658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.517662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405004068206:2112] [node 5] ICP22 created new session: [4:7587241405004068231:2112]
2025-12-24T02:11:39.517667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405004068231:2112] [node 5] ICS09 handshake done sender: [4:7587241405004068213:2116] self: [4:7587241405004068212:0] peer: [5:7587241403201971957:0] socket: 329 qp: -1
2025-12-24T02:11:39.517671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405004068231:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:39.517675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.518799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241403519290885:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.518815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241403519290885:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.519526Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241403201971936:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23764) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241403201971955:2120] held: no
2025-12-24T02:11:39.519678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241403201971962:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39770
2025-12-24T02:11:39.519861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241403201971955:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.519910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241403201971936:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.519922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.519927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241403201971936:2112] [node 4] ICP22 created new session: [5:7587241403201971963:2112]
2025-12-24T02:11:39.519931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241403201971963:2112] [node 4] ICS09 handshake done sender: [5:7587241403201971955:2120] self: [5:7587241403201971957:0] peer: [4:7587241405004068212:0] socket: 347 qp: -1
2025-12-24T02:11:39.519936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241403201971963:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:39.519942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.523400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405004068233:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54364
2025-12-24T02:11:39.526823Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241403519290885:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.526911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241403519290875:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.526927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.526934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241403519290875:2110] [node 4] ICP22 created new session: [3:7587241403519290907:2110]
2025-12-24T02:11:39.526940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241403519290907:2110] [node 4] ICS09 handshake done sender: [3:7587241403519290885:2117] self: [3:7587241403519290884:0] peer: [4:7587241405004068227:0] socket: 320 qp: -1
2025-12-24T02:11:39.526945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241403519290907:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:39.526952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.531940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241405004068136:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241405004068224:2120] held: no
2025-12-24T02:11:39.532177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405004068224:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.538810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405004068136:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.538837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.538847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405004068136:2050] [node 3] ICP22 created new session: [4:7587241405004068234:2050]
2025-12-24T02:11:39.538853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405004068234:2050] [node 3] ICS09 handshake done sender: [4:7587241405004068224:2120] self: [4:7587241405004068227:0] peer: [3:7587241403519290884:0] socket: 357 qp: -1
2025-12-24T02:11:39.538862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405004068234:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.538873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.558518Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:39.558583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241403519290904:2048] [node 1] ICS01 socket: 346 reason# EndOfStream
2025-12-24T02:11:39.558594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241403519290805:2048] [node 1] ICP30 unregister session Session# [3:7587241403519290904:2048] VirtualId# [3:7587241403519290892:0]
2025-12-24T02:11:39.558601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241403519290904:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.558631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.558659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.558102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:39.558212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241403619698889:2048] [node 1] ICS07 socket disconnect 308 reason# EndOfStream
2025-12-24T02:11:39.558222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241403619698889:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.558274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.558297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241403619698889:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.558940Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241403619698798:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.558955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241403619698889:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.558960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241403619698798:2048] [node 1] ICP30 unregister session Session# [2:7587241403619698889:2048] VirtualId# [2:7587241403619698880:0]
2025-12-24T02:11:39.558985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.558992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241403619698798:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.558995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.561733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:39.561828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241405004068228:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-12-24T02:11:39.561838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241405004068134:2048] [node 1] ICP30 unregister session Session# [4:7587241405004068228:2048] VirtualId# [4:7587241405004068225:0]
2025-12-24T02:11:39.561845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241405004068228:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.561893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.561924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.562836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:39.562910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241403201971960:2048] [node 1] ICS01 socket: 331 reason# EndOfStream
2025-12-24T02:11:39.562916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241403201971864:2048] [node 1] ICP30 unregister session Session# [5:7587241403201971960:2048] VirtualId# [5:7587241403201971958:0]
2025-12-24T02:11:39.562922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241403201971960:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.562969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.562991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.567022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:39.567133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241404315871130:2048] [node 1] ICS07 socket disconnect 338 reason# EndOfStream
2025-12-24T02:11:39.567140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241404315871130:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.567182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.567203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241404315871130:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.567574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241404315871038:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.567587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241404315871130:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.567592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241404315871038:2048] [node 1] ICP30 unregister session Session# [6:7587241404315871130:2048] VirtualId# [6:7587241404315871127:0]
2025-12-24T02:11:39.567627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.567634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241404315871038:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.567638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61434) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.569716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:39.569825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241403519290898:2049] [node 2] ICS07 socket disconnect 319 reason# EndOfStream
2025-12-24T02:11:39.569830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241403519290898:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.569864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.569879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241403519290898:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.575324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241403519290806:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:39.575344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241403519290898:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.575614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241403519290806:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.575623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241403519290898:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.575631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241403519290806:2049] [node 2] ICP30 unregister session Session# [3:7587241403519290898:2049] VirtualId# [3:7587241403519290886:0]
2025-12-24T02:11:39.575656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.575669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241403519290806:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.575673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.578232Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:39.578308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241405004068221:2049] [node 2] ICS07 socket disconnect 368 reason# EndOfStream
2025-12-24T02:11:39.578314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241405004068221:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.578353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.578382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241405004068221:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.578722Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241405004068135:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.578735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241405004068221:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.578740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241405004068135:2049] [node 2] ICP30 unregister session Session# [4:7587241405004068221:2049] VirtualId# [4:7587241405004068219:0]
2025-12-24T02:11:39.578757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.578764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241405004068135:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.578767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.579787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241404315871038:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.579824Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:39.579925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241404315871121:2049] [node 2] ICS07 socket disconnect 313 reason# EndOfStream
2025-12-24T02:11:39.579930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241404315871121:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.579975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.579995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241404315871121:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.580279Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241404315871039:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.580289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241404315871121:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.580294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241404315871039:2049] [node 2] ICP30 unregister session Session# [6:7587241404315871121:2049] VirtualId# [6:7587241404315871117:0]
2025-12-24T02:11:39.580344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.580349Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241404315871039:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.580358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31904) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.581966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:39.582105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241403201971953:2049] [node 2] ICS01 socket: 343 reason# EndOfStream
2025-12-24T02:11:39.582113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241403201971865:2049] [node 2] ICP30 unregister session Session# [5:7587241403201971953:2049] VirtualId# [5:7587241403201971944:0]
2025-12-24T02:11:39.582120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241403201971953:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.582173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.582198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.590748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241405004068135:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.598469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:39.598601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241404315871134:2050] [node 3] ICS07 socket disconnect 344 reason# EndOfStream
2025-12-24T02:11:39.598608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241404315871134:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.598682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.598933Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:39.599002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241403201971951:2050] [node 3] ICS07 socket disconnect 359 reason# EndOfStream
2025-12-24T02:11:39.599007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241403201971951:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.599039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.599061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241403201971951:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.599409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241404315871134:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.599447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241404315871039:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.599676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241403201971866:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.599691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241403201971951:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.599696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241403201971866:2050] [node 3] ICP30 unregister session Session# [5:7587241403201971951:2050] VirtualId# [5:7587241403201971948:0]
2025-12-24T02:11:39.599728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.599735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241403201971866:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.599738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.599888Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241404315871040:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.599898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241404315871134:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.599902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241404315871040:2050] [node 3] ICP30 unregister session Session# [6:7587241404315871134:2050] VirtualId# [6:7587241404315871132:0]
2025-12-24T02:11:39.599926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.599930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241404315871040:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.599932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20300) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.606755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241403201971866:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:39.610821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:39.610925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241404315871112:2089] [node 4] ICS01 socket: 314 reason# EndOfStream
2025-12-24T02:11:39.610942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241404315871085:2089] [node 4] ICP30 unregister session Session# [6:7587241404315871112:2089] VirtualId# [6:7587241404315871086:0]
2025-12-24T02:11:39.610955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241404315871112:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.610993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:39.611028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:39.614735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241404315871040:2050] [node 3] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:39.647017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241405078494316:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:39.647032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241405078494317:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:39.652832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403107186171:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.652858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403107186172:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:39.662408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241402582515572:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.662430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241402582515573:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.670398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405500217839:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.670410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405500217840:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.670416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405500217841:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.659110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241404264193121:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.659123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241404264193122:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.659129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241404264193123:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.706746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241403367019268:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:39.706813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241403367019269:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:39.706820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241403367019270:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:39.729659Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241405078494377:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.730905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241403367019330:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.731186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403107186231:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-12-24T02:11:39.731436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193182:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.733107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217900:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.731470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515632:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
2025-12-24T02:11:39.734531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515632:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.735381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241405078494388:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.735398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405500217900:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.735578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405500217902:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:39.736106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405500217902:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.736279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217904:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.736902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217905:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.737059Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405500217900:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.736101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403107186233:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:39.736175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186231:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.736931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241403107186171:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.737152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405500217902:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.737167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.737174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405500217902:2109] [node 6] ICP22 created new session: [5:7587241405500217906:2109]
2025-12-24T02:11:39.737182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405500217906:2109] [node 6] ICS09 handshake done sender: [5:7587241405500217900:2108] self: [5:7587241405500217903:0] peer: [6:7587241403367019331:0] socket: 321 qp: -1
2025-12-24T02:11:39.737190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405500217906:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:39.737209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.735050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241402582515635:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:39.735161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241402582515637:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:39.736240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241403367019332:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.736255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241403367019332:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.736922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241403367019332:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.737058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241403367019330:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.737073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.737081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241403367019330:2108] [node 5] ICP22 created new session: [6:7587241403367019335:2108]
2025-12-24T02:11:39.737088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241403367019335:2108] [node 5] ICS09 handshake done sender: [6:7587241403367019332:2109] self: [6:7587241403367019331:0] peer: [5:7587241405500217903:0] socket: 319 qp: -1
2025-12-24T02:11:39.737096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241403367019335:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:39.737107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.737512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019337:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.738829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494381:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.738846Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241405078494381:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.739216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494379:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.739220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241405078494379:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.739837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241405078494381:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.740463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241405078494379:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.743058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193183:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.745916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403107186240:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-12-24T02:11:39.746494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403107186243:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:39.746645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403107186231:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.753335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241402582515647:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:39.749213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241404264193188:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:39.755050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241403107186248:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:39.755098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241403107186171:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.755115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.755123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403107186171:2048] [node 1] ICP22 created new session: [2:7587241403107186249:2048]
2025-12-24T02:11:39.755132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403107186249:2048] [node 1] ICS09 handshake done sender: [2:7587241403107186231:2107] self: [2:7587241403107186236:0] peer: [1:7587241405078494378:0] socket: 314 qp: -1
2025-12-24T02:11:39.755140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403107186249:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.755167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.758597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241402582515572:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.758662Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241402582515572:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24046005ns) {Configure:104(24045259ns) {ProcessConfigured:135(23533770ns) {IncomingHandshake:261(23515948ns) {DropIncomingHandshake:442(320ns) PrepareNewSessionHandshake:477(870ns) {SwitchToState:179(136ns)} GenerateSessionVirtualId:988(391ns) IssueIncomingHandshakeReply:193(23507431ns) {SwitchToState:179(538ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.760061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405500217912:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:39.760136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217909:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.761133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241405078494394:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:39.761173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241405078494316:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.761188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.761194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241405078494316:2048] [node 2] ICP22 created new session: [1:7587241405078494395:2048]
2025-12-24T02:11:39.761203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241405078494395:2048] [node 2] ICS09 handshake done sender: [1:7587241405078494379:2108] self: [1:7587241405078494378:0] peer: [2:7587241403107186236:0] socket: 312 qp: -1
2025-12-24T02:11:39.761212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241405078494395:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:39.761225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.761595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241405078494317:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.761604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.761609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241405078494317:2049] [node 3] ICP22 created new session: [1:7587241405078494397:2049]
2025-12-24T02:11:39.761617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241405078494397:2049] [node 3] ICS09 handshake done sender: [1:7587241405078494381:2109] self: [1:7587241405078494380:0] peer: [3:7587241402582515636:0] socket: 313 qp: -1
2025-12-24T02:11:39.761621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241405078494397:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:39.761626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.762940Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241403367019330:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(25840168ns) {HandleHandshakeStatus:329(25839226ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(162ns) SwitchToState:179(232ns) ProcessPendingSessionEvents:495(22931861ns) {ForwardSessionEventToSession:612(22917260ns) ForwardSessionEventToSession:612(2465ns) ForwardSessionEventToSession:612(1624ns) ForwardSessionEventToSession:612(810ns) ForwardSessionEventToSession:612(797ns) ForwardSessionEventToSession:612(1273ns) ForwardSessionEventToSession:612(1011ns) ForwardSessionEventToSession:612(645ns) ForwardSessionEventToSession:612(630ns) ForwardSessionEventToSession:612(1514ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.763249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515645:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.762905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241404264193193:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:39.763017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193182:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.764277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241404264193121:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.764333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193194:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-12-24T02:11:39.764371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193195:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-12-24T02:11:39.770896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193194:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.770944Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193183:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.772206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241404264193122:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.772224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241404264193123:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.765016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241402582515632:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.772457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515639:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.772464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241402582515639:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.772999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241402582515572:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.773019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.773027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241402582515572:2048] [node 1] ICP22 created new session: [3:7587241402582515653:2048]
2025-12-24T02:11:39.773036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241402582515653:2048] [node 1] ICS09 handshake done sender: [3:7587241402582515632:2107] self: [3:7587241402582515636:0] peer: [1:7587241405078494380:0] socket: 316 qp: -1
2025-12-24T02:11:39.773045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241402582515653:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.773054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.773663Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515652:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
2025-12-24T02:11:39.773791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515646:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
2025-12-24T02:11:39.773886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241402582515639:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.774033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241405078494401:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60451
2025-12-24T02:11:39.774244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494385:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.774248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241405078494385:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.776281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405500217904:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.776580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217922:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.776595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217923:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.776632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405500217841:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.776397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241402582515635:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.776420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.776429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241402582515635:2108] [node 4] ICP22 created new session: [3:7587241402582515658:2108]
2025-12-24T02:11:39.776436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241402582515658:2108] [node 4] ICS09 handshake done sender: [3:7587241402582515639:2110] self: [3:7587241402582515638:0] peer: [4:7587241404264193205:0] socket: 322 qp: -1
2025-12-24T02:11:39.776443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241402582515658:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:39.776466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.776658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515646:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.765578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241403367019337:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.766033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241403367019269:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.766780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186235:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.766797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403107186235:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.770038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403107186235:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.770149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403107186233:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.770166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.770180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403107186233:2108] [node 6] ICP22 created new session: [2:7587241403107186255:2108]
2025-12-24T02:11:39.770187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403107186255:2108] [node 6] ICS09 handshake done sender: [2:7587241403107186235:2109] self: [2:7587241403107186234:0] peer: [6:7587241403367019338:0] socket: 328 qp: -1
2025-12-24T02:11:39.770196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403107186255:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:39.770209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.773659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403107186253:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
2025-12-24T02:11:39.773756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186245:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.773760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403107186245:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.777927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403107186245:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.778135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403107186243:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.778149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.778156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403107186243:2112] [node 4] ICP22 created new session: [2:7587241403107186259:2112]
2025-12-24T02:11:39.778162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403107186259:2112] [node 4] ICS09 handshake done sender: [2:7587241403107186245:2113] self: [2:7587241403107186244:0] peer: [4:7587241404264193204:0] socket: 338 qp: -1
2025-12-24T02:11:39.778168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403107186259:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:39.778178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.778412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241403107186261:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42540
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.778733Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019340:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.778911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241405078494403:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60451
2025-12-24T02:11:39.779049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241405078494385:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.779069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019339:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.779237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019341:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.779829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515652:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.780132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186253:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.780157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241403107186243:2112] [node 4] ICP18 handshake (actor [2:7587241403107186253:2116]) is too old
2025-12-24T02:11:39.778259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405500217909:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.778297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405500217912:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.778468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405500217923:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.778486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405500217840:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.782846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405500217922:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.782932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405500217839:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.781610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241405078494377:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.781631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.781640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241405078494377:2107] [node 4] ICP22 created new session: [1:7587241405078494406:2107]
2025-12-24T02:11:39.781648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241405078494406:2107] [node 4] ICS09 handshake done sender: [1:7587241405078494385:2110] self: [1:7587241405078494384:0] peer: [4:7587241404264193202:0] socket: 315 qp: -1
2025-12-24T02:11:39.781656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241405078494406:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:39.781667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.781936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494401:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.781980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241405078494377:2107] [node 4] ICP18 handshake (actor [1:7587241405078494401:2114]) is too old
2025-12-24T02:11:39.782340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241402582515573:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.784003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193190:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.784020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241404264193190:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.783935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186252:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.783951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403107186252:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.786788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515641:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.786812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241402582515641:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.787188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494403:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.787241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494390:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.787244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241405078494390:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.788207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241403367019337:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.788436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193195:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.800394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241404264193121:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241404264193182:2108] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.789901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186239:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.789914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241403107186239:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.802837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.800569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193207:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-12-24T02:11:39.802910Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241405078494390:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.801135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241403367019340:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.801238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241403367019269:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.801255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.801289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241403367019269:2049] [node 2] ICP22 created new session: [6:7587241403367019345:2049]
2025-12-24T02:11:39.801296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241403367019345:2049] [node 2] ICS09 handshake done sender: [6:7587241403367019337:2110] self: [6:7587241403367019338:0] peer: [2:7587241403107186234:0] socket: 329 qp: -1
2025-12-24T02:11:39.801303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241403367019345:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.801314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.803271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241403367019270:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.803533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403107186252:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.803933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241403107186261:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.803948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403107186248:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.803962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.803970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403107186248:2114] [node 5] ICP22 created new session: [2:7587241403107186264:2114]
2025-12-24T02:11:39.803979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403107186264:2114] [node 5] ICS09 handshake done sender: [2:7587241403107186252:2115] self: [2:7587241403107186251:0] peer: [5:7587241405500217926:0] socket: 356 qp: -1
2025-12-24T02:11:39.803985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403107186264:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:39.803993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.804324Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241403367019341:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.804354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241403367019268:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.806355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241402582515650:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.806372Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241402582515650:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.807303Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241402582515641:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.811006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494402:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.811020Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241405078494402:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.811521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241405078494388:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.811545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.811557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241405078494388:2111] [node 5] ICP22 created new session: [1:7587241405078494411:2111]
2025-12-24T02:11:39.811566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241405078494411:2111] [node 5] ICS09 handshake done sender: [1:7587241405078494390:2112] self: [1:7587241405078494389:0] peer: [5:7587241405500217927:0] socket: 353 qp: -1
2025-12-24T02:11:39.811575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241405078494411:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:39.811586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.812716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241405078494402:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.814180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241405078494394:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.814198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.814210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241405078494394:2113] [node 6] ICP22 created new session: [1:7587241405078494414:2113]
2025-12-24T02:11:39.814217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241405078494414:2113] [node 6] ICS09 handshake done sender: [1:7587241405078494402:2115] self: [1:7587241405078494400:0] peer: [6:7587241403367019348:0] socket: 361 qp: -1
2025-12-24T02:11:39.814227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241405078494414:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:39.814239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.814466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241405078494413:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60451
2025-12-24T02:11:39.814653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241403107186239:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.812540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193208:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-12-24T02:11:39.814789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241403107186172:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.814809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.814815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241403107186172:2049] [node 3] ICP22 created new session: [2:7587241403107186267:2049]
2025-12-24T02:11:39.814820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241403107186267:2049] [node 3] ICS09 handshake done sender: [2:7587241403107186239:2110] self: [2:7587241403107186238:0] peer: [3:7587241402582515660:0] socket: 337 qp: -1
2025-12-24T02:11:39.814825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241403107186267:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:39.814832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.815043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241405078494413:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.815115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241405078494394:2113] [node 6] ICP18 handshake (actor [1:7587241405078494413:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.819438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019349:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.819471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019350:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.819654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241402582515650:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.818900Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241404264193122:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241404264193194:2114] held: no
2025-12-24T02:11:39.819253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241404264193206:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37995
2025-12-24T02:11:39.822986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241403367019340:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.823454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241403367019341:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.824071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241403367019270:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.824093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.824102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241403367019270:2050] [node 3] ICP22 created new session: [6:7587241403367019351:2050]
2025-12-24T02:11:39.824108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241403367019351:2050] [node 3] ICS09 handshake done sender: [6:7587241403367019340:2112] self: [6:7587241403367019347:0] peer: [3:7587241402582515648:0] socket: 368 qp: -1
2025-12-24T02:11:39.824116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241403367019351:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.824126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.824227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241403367019268:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.824240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.824245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241403367019268:2048] [node 1] ICP22 created new session: [6:7587241403367019353:2048]
2025-12-24T02:11:39.824249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241403367019353:2048] [node 1] ICS09 handshake done sender: [6:7587241403367019341:2113] self: [6:7587241403367019348:0] peer: [1:7587241405078494400:0] socket: 369 qp: -1
2025-12-24T02:11:39.824253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241403367019353:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.824257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.824309Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241403367019268:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:39.826865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405500217922:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.827027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405500217839:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241405500217922:2118] held: no
2025-12-24T02:11:39.827106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405500217839:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.827122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.827131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405500217839:2048] [node 1] ICP22 created new session: [5:7587241405500217932:2048]
2025-12-24T02:11:39.827138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405500217932:2048] [node 1] ICS09 handshake done sender: [5:7587241405500217922:2118] self: [5:7587241405500217927:0] peer: [1:7587241405078494389:0] socket: 364 qp: -1
2025-12-24T02:11:39.827148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405500217932:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.827159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.827415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217931:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.827544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217929:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.827806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405500217930:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23651
2025-12-24T02:11:39.828037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405500217923:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.828095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405500217904:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.828157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405500217840:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.828172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.828179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405500217840:2049] [node 2] ICP22 created new session: [5:7587241405500217934:2049]
2025-12-24T02:11:39.828188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405500217934:2049] [node 2] ICS09 handshake done sender: [5:7587241405500217923:2119] self: [5:7587241405500217926:0] peer: [2:7587241403107186251:0] socket: 365 qp: -1
2025-12-24T02:11:39.828193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405500217934:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.828203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.828539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405500217841:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.828548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.828552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405500217841:2050] [node 3] ICP22 created new session: [5:7587241405500217936:2050]
2025-12-24T02:11:39.828557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405500217936:2050] [node 3] ICS09 handshake done sender: [5:7587241405500217904:2110] self: [5:7587241405500217924:0] peer: [3:7587241402582515640:0] socket: 324 qp: -1
2025-12-24T02:11:39.828562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405500217936:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.828567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.830887Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241403107186172:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(16022540ns) {HandleHandshakeStatus:329(16021770ns) {DropIncomingHandshake:442(4031ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(40ns) ProcessPendingSessionEvents:495(15936521ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 15926156ns max# 15887409ns}}}
2025-12-24T02:11:39.831526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241404264193194:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.834862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241402582515637:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.834896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.834905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241402582515637:2109] [node 5] ICP22 created new session: [3:7587241402582515665:2109]
2025-12-24T02:11:39.834912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241402582515665:2109] [node 5] ICS09 handshake done sender: [3:7587241402582515641:2111] self: [3:7587241402582515640:0] peer: [5:7587241405500217924:0] socket: 323 qp: -1
2025-12-24T02:11:39.834919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241402582515665:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:39.834930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:39.835229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241402582515647:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.835235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.835239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241402582515647:2112] [node 6] ICP22 created new session: [3:7587241402582515667:2112]
2025-12-24T02:11:39.835244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241402582515667:2112] [node 6] ICS09 handshake done sender: [3:7587241402582515650:2115] self: [3:7587241402582515648:0] peer: [6:7587241403367019347:0] socket: 358 qp: -1
2025-12-24T02:11:39.835249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241402582515667:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:39.835254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:39.836038Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241402582515573:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241402582515646:2114] held: no
2025-12-24T02:11:39.836357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515664:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
2025-12-24T02:11:39.836483Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241402582515646:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:39.836527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241402582515573:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.836537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.836542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241402582515573:2049] [node 2] ICP22 created new session: [3:7587241402582515669:2049]
2025-12-24T02:11:39.836547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241402582515669:2049] [node 2] ICS09 handshake done sender: [3:7587241402582515646:2114] self: [3:7587241402582515660:0] peer: [2:7587241403107186238:0] socket: 341 qp: -1
2025-12-24T02:11:39.836554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241402582515669:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.836559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.842958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241404264193182:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:39.843078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241404264193122:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.843092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:39.843098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241404264193122:2049] [node 2] ICP22 created new session: [4:7587241404264193211:2049]
2025-12-24T02:11:39.843106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241404264193211:2049] [node 2] ICS09 handshake done sender: [4:7587241404264193194:2114] self: [4:7587241404264193204:0] peer: [2:7587241403107186244:0] socket: 346 qp: -1
2025-12-24T02:11:39.843113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241404264193211:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:39.843121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:39.843418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241404264193183:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:39.847123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241404264193190:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:39.847336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241404264193121:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.847355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:39.847365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241404264193121:2048] [node 1] ICP22 created new session: [4:7587241404264193215:2048]
2025-12-24T02:11:39.847372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241404264193215:2048] [node 1] ICS09 handshake done sender: [4:7587241404264193182:2108] self: [4:7587241404264193202:0] peer: [1:7587241405078494384:0] socket: 317 qp: -1
2025-12-24T02:11:39.847381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241404264193215:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:39.847387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:39.847628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241404264193123:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.847637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:39.847643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241404264193123:2050] [node 3] ICP22 created new session: [4:7587241404264193217:2050]
2025-12-24T02:11:39.847648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241404264193217:2050] [node 3] ICS09 handshake done sender: [4:7587241404264193183:2109] self: [4:7587241404264193205:0] peer: [3:7587241402582515638:0] socket: 331 qp: -1
2025-12-24T02:11:39.847653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241404264193217:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:39.847659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:39.848838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241404264193188:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.848858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:39.848864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241404264193188:2111] [node 5] ICP22 created new session: [4:7587241404264193219:2111]
2025-12-24T02:11:39.848870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241404264193219:2111] [node 5] ICS09 handshake done sender: [4:7587241404264193190:2112] self: [4:7587241404264193189:0] peer: [5:7587241405500217925:0] socket: 339 qp: -1
2025-12-24T02:11:39.848875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241404264193219:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:39.848880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.851439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019355:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.854870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405500217912:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241405500217909:2113] held: no
2025-12-24T02:11:39.854981Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405500217909:2113] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.855412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405500217912:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.855426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.855433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405500217912:2112] [node 4] ICP22 created new session: [5:7587241405500217938:2112]
2025-12-24T02:11:39.855439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405500217938:2112] [node 4] ICS09 handshake done sender: [5:7587241405500217909:2113] self: [5:7587241405500217925:0] peer: [4:7587241404264193189:0] socket: 342 qp: -1
2025-12-24T02:11:39.855447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405500217938:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:39.855456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.855385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241402582515671:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38246
2025-12-24T02:11:39.858946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241403367019355:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.859200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241403367019357:2118] [node 4] ICP01 ready to work
2025-12-24T02:11:39.859599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241403367019357:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:39.862172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241404264193201:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:39.862185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241404264193201:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:39.862946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241404264193201:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:39.865978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241404264193193:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:39.865998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:39.866017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241404264193193:2113] [node 6] ICP22 created new session: [4:7587241404264193222:2113]
2025-12-24T02:11:39.866025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241404264193222:2113] [node 6] ICS09 handshake done sender: [4:7587241404264193201:2118] self: [4:7587241404264193200:0] peer: [6:7587241403367019358:0] socket: 343 qp: -1
2025-12-24T02:11:39.866033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241404264193222:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:39.866043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:39.871113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241403367019359:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7776
2025-12-24T02:11:39.871407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241403367019355:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:39.886514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241403367019357:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:39.886549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:39.886585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241403367019357:2118] [node 4] ICP22 created new session: [6:7587241403367019360:2118]
2025-12-24T02:11:39.886593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241403367019360:2118] [node 4] ICS09 handshake done sender: [6:7587241403367019355:2117] self: [6:7587241403367019358:0] peer: [4:7587241404264193200:0] socket: 344 qp: -1
2025-12-24T02:11:39.886606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241403367019360:2118] [node 4] ICS10 traffic start
2025-12-24T02:11:39.886619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:39.932709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:39.932820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241403107186249:2048] [node 1] ICS07 socket disconnect 314 reason# EndOfStream
2025-12-24T02:11:39.932831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241403107186249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.932870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.932886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241403107186249:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.934787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:39.934896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241402582515653:2048] [node 1] ICS07 socket disconnect 316 reason# EndOfStream
2025-12-24T02:11:39.934903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241402582515653:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.934964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.934988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241402582515653:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.938913Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241402582515572:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.938937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241402582515653:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.938949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241402582515572:2048] [node 1] ICP30 unregister session Session# [3:7587241402582515653:2048] VirtualId# [3:7587241402582515636:0]
2025-12-24T02:11:39.939016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.939027Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241402582515572:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.939030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.939427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241403107186171:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:39.939443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241403107186249:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.939730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241403107186171:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.939741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241403107186249:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.939746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241403107186171:2048] [node 1] ICP30 unregister session Session# [2:7587241403107186249:2048] VirtualId# [2:7587241403107186236:0]
2025-12-24T02:11:39.939793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.939798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241403107186171:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.939801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.942543Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:39.942637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405500217932:2048] [node 1] ICS01 socket: 364 reason# EndOfStream
2025-12-24T02:11:39.942644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405500217839:2048] [node 1] ICP30 unregister session Session# [5:7587241405500217932:2048] VirtualId# [5:7587241405500217927:0]
2025-12-24T02:11:39.942651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405500217932:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.942712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.942740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.942839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:39.942935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241404264193215:2048] [node 1] ICS01 socket: 317 reason# EndOfStream
2025-12-24T02:11:39.942943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241404264193121:2048] [node 1] ICP30 unregister session Session# [4:7587241404264193215:2048] VirtualId# [4:7587241404264193202:0]
2025-12-24T02:11:39.942949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241404264193215:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.943002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.943030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.942854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:39.942954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241403367019353:2048] [node 1] ICS07 socket disconnect 369 reason# EndOfStream
2025-12-24T02:11:39.942959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241403367019353:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.943010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:39.943033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241403367019353:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:39.943369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241403367019268:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.943377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241403367019353:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.943383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241403367019268:2048] [node 1] ICP30 unregister session Session# [6:7587241403367019353:2048] VirtualId# [6:7587241403367019348:0]
2025-12-24T02:11:39.943399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:39.943405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241403367019268:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.943408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9196) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.948624Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:39.948705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241402582515669:2049] [node 2] ICS07 socket disconnect 341 reason# EndOfStream
2025-12-24T02:11:39.948711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241402582515669:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.948743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.948758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241402582515669:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.954738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241403367019268:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:39.958771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:39.958890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241404264193211:2049] [node 2] ICS07 socket disconnect 346 reason# EndOfStream
2025-12-24T02:11:39.958899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241404264193211:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.958953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.958987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241404264193211:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.958813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:39.958898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241403367019345:2049] [node 2] ICS07 socket disconnect 329 reason# EndOfStream
2025-12-24T02:11:39.958904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241403367019345:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.958953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.958978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241403367019345:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.959367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241403367019269:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.959381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241403367019345:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.959391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241403367019269:2049] [node 2] ICP30 unregister session Session# [6:7587241403367019345:2049] VirtualId# [6:7587241403367019338:0]
2025-12-24T02:11:39.959425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.959432Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241403367019269:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.959435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.959411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241404264193122:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.959423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241404264193211:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.959428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241404264193122:2049] [node 2] ICP30 unregister session Session# [4:7587241404264193211:2049] VirtualId# [4:7587241404264193204:0]
2025-12-24T02:11:39.959446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.959452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241404264193122:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.959455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.959490Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:39.959568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241405500217934:2049] [node 2] ICS07 socket disconnect 365 reason# EndOfStream
2025-12-24T02:11:39.959574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405500217934:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.959624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:39.959649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241405500217934:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:39.959960Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405500217840:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.959970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405500217934:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.959974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405500217840:2049] [node 2] ICP30 unregister session Session# [5:7587241405500217934:2049] VirtualId# [5:7587241405500217926:0]
2025-12-24T02:11:39.960002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:39.960006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241405500217840:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.960008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11430) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.970752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241404264193122:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.970752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241405500217840:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.974742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241403367019269:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:39.986775Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:39.986892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241403367019351:2050] [node 3] ICS07 socket disconnect 368 reason# EndOfStream
2025-12-24T02:11:39.986901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241403367019351:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.986957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.986982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241403367019351:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.987394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241403367019270:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.987407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241403367019351:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.987412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241403367019270:2050] [node 3] ICP30 unregister session Session# [6:7587241403367019351:2050] VirtualId# [6:7587241403367019347:0]
2025-12-24T02:11:39.987444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.987451Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241403367019270:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.987454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.987636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:39.987721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241405500217936:2050] [node 3] ICS07 socket disconnect 324 reason# EndOfStream
2025-12-24T02:11:39.987728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405500217936:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:39.987781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:39.987803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241405500217936:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:39.988122Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405500217841:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:39.988134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405500217936:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:39.988138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405500217841:2050] [node 3] ICP30 unregister session Session# [5:7587241405500217936:2050] VirtualId# [5:7587241405500217924:0]
2025-12-24T02:11:39.988161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:39.988167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241405500217841:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:39.988170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26261) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.006517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241403367019270:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:40.006573Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:40.006680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241403367019360:2118] [node 4] ICS07 socket disconnect 344 reason# EndOfStream
2025-12-24T02:11:40.006689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241403367019360:2118] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.006760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:40.006782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241403367019360:2118] [node 4] ICS15 start handshake
2025-12-24T02:11:40.014912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241403367019357:2118] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:40.014930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241403367019360:2118] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:40.014936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241403367019357:2118] [node 4] ICP30 unregister session Session# [6:7587241403367019360:2118] VirtualId# [6:7587241403367019358:0]
2025-12-24T02:11:40.015007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:40.015016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241403367019357:2118] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.015020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27540) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:40.079034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407874973355:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.079052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407874973356:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:40.078840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409786776717:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:40.078861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409786776718:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:40.084629Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241408255347951:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.084640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241408255347952:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.094922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241407502514582:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.094936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241407502514583:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.094943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241407502514584:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.106859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241409369675663:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.106877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241409369675664:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.106884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241409369675665:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.138686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407164375667:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.138715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407164375668:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.138747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407164375666:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.138849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241408255348005:2092] [node 4] ICP01 ready to work
2025-12-24T02:11:40.138867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241408255348006:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:40.138585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407874973413:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:40.138746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409786776776:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:40.138879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409786776778:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:40.138866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241407502514612:2073] [node 6] ICP01 ready to work
2025-12-24T02:11:40.138942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241407502514615:2075] [node 5] ICP01 ready to work
2025-12-24T02:11:40.147603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241409369675684:2065] [node 6] ICP01 ready to work
2025-12-24T02:11:40.146639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407874973420:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:40.147783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407164375688:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.149823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514630:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.155024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241409786776776:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16087125ns) {Configure:104(16086641ns) {ProcessConfigured:135(15609583ns) {StartInitialHandshake:157(15609378ns) {DropHandshakes:470(305ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1028ns) {SwitchToState:179(105ns)}}}}}
2025-12-24T02:11:40.155630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409786776788:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:40.156117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409786776789:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38729
2025-12-24T02:11:40.156168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409786776790:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38729
2025-12-24T02:11:40.156572Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241409786776778:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17343088ns) {Configure:104(17341855ns) {ProcessConfigured:135(16930237ns) {StartInitialHandshake:157(16930055ns) {DropHandshakes:470(252ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(593ns) {SwitchToState:179(273ns)}}}}}
2025-12-24T02:11:40.154748Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375697:2071] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.154826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375698:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.156886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675703:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.156974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776789:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.158924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407164375697:2071] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.159753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407164375706:2078] [node 5] ICP01 ready to work
2025-12-24T02:11:40.160086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407164375706:2078] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.160318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241408255347952:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21514450ns) {Configure:104(21513441ns) {ProcessConfigured:135(20188788ns) {StartInitialHandshake:157(20187861ns) {DropHandshakes:470(268ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(35ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1206ns) {SwitchToState:179(277ns)}}}}}
2025-12-24T02:11:40.160458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241408255348018:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:40.161591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348019:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.161610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348020:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.161633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241409369675686:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.161647Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241409369675686:2066] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.161967Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241408255348006:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23051096ns) {Configure:104(23050283ns) {ProcessConfigured:135(21728234ns) {StartInitialHandshake:157(21727692ns) {DropHandshakes:470(183ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(52ns) PrepareNewSessionHandshake:477(658ns) {SwitchToState:179(213ns)}}}}}
2025-12-24T02:11:40.163455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407164375698:2072] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.163485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407164375688:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.163612Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776790:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.166727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973427:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.169560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241409369675703:2079] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.170873Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241409786776718:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19445893ns) {Configure:104(19445394ns) {ProcessConfigured:135(19144377ns) {StartInitialHandshake:157(19143836ns) {DropHandshakes:470(414ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1343ns) {SwitchToState:179(534ns)}}}}}
2025-12-24T02:11:40.170862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407874973429:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:40.175171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241407502514582:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(36697337ns) {Configure:104(36695707ns) {ProcessConfigured:135(36208613ns) {StartInitialHandshake:157(36207760ns) {DropHandshakes:470(2433ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(1620ns) {SwitchToState:179(727ns)}}}}}
2025-12-24T02:11:40.175240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514629:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.175245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241407502514629:2089] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.176052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514630:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.178367Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241407502514615:2075] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39366251ns) {Configure:104(39365721ns) {ProcessConfigured:135(39050180ns) {StartInitialHandshake:157(39049981ns) {DropHandshakes:470(394ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(176ns) PrepareNewSessionHandshake:477(1268ns) {SwitchToState:179(220ns)}}}}}
2025-12-24T02:11:40.176209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241407164375668:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17170782ns) {Configure:104(17170217ns) {ProcessConfigured:135(16818625ns) {StartInitialHandshake:157(16818217ns) {DropHandshakes:470(417ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1203ns) {SwitchToState:179(470ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.178818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241407502514629:2089] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.178841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348020:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.180596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241407502514612:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.180629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.180638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241407502514612:2073] [node 6] ICP22 created new session: [4:7587241407502514664:2073]
2025-12-24T02:11:40.180648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241407502514664:2073] [node 6] ICS09 handshake done sender: [4:7587241407502514629:2089] self: [4:7587241407502514616:0] peer: [6:7587241407164375731:0] socket: 328 qp: -1
2025-12-24T02:11:40.180656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241407502514664:2073] [node 6] ICS10 traffic start
2025-12-24T02:11:40.180681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.178829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409786776800:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38729
2025-12-24T02:11:40.178919Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241407874973355:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31472861ns) {Configure:104(31472546ns) {ProcessConfigured:135(31248010ns) {StartInitialHandshake:157(31247237ns) {DropHandshakes:470(235ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(151ns) PrepareNewSessionHandshake:477(1429ns) {SwitchToState:179(827ns)}}}}}
2025-12-24T02:11:40.178975Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973427:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.183081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241409369675732:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.186794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514663:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.187148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407164375688:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407164375698:2072] held: no
2025-12-24T02:11:40.190942Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241408255348005:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52009015ns) {Configure:104(52008396ns) {ProcessConfigured:135(50571843ns) {StartInitialHandshake:157(50570908ns) {DropHandshakes:470(256ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1491ns) {SwitchToState:179(464ns)}}}}}
2025-12-24T02:11:40.191041Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776800:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.191181Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241407874973429:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19953675ns) {Configure:104(19953065ns) {ProcessConfigured:135(19532127ns) {StartInitialHandshake:157(19531847ns) {DropHandshakes:470(849ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(233ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(430ns) {SwitchToState:179(78ns)}}}}}
2025-12-24T02:11:40.191314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241409369675663:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.191345Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241409369675663:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(21606547ns) {IncomingHandshake:261(21606180ns) {DropIncomingHandshake:442(73ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(262ns)} GenerateSessionVirtualId:988(649ns) IssueIncomingHandshakeReply:193(21600333ns) {SwitchToState:179(294ns)}}}
2025-12-24T02:11:40.191449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348019:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.191487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241408255347952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.191423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241409369675686:2066] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.191451Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675742:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.191641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241409369675742:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.191665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241409369675665:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.194725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776785:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.194741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409786776785:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.194799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514663:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.196307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241409369675684:2065] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.196331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.196344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241409369675684:2065] [node 6] ICP22 created new session: [5:7587241409369675746:2065]
2025-12-24T02:11:40.196353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241409369675746:2065] [node 6] ICS09 handshake done sender: [5:7587241409369675686:2066] self: [5:7587241409369675685:0] peer: [6:7587241407164375707:0] socket: 322 qp: -1
2025-12-24T02:11:40.196363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241409369675746:2065] [node 6] ICS10 traffic start
2025-12-24T02:11:40.196390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.198754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348022:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.198787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973422:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.198772Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241408255348022:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.198804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407874973422:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.196990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241407502514582:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.197229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514668:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.202934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375747:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.203170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973436:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.203223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973437:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.203260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973438:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.203300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973439:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.207913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776792:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.207930Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409786776792:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.207436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973436:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.207471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973439:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.208065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973437:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.208124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973438:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.208157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241407874973355:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.208509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409786776785:2110] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.210928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675749:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.208306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514668:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.208345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241407502514583:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.211172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241409369675703:2079] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.210939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375748:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.211011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375749:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.212386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407164375697:2071] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.212983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407164375748:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.212999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407164375706:2078] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.213016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.213027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407164375706:2078] [node 5] ICP22 created new session: [6:7587241407164375750:2078]
2025-12-24T02:11:40.213034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241407164375750:2078] [node 5] ICS09 handshake done sender: [6:7587241407164375697:2071] self: [6:7587241407164375707:0] peer: [5:7587241409369675685:0] socket: 333 qp: -1
2025-12-24T02:11:40.213042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241407164375750:2078] [node 5] ICS10 traffic start
2025-12-24T02:11:40.213064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.214879Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241408255348022:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.214997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241408255348018:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.215009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.215016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241408255348018:2105] [node 5] ICP22 created new session: [3:7587241408255348035:2105]
2025-12-24T02:11:40.215022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241408255348035:2105] [node 5] ICS09 handshake done sender: [3:7587241408255348022:2111] self: [3:7587241408255348021:0] peer: [5:7587241409369675744:0] socket: 345 qp: -1
2025-12-24T02:11:40.215029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241408255348035:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:40.215038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.213178Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407874973422:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.213249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407874973356:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.213265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.213271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407874973356:2049] [node 3] ICP22 created new session: [2:7587241407874973445:2049]
2025-12-24T02:11:40.213278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407874973445:2049] [node 3] ICS09 handshake done sender: [2:7587241407874973422:2110] self: [2:7587241407874973421:0] peer: [3:7587241408255348033:0] socket: 325 qp: -1
2025-12-24T02:11:40.213283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407874973445:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:40.213289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.214764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973424:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.214775Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407874973424:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.216034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776787:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.216046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409786776787:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.216396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409786776776:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.216410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.216416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409786776776:2101] [node 5] ICP22 created new session: [1:7587241409786776805:2101]
2025-12-24T02:11:40.216423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409786776805:2101] [node 5] ICS09 handshake done sender: [1:7587241409786776785:2110] self: [1:7587241409786776781:0] peer: [5:7587241409369675739:0] socket: 337 qp: -1
2025-12-24T02:11:40.216428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409786776805:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:40.216436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.217338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409786776792:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.217678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409786776787:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.220206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407874973448:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47934
2025-12-24T02:11:40.217185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514670:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.217512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241407502514663:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.217746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241407502514582:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.217761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.217768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241407502514582:2048] [node 1] ICP22 created new session: [4:7587241407502514671:2048]
2025-12-24T02:11:40.217774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241407502514671:2048] [node 1] ICS09 handshake done sender: [4:7587241407502514663:2116] self: [4:7587241407502514667:0] peer: [1:7587241409786776791:0] socket: 348 qp: -1
2025-12-24T02:11:40.217780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241407502514671:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.217787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.218784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409786776788:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.218802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.218810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409786776788:2105] [node 4] ICP22 created new session: [1:7587241409786776809:2105]
2025-12-24T02:11:40.218816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409786776809:2105] [node 4] ICS09 handshake done sender: [1:7587241409786776792:2114] self: [1:7587241409786776791:0] peer: [4:7587241407502514667:0] socket: 339 qp: -1
2025-12-24T02:11:40.218820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409786776809:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:40.218826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.221027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514674:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.220636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409786776808:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38729
2025-12-24T02:11:40.220868Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241409786776788:2105] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.220875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409786776717:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:40.220886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
::ffff:127.1.0.0
2025-12-24T02:11:40.220893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409786776717:2048] [node 2] ICP22 created new session: [1:7587241409786776813:2048]
2025-12-24T02:11:40.220899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409786776813:2048] [node 2] ICS09 handshake done sender: [1:7587241409786776787:2111] self: [1:7587241409786776786:0] peer: [2:7587241407874973443:0] socket: 338 qp: -1
2025-12-24T02:11:40.220904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409786776813:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:40.220912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.221142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409786776812:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38729
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.222776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241409369675663:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.222801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.222810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241409369675663:2048] [node 1] ICP22 created new session: [5:7587241409369675750:2048]
2025-12-24T02:11:40.222817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241409369675750:2048] [node 1] ICS09 handshake done sender: [5:7587241409369675703:2079] self: [5:7587241409369675739:0] peer: [1:7587241409786776781:0] socket: 340 qp: -1
2025-12-24T02:11:40.222824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241409369675750:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.222846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.223948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241409369675663:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.224015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675752:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.224292Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675753:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.227274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241409369675742:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.226985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407874973424:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.227126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407874973438:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.227404Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348040:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.227455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241408255347952:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241408255348019:2109] held: no
2025-12-24T02:11:40.227551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348037:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.227605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348038:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.227642Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348039:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.228113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407164375698:2072] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.228258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407164375667:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.228320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375752:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.228347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.229300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348037:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.229333Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348038:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.229366Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348040:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.229389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241408255348019:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.227846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241409369675753:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.229418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241409369675732:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.229676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973426:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.229687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407874973426:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.231579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407874973426:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.231657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407164375752:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.231698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407164375666:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.231389Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241408255348018:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(16271265ns) {HandleHandshakeStatus:329(16270764ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(182ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(5928346ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5903348ns max# 5811934ns}}}
2025-12-24T02:11:40.231408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241408255347952:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.231417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.231422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241408255347952:2049] [node 2] ICP22 created new session: [3:7587241408255348043:2049]
2025-12-24T02:11:40.231432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241408255348043:2049] [node 2] ICS09 handshake done sender: [3:7587241408255348019:2109] self: [3:7587241408255348033:0] peer: [2:7587241407874973421:0] socket: 342 qp: -1
2025-12-24T02:11:40.231439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241408255348043:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.231446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.231644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241408255347951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.231872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241408255348018:2105] [node 5] ICP18 handshake (actor [3:7587241408255348037:2117]) is too old
2025-12-24T02:11:40.230763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241409369675665:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.230790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.230797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241409369675665:2050] [node 3] ICP22 created new session: [5:7587241409369675756:2050]
2025-12-24T02:11:40.230805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241409369675756:2050] [node 3] ICS09 handshake done sender: [5:7587241409369675742:2115] self: [5:7587241409369675744:0] peer: [3:7587241408255348021:0] socket: 350 qp: -1
2025-12-24T02:11:40.230813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241409369675756:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.230823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.232078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407164375688:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.232093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.232098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407164375688:2059] [node 4] ICP22 created new session: [6:7587241407164375756:2059]
2025-12-24T02:11:40.232103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241407164375756:2059] [node 4] ICS09 handshake done sender: [6:7587241407164375698:2072] self: [6:7587241407164375731:0] peer: [4:7587241407502514616:0] socket: 334 qp: -1
2025-12-24T02:11:40.232110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241407164375756:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:40.232118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.231878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776796:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.231890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409786776796:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.232246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776798:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.232248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409786776798:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.232871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409786776796:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.232982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409786776798:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.233051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407874973420:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.233075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.233082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407874973420:2109] [node 4] ICP22 created new session: [2:7587241407874973452:2109]
2025-12-24T02:11:40.233088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407874973452:2109] [node 4] ICS09 handshake done sender: [2:7587241407874973424:2111] self: [2:7587241407874973423:0] peer: [4:7587241407502514669:0] socket: 354 qp: -1
2025-12-24T02:11:40.233094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407874973452:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:40.233102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.233434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241407874973355:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.233442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.233446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407874973355:2048] [node 1] ICP22 created new session: [2:7587241407874973454:2048]
2025-12-24T02:11:40.233451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407874973454:2048] [node 1] ICS09 handshake done sender: [2:7587241407874973438:2118] self: [2:7587241407874973443:0] peer: [1:7587241409786776786:0] socket: 365 qp: -1
2025-12-24T02:11:40.233455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407874973454:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.233460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.233814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409786776778:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.233831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.233838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409786776778:2103] [node 6] ICP22 created new session: [1:7587241409786776817:2103]
2025-12-24T02:11:40.233843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409786776817:2103] [node 6] ICS09 handshake done sender: [1:7587241409786776796:2115] self: [1:7587241409786776782:0] peer: [6:7587241407164375755:0] socket: 372 qp: -1
2025-12-24T02:11:40.233848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409786776817:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:40.233854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.234056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409786776812:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.234396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409786776718:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.234412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.234418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409786776718:2049] [node 3] ICP22 created new session: [1:7587241409786776819:2049]
2025-12-24T02:11:40.234423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409786776819:2049] [node 3] ICS09 handshake done sender: [1:7587241409786776798:2116] self: [1:7587241409786776783:0] peer: [3:7587241408255348046:0] socket: 326 qp: -1
2025-12-24T02:11:40.234428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409786776819:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:40.234433Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.234830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241409786776717:2048] [node 2] ICP18 handshake (actor [1:7587241409786776812:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.235046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407874973413:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.235065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.235070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407874973413:2104] [node 6] ICP22 created new session: [2:7587241407874973456:2104]
2025-12-24T02:11:40.236913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675758:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.235076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407874973456:2104] [node 6] ICS09 handshake done sender: [2:7587241407874973426:2112] self: [2:7587241407874973419:0] peer: [6:7587241407164375754:0] socket: 353 qp: -1
2025-12-24T02:11:40.235081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407874973456:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:40.235086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.237171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241408255348047:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50812
2025-12-24T02:11:40.236610Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514676:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.236742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514675:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.236789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241407502514668:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.237198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407164375753:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.237358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407164375668:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.237454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348028:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.237461Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241408255348028:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.237698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514662:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.237706Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241407502514662:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.237807Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407164375666:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18839) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407164375752:2119] held: no
2025-12-24T02:11:40.238145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514676:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.238160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241407502514583:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.238173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.238181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241407502514583:2049] [node 2] ICP22 created new session: [4:7587241407502514677:2049]
2025-12-24T02:11:40.238187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241407502514677:2049] [node 2] ICS09 handshake done sender: [4:7587241407502514668:2117] self: [4:7587241407502514669:0] peer: [2:7587241407874973423:0] socket: 357 qp: -1
2025-12-24T02:11:40.238193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241407502514677:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.238200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.238524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241407502514675:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.239977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241407502514584:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.243014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241409369675758:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.243061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241409369675664:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.246880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407874973435:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.246895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407874973435:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.247460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241408255348031:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.247475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241408255348031:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.249436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241408255348040:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.249634Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407164375667:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407164375748:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.250870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375759:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.251177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375760:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.250870Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241408255347951:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18839) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241408255348040:2120] held: no
2025-12-24T02:11:40.251208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241408255347951:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.251223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.251231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241408255347951:2048] [node 1] ICP22 created new session: [3:7587241408255348050:2048]
2025-12-24T02:11:40.251239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241408255348050:2048] [node 1] ICS09 handshake done sender: [3:7587241408255348040:2120] self: [3:7587241408255348046:0] peer: [1:7587241409786776783:0] socket: 382 qp: -1
2025-12-24T02:11:40.251248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241408255348050:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.251267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.252109Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407874973435:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.252967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407874973429:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.252985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.252990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407874973429:2106] [node 5] ICP22 created new session: [2:7587241407874973459:2106]
2025-12-24T02:11:40.252997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407874973459:2106] [node 5] ICS09 handshake done sender: [2:7587241407874973435:2115] self: [2:7587241407874973432:0] peer: [5:7587241409369675759:0] socket: 389 qp: -1
2025-12-24T02:11:40.253002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407874973459:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:40.253010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.251730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241407502514662:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.252079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241407502514682:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9582
2025-12-24T02:11:40.252194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241407502514615:2075] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.252210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.252218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241407502514615:2075] [node 5] ICP22 created new session: [4:7587241407502514683:2075]
2025-12-24T02:11:40.252224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241407502514683:2075] [node 5] ICS09 handshake done sender: [4:7587241407502514662:2115] self: [4:7587241407502514617:0] peer: [5:7587241409369675755:0] socket: 375 qp: -1
2025-12-24T02:11:40.252229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241407502514683:2075] [node 5] ICS10 traffic start
2025-12-24T02:11:40.252236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.258857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241408255348031:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.258940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241408255348028:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.259395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675760:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.260795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241407502514676:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.263045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407164375752:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.263724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241408255348005:2092] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.263758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.263766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241408255348005:2092] [node 4] ICP22 created new session: [3:7587241408255348052:2092]
2025-12-24T02:11:40.263773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241408255348052:2092] [node 4] ICS09 handshake done sender: [3:7587241408255348031:2116] self: [3:7587241408255348011:0] peer: [4:7587241407502514679:0] socket: 384 qp: -1
2025-12-24T02:11:40.263781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241408255348052:2092] [node 4] ICS10 traffic start
2025-12-24T02:11:40.263789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.266934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241407502514584:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.266959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.266969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241407502514584:2050] [node 3] ICP22 created new session: [4:7587241407502514685:2050]
2025-12-24T02:11:40.266977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241407502514685:2050] [node 3] ICS09 handshake done sender: [4:7587241407502514676:2121] self: [4:7587241407502514679:0] peer: [3:7587241408255348011:0] socket: 387 qp: -1
2025-12-24T02:11:40.266985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241407502514685:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.266996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.270784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407164375748:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.271631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407164375666:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.271655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.271664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407164375666:2048] [node 1] ICP22 created new session: [6:7587241407164375761:2048]
2025-12-24T02:11:40.271672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241407164375761:2048] [node 1] ICS09 handshake done sender: [6:7587241407164375752:2119] self: [6:7587241407164375755:0] peer: [1:7587241409786776782:0] socket: 383 qp: -1
2025-12-24T02:11:40.271680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241407164375761:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.271692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.271983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407164375667:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.271991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.271997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407164375667:2049] [node 2] ICP22 created new session: [6:7587241407164375763:2049]
2025-12-24T02:11:40.272002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241407164375763:2049] [node 2] ICS09 handshake done sender: [6:7587241407164375748:2117] self: [6:7587241407164375754:0] peer: [2:7587241407874973419:0] socket: 362 qp: -1
2025-12-24T02:11:40.272007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241407164375763:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.272014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.275224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241408255348006:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.275264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.275273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241408255348006:2096] [node 6] ICP22 created new session: [3:7587241408255348054:2096]
2025-12-24T02:11:40.275280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241408255348054:2096] [node 6] ICS09 handshake done sender: [3:7587241408255348028:2113] self: [3:7587241408255348010:0] peer: [6:7587241407164375758:0] socket: 344 qp: -1
2025-12-24T02:11:40.275288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241408255348054:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:40.275298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.279910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407164375668:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31942) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407164375753:2120] held: no
2025-12-24T02:11:40.280100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407164375765:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29968
2025-12-24T02:11:40.280188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407164375753:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.283608Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241409369675732:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28197) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241409369675753:2119] held: no
2025-12-24T02:11:40.283755Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241409369675664:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16059) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241409369675758:2120] held: no
2025-12-24T02:11:40.283856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241409369675753:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.284195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241407502514584:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(17140259ns) {HandleHandshakeStatus:329(17139530ns) {DropIncomingHandshake:442(178ns) DropOutgoingHandshake:457(1509ns) SwitchToState:179(209ns) ProcessPendingSessionEvents:495(17026116ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16990108ns max# 16900695ns}}}
2025-12-24T02:11:40.285032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407164375668:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.285055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.285063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407164375668:2050] [node 3] ICP22 created new session: [6:7587241407164375766:2050]
2025-12-24T02:11:40.285073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241407164375766:2050] [node 3] ICS09 handshake done sender: [6:7587241407164375753:2120] self: [6:7587241407164375758:0] peer: [3:7587241408255348010:0] socket: 385 qp: -1
2025-12-24T02:11:40.285081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241407164375766:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.285089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.285700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241409369675761:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49771
2025-12-24T02:11:40.285797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241409369675732:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.285810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.285822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241409369675732:2059] [node 4] ICP22 created new session: [5:7587241409369675762:2059]
2025-12-24T02:11:40.285829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241409369675762:2059] [node 4] ICS09 handshake done sender: [5:7587241409369675753:2119] self: [5:7587241409369675755:0] peer: [4:7587241407502514617:0] socket: 379 qp: -1
2025-12-24T02:11:40.285836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241409369675762:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:40.285842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.286252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241409369675758:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.286307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241409369675664:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.286320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.286325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241409369675664:2049] [node 2] ICP22 created new session: [5:7587241409369675764:2049]
2025-12-24T02:11:40.286331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241409369675764:2049] [node 2] ICS09 handshake done sender: [5:7587241409369675758:2120] self: [5:7587241409369675759:0] peer: [2:7587241407874973432:0] socket: 391 qp: -1
2025-12-24T02:11:40.286336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241409369675764:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.286341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.319309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:40.319424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241407874973454:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-12-24T02:11:40.319436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241407874973355:2048] [node 1] ICP30 unregister session Session# [2:7587241407874973454:2048] VirtualId# [2:7587241407874973443:0]
2025-12-24T02:11:40.319447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241407874973454:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.319493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.319523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.320069Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:40.320145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241408255348050:2048] [node 1] ICS01 socket: 382 reason# EndOfStream
2025-12-24T02:11:40.320152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241408255347951:2048] [node 1] ICP30 unregister session Session# [3:7587241408255348050:2048] VirtualId# [3:7587241408255348046:0]
2025-12-24T02:11:40.320159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241408255348050:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.320189Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.320210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.322687Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:40.322815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241407164375761:2048] [node 1] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:40.322822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407164375666:2048] [node 1] ICP30 unregister session Session# [6:7587241407164375761:2048] VirtualId# [6:7587241407164375755:0]
2025-12-24T02:11:40.322830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241407164375761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.322891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.322918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.323003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:40.323084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241409369675750:2048] [node 1] ICS01 socket: 340 reason# EndOfStream
2025-12-24T02:11:40.323091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241409369675663:2048] [node 1] ICP30 unregister session Session# [5:7587241409369675750:2048] VirtualId# [5:7587241409369675739:0]
2025-12-24T02:11:40.323098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241409369675750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.323148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.323170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.323211Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:40.323273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241407502514671:2048] [node 1] ICS01 socket: 348 reason# EndOfStream
2025-12-24T02:11:40.323278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241407502514582:2048] [node 1] ICP30 unregister session Session# [4:7587241407502514671:2048] VirtualId# [4:7587241407502514667:0]
2025-12-24T02:11:40.323283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241407502514671:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.323308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.323330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.334839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:40.334989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241408255348043:2049] [node 2] ICS01 socket: 342 reason# EndOfStream
2025-12-24T02:11:40.334999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241408255347952:2049] [node 2] ICP30 unregister session Session# [3:7587241408255348043:2049] VirtualId# [3:7587241408255348033:0]
2025-12-24T02:11:40.335010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241408255348043:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.335061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.335088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.345666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:40.345806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241407164375763:2049] [node 2] ICS01 socket: 362 reason# EndOfStream
2025-12-24T02:11:40.345816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407164375667:2049] [node 2] ICP30 unregister session Session# [6:7587241407164375763:2049] VirtualId# [6:7587241407164375754:0]
2025-12-24T02:11:40.345824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241407164375763:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.345889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.345915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.346711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:40.346800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241407502514677:2049] [node 2] ICS01 socket: 357 reason# EndOfStream
2025-12-24T02:11:40.346807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241407502514583:2049] [node 2] ICP30 unregister session Session# [4:7587241407502514677:2049] VirtualId# [4:7587241407502514669:0]
2025-12-24T02:11:40.346813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241407502514677:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.346858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.346882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.346976Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:40.347041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241409369675764:2049] [node 2] ICS01 socket: 391 reason# EndOfStream
2025-12-24T02:11:40.347046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241409369675664:2049] [node 2] ICP30 unregister session Session# [5:7587241409369675764:2049] VirtualId# [5:7587241409369675759:0]
2025-12-24T02:11:40.347053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241409369675764:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.347083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.347105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.380003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:40.380096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241409369675756:2050] [node 3] ICS01 socket: 350 reason# EndOfStream
2025-12-24T02:11:40.380102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241409369675665:2050] [node 3] ICP30 unregister session Session# [5:7587241409369675756:2050] VirtualId# [5:7587241409369675744:0]
2025-12-24T02:11:40.380107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241409369675756:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.380137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:40.380162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:40.379699Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:40.379836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241407164375766:2050] [node 3] ICS01 socket: 385 reason# EndOfStream
2025-12-24T02:11:40.379848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407164375668:2050] [node 3] ICP30 unregister session Session# [6:7587241407164375766:2050] VirtualId# [6:7587241407164375758:0]
2025-12-24T02:11:40.379858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241407164375766:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.379918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:40.379949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:40.426907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:40.427036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241407164375756:2059] [node 4] ICS01 socket: 334 reason# EndOfStream
2025-12-24T02:11:40.427049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407164375688:2059] [node 4] ICP30 unregister session Session# [6:7587241407164375756:2059] VirtualId# [6:7587241407164375731:0]
2025-12-24T02:11:40.427058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241407164375756:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.427119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:40.427149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:40.472119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407985497771:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.472129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407985497772:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.474899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405577947660:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.474906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405577947661:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.466540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241406758530331:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:40.474910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405577947662:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.466552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241406758530332:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:40.482730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241409103204413:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.482768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241409103204414:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.482775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241409103204415:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.467771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407107232100:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.467785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407107232101:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:40.502981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241406979787683:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.503010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241406979787684:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.506717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241406979787682:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.543157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241406758530392:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:40.547118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405577947681:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:40.545811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241406758530396:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.549766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241409103204471:2102] [node 6] ICP01 ready to work
2025-12-24T02:11:40.549820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204469:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.547234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241406979787705:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:40.547339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241406979787706:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:40.554375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407107232163:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:40.555015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204470:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.555113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204469:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.559174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241406758530407:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:40.562530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241405577947693:2067] [node 6] ICP01 ready to work
2025-12-24T02:11:40.562747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407107232160:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.566982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204480:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.567133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241406758530410:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44837
2025-12-24T02:11:40.570041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407985497835:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:40.570167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407985497836:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.571042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241407107232171:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.571782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947730:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
2025-12-24T02:11:40.571811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947729:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
2025-12-24T02:11:40.574706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787758:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.574837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241409103204471:2102] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24971691ns) {Configure:104(24971106ns) {ProcessConfigured:135(24525128ns) {StartInitialHandshake:157(24524797ns) {DropHandshakes:470(557ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1330ns) {SwitchToState:179(687ns)}}}}}
2025-12-24T02:11:40.582259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787759:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.582723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530398:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.582756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241406758530398:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.582835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241409103204488:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:40.583391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241409103204413:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.583442Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241409103204413:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28263728ns) {Configure:104(28263310ns) {ProcessConfigured:135(27900367ns) {IncomingHandshake:261(27881222ns) {DropIncomingHandshake:442(67ns) PrepareNewSessionHandshake:477(579ns) {SwitchToState:179(257ns)} GenerateSessionVirtualId:988(595ns) IssueIncomingHandshakeReply:193(27872464ns) {SwitchToState:179(412ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.584030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497846:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.584141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497847:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.584180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497848:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.584193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497849:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.584286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241405577947693:2067] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(21530672ns) {Configure:104(21530150ns) {ProcessConfigured:135(21045512ns) {StartInitialHandshake:157(21044829ns) {DropHandshakes:470(1153ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(282ns)} GenerateSessionVirtualId:988(429ns) PrepareNewSessionHandshake:477(1210ns) {SwitchToState:179(106ns)}}}}}
2025-12-24T02:11:40.584520Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241407107232100:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21660220ns) {Configure:104(21659032ns) {ProcessConfigured:135(21268548ns) {StartInitialHandshake:157(21268293ns) {DropHandshakes:470(631ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(467ns) PrepareNewSessionHandshake:477(572ns) {SwitchToState:179(315ns)}}}}}
2025-12-24T02:11:40.584627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232176:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.585689Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497846:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.585719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497847:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.585774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241407985497771:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.585983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407985497852:2117] [node 5] ICP01 ready to work
2025-12-24T02:11:40.586439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241407985497852:2117] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.586674Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530395:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.586688Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241406758530395:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.588220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232176:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.590213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947687:2070] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.590233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241405577947687:2070] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.593466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232179:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.593534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232180:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.593562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241407107232100:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.594860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241406758530411:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44837
2025-12-24T02:11:40.598472Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241406979787684:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49659184ns) {Configure:104(49658849ns) {ProcessConfigured:135(49411972ns) {StartInitialHandshake:157(49410680ns) {DropHandshakes:470(210ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(2291ns) {SwitchToState:179(1279ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.600456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947729:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.600530Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204493:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.600566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204494:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.601331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241406758530398:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.600262Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241407107232160:2106] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(36877094ns) {Configure:104(17270744ns) {ProcessConfigured:135(16839878ns) {StartInitialHandshake:157(16838251ns) {DropHandshakes:470(818ns) {DropIncomingHandshake:442(126ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(1025ns) {SwitchToState:179(216ns)}}}}}
2025-12-24T02:11:40.602827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232179:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.603407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409103204469:2103] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.603747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947741:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
2025-12-24T02:11:40.600668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787762:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.604038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241406979787762:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.604073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241406979787759:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.604119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241406979787705:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.604131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241406979787683:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.603918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204470:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.603934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241409103204413:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.603949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.603957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241409103204413:2048] [node 1] ICP22 created new session: [4:7587241409103204495:2048]
2025-12-24T02:11:40.603965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409103204495:2048] [node 1] ICS09 handshake done sender: [4:7587241409103204469:2103] self: [4:7587241409103204473:0] peer: [1:7587241406758530397:0] socket: 333 qp: -1
2025-12-24T02:11:40.603972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409103204495:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.603981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.602811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241406758530395:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.602966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241406758530396:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.602982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.602992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241406758530396:2108] [node 4] ICP22 created new session: [1:7587241406758530414:2108]
2025-12-24T02:11:40.603000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241406758530414:2108] [node 4] ICS09 handshake done sender: [1:7587241406758530398:2110] self: [1:7587241406758530397:0] peer: [4:7587241409103204473:0] socket: 325 qp: -1
2025-12-24T02:11:40.603008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241406758530414:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:40.603020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.605192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241406758530332:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.605206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.605214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241406758530332:2049] [node 3] ICP22 created new session: [1:7587241406758530417:2049]
2025-12-24T02:11:40.605220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241406758530417:2049] [node 3] ICS09 handshake done sender: [1:7587241406758530395:2109] self: [1:7587241406758530394:0] peer: [3:7587241407985497851:0] socket: 324 qp: -1
2025-12-24T02:11:40.605226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241406758530417:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:40.605233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.611027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405577947687:2070] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.611197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.611245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947741:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405577947660:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.612047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497838:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.612061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241407985497838:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.612402Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241407985497771:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241407985497846:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.612878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407985497838:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.613229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241407985497835:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.613243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.613249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407985497835:2107] [node 6] ICP22 created new session: [3:7587241407985497855:2107]
2025-12-24T02:11:40.613254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407985497855:2107] [node 6] ICS09 handshake done sender: [3:7587241407985497838:2110] self: [3:7587241407985497837:0] peer: [6:7587241406979787767:0] socket: 347 qp: -1
2025-12-24T02:11:40.613258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407985497855:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:40.613264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.611344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530411:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530410:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241406758530332:2049] [node 3] ICP18 handshake (actor [1:7587241406758530411:2116]) is too old
2025-12-24T02:11:40.611583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530405:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241406758530405:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.611754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241406979787765:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241406979787758:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.611968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241406979787684:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.612799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787768:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.613064Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241406979787758:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.613115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241406979787684:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.613130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.613137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241406979787684:2050] [node 3] ICP22 created new session: [6:7587241406979787769:2050]
2025-12-24T02:11:40.613144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241406979787769:2050] [node 3] ICS09 handshake done sender: [6:7587241406979787758:2113] self: [6:7587241406979787767:0] peer: [3:7587241407985497837:0] socket: 355 qp: -1
2025-12-24T02:11:40.613152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241406979787769:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.613160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.614825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204480:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.615168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497848:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.615152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232184:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.616272Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204489:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.616283Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241409103204489:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.617047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530409:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.617061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241406758530409:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.615706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947730:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.618765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241406758530400:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.618789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241406758530400:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.618886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232173:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.618902Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407107232173:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.618954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497849:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.619081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241407985497772:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.619195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232184:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.619260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232168:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.619264Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407107232168:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.618950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405577947661:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.619133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241405577947662:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.619152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.619162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405577947662:2050] [node 3] ICP22 created new session: [5:7587241405577947746:2050]
2025-12-24T02:11:40.619169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405577947746:2050] [node 3] ICS09 handshake done sender: [5:7587241405577947687:2070] self: [5:7587241405577947686:0] peer: [3:7587241407985497853:0] socket: 332 qp: -1
2025-12-24T02:11:40.619177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405577947746:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.619225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.622913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204493:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.622962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241409103204415:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.622870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232185:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.623027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497860:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.623277Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497861:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.623375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407985497840:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.623382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241407985497840:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.623405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407985497846:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.623416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407985497847:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.623765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241407985497852:2117] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.623779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.623788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407985497852:2117] [node 5] ICP22 created new session: [3:7587241407985497862:2117]
2025-12-24T02:11:40.623795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407985497862:2117] [node 5] ICS09 handshake done sender: [3:7587241407985497847:2114] self: [3:7587241407985497853:0] peer: [5:7587241405577947686:0] socket: 362 qp: -1
2025-12-24T02:11:40.623802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407985497862:2117] [node 5] ICS10 traffic start
2025-12-24T02:11:40.623811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.624057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241407985497771:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.624065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.624073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407985497771:2048] [node 1] ICP22 created new session: [3:7587241407985497864:2048]
2025-12-24T02:11:40.624077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407985497864:2048] [node 1] ICS09 handshake done sender: [3:7587241407985497846:2113] self: [3:7587241407985497851:0] peer: [1:7587241406758530394:0] socket: 328 qp: -1
2025-12-24T02:11:40.624083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407985497864:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.624088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.624701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407985497840:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.624794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241407985497836:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.624820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.624828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407985497836:2108] [node 4] ICP22 created new session: [3:7587241407985497867:2108]
2025-12-24T02:11:40.624834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407985497867:2108] [node 4] ICS09 handshake done sender: [3:7587241407985497840:2111] self: [3:7587241407985497839:0] peer: [4:7587241409103204499:0] socket: 348 qp: -1
2025-12-24T02:11:40.624839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407985497867:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:40.624850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.623285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241406758530405:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.626835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947748:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
2025-12-24T02:11:40.631085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405577947741:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.631549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405577947660:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.631569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.631577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405577947660:2048] [node 1] ICP22 created new session: [5:7587241405577947749:2048]
2025-12-24T02:11:40.631584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405577947749:2048] [node 1] ICS09 handshake done sender: [5:7587241405577947741:2117] self: [5:7587241405577947743:0] peer: [1:7587241406758530404:0] socket: 371 qp: -1
2025-12-24T02:11:40.631591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405577947749:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.631599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.631123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241406758530392:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.631155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.631168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241406758530392:2107] [node 5] ICP22 created new session: [1:7587241406758530423:2107]
2025-12-24T02:11:40.631175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241406758530423:2107] [node 5] ICS09 handshake done sender: [1:7587241406758530405:2112] self: [1:7587241406758530404:0] peer: [5:7587241405577947743:0] socket: 370 qp: -1
2025-12-24T02:11:40.631184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241406758530423:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:40.631195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.631329Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241409103204415:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241409103204493:2117] held: no
2025-12-24T02:11:40.635134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232166:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.635155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407107232166:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.639096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232180:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.638868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241406979787682:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.638923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241406979787682:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27104528ns) {IncomingHandshake:261(27104095ns) {DropIncomingHandshake:442(193ns) PrepareNewSessionHandshake:477(399ns) {SwitchToState:179(116ns)} GenerateSessionVirtualId:988(263ns) IssueIncomingHandshakeReply:193(27098865ns) {SwitchToState:179(1720ns)}}}
2025-12-24T02:11:40.639223Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241406979787684:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(25999572ns) {HandleHandshakeStatus:329(25998688ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(1081ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(19832713ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19805389ns max# 19717362ns}}}
2025-12-24T02:11:40.641774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241406979787682:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9647) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241406979787765:2118] held: no
2025-12-24T02:11:40.639222Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241406758530422:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44837
2025-12-24T02:11:40.639869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241406758530392:2107] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.640533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241406758530409:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.640612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241406758530407:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.640628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.640636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241406758530407:2113] [node 6] ICP22 created new session: [1:7587241406758530426:2113]
2025-12-24T02:11:40.640644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241406758530426:2113] [node 6] ICS09 handshake done sender: [1:7587241406758530409:2114] self: [1:7587241406758530408:0] peer: [6:7587241406979787766:0] socket: 372 qp: -1
2025-12-24T02:11:40.640651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241406758530426:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:40.640661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.648581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407985497869:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42603
2025-12-24T02:11:40.648723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241406758530400:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.648846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241406758530331:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.648868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.648877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241406758530331:2048] [node 2] ICP22 created new session: [1:7587241406758530429:2048]
2025-12-24T02:11:40.648885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241406758530429:2048] [node 2] ICS09 handshake done sender: [1:7587241406758530400:2111] self: [1:7587241406758530399:0] peer: [2:7587241407107232181:0] socket: 326 qp: -1
2025-12-24T02:11:40.648893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241406758530429:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:40.648904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.651141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407107232173:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.651222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407107232168:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.651534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407985497849:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.651622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241407985497772:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.651639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.651648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407985497772:2049] [node 2] ICP22 created new session: [3:7587241407985497870:2049]
2025-12-24T02:11:40.651655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407985497870:2049] [node 2] ICS09 handshake done sender: [3:7587241407985497849:2116] self: [3:7587241407985497859:0] peer: [2:7587241407107232167:0] socket: 364 qp: -1
2025-12-24T02:11:40.651663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407985497870:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.651672Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.655017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407107232171:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.655039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.655052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407107232171:2111] [node 6] ICP22 created new session: [2:7587241407107232191:2111]
2025-12-24T02:11:40.655060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407107232191:2111] [node 6] ICS09 handshake done sender: [2:7587241407107232173:2112] self: [2:7587241407107232172:0] peer: [6:7587241406979787764:0] socket: 351 qp: -1
2025-12-24T02:11:40.655068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407107232191:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:40.655079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.655538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407107232101:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.655559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.655567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407107232101:2049] [node 3] ICP22 created new session: [2:7587241407107232193:2049]
2025-12-24T02:11:40.655573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407107232193:2049] [node 3] ICS09 handshake done sender: [2:7587241407107232168:2110] self: [2:7587241407107232167:0] peer: [3:7587241407985497859:0] socket: 350 qp: -1
2025-12-24T02:11:40.655580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407107232193:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:40.655603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.656260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241405577947662:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(44879464ns) {HandleHandshakeStatus:329(44878540ns) {DropIncomingHandshake:442(176ns) DropOutgoingHandshake:457(316ns) SwitchToState:179(82ns) ProcessPendingSessionEvents:495(17315493ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17292453ns max# 17211270ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.658949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787772:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.659004Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409103204489:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.659004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787774:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.659058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232185:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.659160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241409103204471:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.659182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.659190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241409103204471:2102] [node 6] ICP22 created new session: [4:7587241409103204502:2102]
2025-12-24T02:11:40.659198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409103204502:2102] [node 6] ICS09 handshake done sender: [4:7587241409103204489:2115] self: [4:7587241409103204472:0] peer: [6:7587241406979787763:0] socket: 360 qp: -1
2025-12-24T02:11:40.659205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409103204502:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:40.659216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.659519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241406979787759:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.662757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787773:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.662835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407107232166:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.666771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241407107232190:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2644
2025-12-24T02:11:40.667072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241407107232171:2111] [node 6] ICP18 handshake (actor [2:7587241407107232185:2119]) is too old
2025-12-24T02:11:40.667202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407107232163:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.667222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.667230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407107232163:2108] [node 5] ICP22 created new session: [2:7587241407107232198:2108]
2025-12-24T02:11:40.667237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407107232198:2108] [node 5] ICS09 handshake done sender: [2:7587241407107232166:2109] self: [2:7587241407107232164:0] peer: [5:7587241405577947745:0] socket: 345 qp: -1
2025-12-24T02:11:40.667246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407107232198:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:40.667256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.671501Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241407985497772:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(19764721ns) {HandleHandshakeStatus:329(19763958ns) {DropIncomingHandshake:442(289ns) DropOutgoingHandshake:457(2752ns) SwitchToState:179(117ns) ProcessPendingSessionEvents:495(19671947ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19647459ns max# 19570074ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.674763Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947753:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
2025-12-24T02:11:40.674938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407107232176:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.674998Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405577947661:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21514) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241405577947730:2107] held: no
2025-12-24T02:11:40.678354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241406979787772:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.678376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241406979787683:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.678394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.678407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241406979787683:2049] [node 2] ICP22 created new session: [6:7587241406979787776:2049]
2025-12-24T02:11:40.678413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241406979787776:2049] [node 2] ICS09 handshake done sender: [6:7587241406979787759:2114] self: [6:7587241406979787764:0] peer: [2:7587241407107232172:0] socket: 356 qp: -1
2025-12-24T02:11:40.678423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241406979787776:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.678434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.682816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241407107232100:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.682857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.682869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407107232100:2048] [node 1] ICP22 created new session: [2:7587241407107232200:2048]
2025-12-24T02:11:40.682877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407107232200:2048] [node 1] ICS09 handshake done sender: [2:7587241407107232176:2113] self: [2:7587241407107232181:0] peer: [1:7587241406758530399:0] socket: 329 qp: -1
2025-12-24T02:11:40.682890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407107232200:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.682900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.682951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405577947730:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.683683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241406979787683:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.683825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241406979787765:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.683224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947752:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.687583Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241406758530431:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44837
2025-12-24T02:11:40.690971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204504:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.690854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405577947661:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.690884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.690895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405577947661:2049] [node 2] ICP22 created new session: [5:7587241405577947754:2049]
2025-12-24T02:11:40.690907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405577947754:2049] [node 2] ICS09 handshake done sender: [5:7587241405577947730:2107] self: [5:7587241405577947745:0] peer: [2:7587241407107232164:0] socket: 352 qp: -1
2025-12-24T02:11:40.690910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947752:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.690916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405577947754:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.690927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.690970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241405577947681:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.691202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787775:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.691272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241406979787682:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.691285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.691293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241406979787682:2048] [node 1] ICP22 created new session: [6:7587241406979787778:2048]
2025-12-24T02:11:40.691300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241406979787778:2048] [node 1] ICS09 handshake done sender: [6:7587241406979787765:2118] self: [6:7587241406979787766:0] peer: [1:7587241406758530408:0] socket: 374 qp: -1
2025-12-24T02:11:40.691308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241406979787778:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.691315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.691222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204492:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.691232Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241409103204492:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.696393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241406979787706:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.696458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241406979787705:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241406979787762:2117] held: no
2025-12-24T02:11:40.696683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241406979787762:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.696876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241406979787705:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.696896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.696908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241406979787705:2060] [node 4] ICP22 created new session: [6:7587241406979787781:2060]
2025-12-24T02:11:40.696915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241406979787781:2060] [node 4] ICS09 handshake done sender: [6:7587241406979787762:2117] self: [6:7587241406979787763:0] peer: [4:7587241409103204472:0] socket: 361 qp: -1
2025-12-24T02:11:40.696925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241406979787781:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:40.696935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.695338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409103204493:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.698834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241405577947739:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.698850Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241405577947739:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.699543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405577947739:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.699637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241405577947693:2067] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.699659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.699666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405577947693:2067] [node 6] ICP22 created new session: [5:7587241405577947758:2067]
2025-12-24T02:11:40.699673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405577947758:2067] [node 6] ICS09 handshake done sender: [5:7587241405577947739:2116] self: [5:7587241405577947725:0] peer: [6:7587241406979787780:0] socket: 386 qp: -1
2025-12-24T02:11:40.699680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405577947758:2067] [node 6] ICS10 traffic start
2025-12-24T02:11:40.699688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.704613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241406979787706:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1606) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241406979787772:2120] held: no
2025-12-24T02:11:40.704783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241406979787783:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59480
2025-12-24T02:11:40.704983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241406979787772:2120] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.705042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241406979787706:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.705054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.705061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241406979787706:2064] [node 5] ICP22 created new session: [6:7587241406979787784:2064]
2025-12-24T02:11:40.705068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241406979787784:2064] [node 5] ICS09 handshake done sender: [6:7587241406979787772:2120] self: [6:7587241406979787780:0] peer: [5:7587241405577947725:0] socket: 375 qp: -1
2025-12-24T02:11:40.705075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241406979787784:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:40.705082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.703013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204505:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.703154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241409103204415:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.703170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.703178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241409103204415:2050] [node 3] ICP22 created new session: [4:7587241409103204506:2050]
2025-12-24T02:11:40.703186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409103204506:2050] [node 3] ICS09 handshake done sender: [4:7587241409103204493:2117] self: [4:7587241409103204499:0] peer: [3:7587241407985497839:0] socket: 358 qp: -1
2025-12-24T02:11:40.703193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409103204506:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.703217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.707137Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241407107232100:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24231110ns) {HandleHandshakeStatus:329(24229779ns) {DropIncomingHandshake:442(578ns) DropOutgoingHandshake:457(7573ns) SwitchToState:179(292ns) ProcessPendingSessionEvents:495(16085471ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16067156ns max# 15986977ns}}}
2025-12-24T02:11:40.707250Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409103204505:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.707307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241409103204414:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.711163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241406979787682:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(19865808ns) {HandleHandshakeStatus:329(19864933ns) {DropIncomingHandshake:442(202ns) DropOutgoingHandshake:457(71ns) SwitchToState:179(88ns) ProcessPendingSessionEvents:495(186ns)}}
2025-12-24T02:11:40.718793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241407107232182:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.718821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241407107232182:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.719405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241407107232182:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.719535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241407107232160:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.719561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.719567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241407107232160:2106] [node 4] ICP22 created new session: [2:7587241407107232203:2106]
2025-12-24T02:11:40.719573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241407107232203:2106] [node 4] ICS09 handshake done sender: [2:7587241407107232182:2117] self: [2:7587241407107232177:0] peer: [4:7587241409103204508:0] socket: 395 qp: -1
2025-12-24T02:11:40.719578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241407107232203:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:40.719592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.720320Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405577947681:2064] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27733) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241405577947752:2119] held: no
2025-12-24T02:11:40.718844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409103204492:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.719001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241409103204488:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.719025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.719036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241409103204488:2114] [node 5] ICP22 created new session: [4:7587241409103204510:2114]
2025-12-24T02:11:40.719044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409103204510:2114] [node 5] ICS09 handshake done sender: [4:7587241409103204492:2116] self: [4:7587241409103204490:0] peer: [5:7587241405577947756:0] socket: 343 qp: -1
2025-12-24T02:11:40.719051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409103204510:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:40.719066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.722971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241405577947760:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17926
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.723262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409103204512:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21868
2025-12-24T02:11:40.723460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409103204505:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:40.723559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241409103204414:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.723576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:40.723581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241409103204414:2049] [node 2] ICP22 created new session: [4:7587241409103204513:2049]
2025-12-24T02:11:40.723588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409103204513:2049] [node 2] ICS09 handshake done sender: [4:7587241409103204505:2120] self: [4:7587241409103204508:0] peer: [2:7587241407107232177:0] socket: 383 qp: -1
2025-12-24T02:11:40.723594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409103204513:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:40.723604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:40.726942Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241406758530331:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.726963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241405577947752:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.727074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241405577947681:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.727086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.727095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241405577947681:2064] [node 4] ICP22 created new session: [5:7587241405577947761:2064]
2025-12-24T02:11:40.727102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241405577947761:2064] [node 4] ICS09 handshake done sender: [5:7587241405577947752:2119] self: [5:7587241405577947756:0] peer: [4:7587241409103204490:0] socket: 389 qp: -1
2025-12-24T02:11:40.727110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241405577947761:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:40.727121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:40.742784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:40.742903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241407985497864:2048] [node 1] ICS01 socket: 328 reason# EndOfStream
2025-12-24T02:11:40.742918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241407985497771:2048] [node 1] ICP30 unregister session Session# [3:7587241407985497864:2048] VirtualId# [3:7587241407985497851:0]
2025-12-24T02:11:40.742927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241407985497864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.742996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.743026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.742883Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:40.742950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241407107232200:2048] [node 1] ICS07 socket disconnect 329 reason# EndOfStream
2025-12-24T02:11:40.742957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241407107232200:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.743000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.743020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241407107232200:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:40.743506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241407107232100:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9647) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:40.743517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241407107232200:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:40.743522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241407107232100:2048] [node 1] ICP30 unregister session Session# [2:7587241407107232200:2048] VirtualId# [2:7587241407107232181:0]
2025-12-24T02:11:40.743576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.743582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241407107232100:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.743586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9647) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.749410Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:40.749494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241409103204495:2048] [node 1] ICS01 socket: 333 reason# EndOfStream
2025-12-24T02:11:40.749502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241409103204413:2048] [node 1] ICP30 unregister session Session# [4:7587241409103204495:2048] VirtualId# [4:7587241409103204473:0]
2025-12-24T02:11:40.749509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241409103204495:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.749556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.749582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.750756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:40.750830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405577947749:2048] [node 1] ICS01 socket: 371 reason# EndOfStream
2025-12-24T02:11:40.750838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405577947660:2048] [node 1] ICP30 unregister session Session# [5:7587241405577947749:2048] VirtualId# [5:7587241405577947743:0]
2025-12-24T02:11:40.750845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405577947749:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.750891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.750916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.753021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:40.753099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241406979787778:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-12-24T02:11:40.753105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241406979787682:2048] [node 1] ICP30 unregister session Session# [6:7587241406979787778:2048] VirtualId# [6:7587241406979787766:0]
2025-12-24T02:11:40.753111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241406979787778:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.753146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:40.753167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:40.766791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:40.766950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241407985497870:2049] [node 2] ICS07 socket disconnect 364 reason# EndOfStream
2025-12-24T02:11:40.766959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241407985497870:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.767023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.767052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241407985497870:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:40.770806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:40.770898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241409103204513:2049] [node 2] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:40.770907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241409103204414:2049] [node 2] ICP30 unregister session Session# [4:7587241409103204513:2049] VirtualId# [4:7587241409103204508:0]
2025-12-24T02:11:40.770915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241409103204513:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.770971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.771002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.771042Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:40.771105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241406979787776:2049] [node 2] ICS01 socket: 356 reason# EndOfStream
2025-12-24T02:11:40.771110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241406979787683:2049] [node 2] ICP30 unregister session Session# [6:7587241406979787776:2049] VirtualId# [6:7587241406979787764:0]
2025-12-24T02:11:40.771114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241406979787776:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.771184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.771204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.774749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:40.774851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405577947754:2049] [node 2] ICS01 socket: 352 reason# EndOfStream
2025-12-24T02:11:40.774860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405577947661:2049] [node 2] ICP30 unregister session Session# [5:7587241405577947754:2049] VirtualId# [5:7587241405577947745:0]
2025-12-24T02:11:40.774868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405577947754:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.774917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:40.774944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:40.791443Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:40.791512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241405577947746:2050] [node 3] ICS07 socket disconnect 332 reason# EndOfStream
2025-12-24T02:11:40.791517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241405577947746:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.791545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:40.791564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241405577947746:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:40.791978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241405577947662:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:40.791991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241405577947746:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:40.791996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241405577947662:2050] [node 3] ICP30 unregister session Session# [5:7587241405577947746:2050] VirtualId# [5:7587241405577947686:0]
2025-12-24T02:11:40.792051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:40.792057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241405577947662:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.792060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.791023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:40.791133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241406979787769:2050] [node 3] ICS07 socket disconnect 355 reason# EndOfStream
2025-12-24T02:11:40.791140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241406979787769:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.791182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:40.791213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241406979787769:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:40.792288Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241406979787684:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:40.792294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241406979787769:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:40.792299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241406979787684:2050] [node 3] ICP30 unregister session Session# [6:7587241406979787769:2050] VirtualId# [6:7587241406979787767:0]
2025-12-24T02:11:40.792346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:40.792350Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241406979787684:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.792352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27558) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:40.802761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241405577947662:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:40.801769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241406979787684:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:40.822378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:40.822493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241406979787781:2060] [node 4] ICS01 socket: 361 reason# EndOfStream
2025-12-24T02:11:40.822504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241406979787705:2060] [node 4] ICP30 unregister session Session# [6:7587241406979787781:2060] VirtualId# [6:7587241406979787763:0]
2025-12-24T02:11:40.822513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241406979787781:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:40.822578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:40.822606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:40.875682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241407215202937:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.875693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241407215202938:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.874833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241406888536278:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.874848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241406888536279:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:40.865947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407937762440:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.865958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407937762441:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.875501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407701819182:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.875510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407701819183:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.875698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241407215202939:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.906053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405659041014:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:40.911517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405659041016:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.910714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409519876484:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:40.914718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405659041015:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:40.921609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405659041072:2102] [node 6] ICP01 ready to work
2025-12-24T02:11:40.875516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407701819184:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:40.921035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407701819196:2060] [node 5] ICP01 ready to work
2025-12-24T02:11:40.929071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409519876485:2049] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.939218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241405659041086:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:40.940003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409519876550:2103] [node 5] ICP01 ready to work
2025-12-24T02:11:40.939265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241407215203001:2094] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.940014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241407701819250:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:40.940660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407937762508:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:40.941098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407937762506:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-12-24T02:11:40.941118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407937762507:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-12-24T02:11:40.941207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241407215203005:2101] [node 6] ICP01 ready to work
2025-12-24T02:11:40.941493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241407937762440:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18651970ns) {Configure:104(18651349ns) {ProcessConfigured:135(18281236ns) {StartInitialHandshake:157(18281053ns) {DropHandshakes:470(164ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(710ns) {SwitchToState:179(23ns)}}}}}
2025-12-24T02:11:40.941371Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241405659041014:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18591845ns) {Configure:104(18591094ns) {ProcessConfigured:135(18275801ns) {StartInitialHandshake:157(18275306ns) {DropHandshakes:470(774ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(427ns) PrepareNewSessionHandshake:477(634ns) {SwitchToState:179(24ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.940784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409519876554:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:40.944459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241406888536345:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:40.942815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215202979:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.946957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241409519876557:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:40.948535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407701819249:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.949486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407937762505:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:40.949004Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203006:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:40.949093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203008:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:40.949418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203007:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:40.950597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215202979:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.950766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241406888536341:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:40.950852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241406888536342:2105] [node 5] ICP01 ready to work
2025-12-24T02:11:40.954933Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241407215202939:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32906494ns) {Configure:104(32905582ns) {ProcessConfigured:135(32534509ns) {StartInitialHandshake:157(32534125ns) {DropHandshakes:470(458ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(1408ns) {SwitchToState:179(372ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.955325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241405659041015:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32875869ns) {Configure:104(32875125ns) {ProcessConfigured:135(32608475ns) {StartInitialHandshake:157(32608336ns) {DropHandshakes:470(335ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1130ns) {SwitchToState:179(617ns)}}}}}
2025-12-24T02:11:40.955483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407937762518:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.958858Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241407215202937:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19503331ns) {Configure:104(19502926ns) {ProcessConfigured:135(18983200ns) {StartInitialHandshake:157(18982734ns) {DropHandshakes:470(472ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(1335ns) {SwitchToState:179(684ns)}}}}}
2025-12-24T02:11:40.959037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409519876562:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-12-24T02:11:40.959231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241406888536278:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20422451ns) {Configure:104(20421730ns) {ProcessConfigured:135(16717626ns) {StartInitialHandshake:157(16716942ns) {DropHandshakes:470(304ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(25ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(916ns) {SwitchToState:179(59ns)}}}}}
2025-12-24T02:11:40.959973Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241407701819250:2089] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19879046ns) {Configure:104(19878294ns) {ProcessConfigured:135(19470022ns) {StartInitialHandshake:157(19469895ns) {DropHandshakes:470(273ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(914ns) {SwitchToState:179(410ns)}}}}}
2025-12-24T02:11:40.960975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241407701819182:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21929834ns) {Configure:104(21927423ns) {ProcessConfigured:135(21482690ns) {StartInitialHandshake:157(21481870ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(464ns) PrepareNewSessionHandshake:477(1048ns) {SwitchToState:179(745ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.963380Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203020:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:40.963203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241407701819184:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40115456ns) {Configure:104(40114828ns) {ProcessConfigured:135(39848186ns) {StartInitialHandshake:157(39847940ns) {DropHandshakes:470(238ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(985ns) {SwitchToState:179(195ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.963036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241409519876484:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24203147ns) {Configure:104(24201622ns) {ProcessConfigured:135(23693639ns) {StartInitialHandshake:157(23692935ns) {DropHandshakes:470(456ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(949ns) {SwitchToState:179(583ns)}}}}}
2025-12-24T02:11:40.964721Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405659041093:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:40.964778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405659041094:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:40.965058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405659041092:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:40.964569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241407937762516:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:40.970134Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241406888536345:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25575283ns) {Configure:104(25574816ns) {ProcessConfigured:135(25286292ns) {StartInitialHandshake:157(25285770ns) {DropHandshakes:470(344ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(588ns)}}}}}
2025-12-24T02:11:40.974578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041094:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.976271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215203008:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.976317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241407215202939:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.976474Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762510:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.976487Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241407937762510:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.976710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041092:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.976754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405659041014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.976863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241406888536357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
2025-12-24T02:11:40.977168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241406888536356:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
2025-12-24T02:11:40.977224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536357:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.977003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876562:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.980369Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241406888536341:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(29396692ns) {Configure:104(29396243ns) {ProcessConfigured:135(29080749ns) {StartInitialHandshake:157(29080162ns) {DropHandshakes:470(450ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1162ns) {SwitchToState:179(403ns)}}}}}
2025-12-24T02:11:40.980270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407701819262:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
2025-12-24T02:11:40.981677Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215203007:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.981726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241407215202937:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.985599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876552:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.985616Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409519876552:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.985561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536356:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.985751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762506:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.986059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407701819249:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.986115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407701819250:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.986288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409519876552:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.986501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409519876550:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.986523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.986530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409519876550:2103] [node 5] ICP22 created new session: [1:7587241409519876567:2103]
2025-12-24T02:11:40.986539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409519876567:2103] [node 5] ICS09 handshake done sender: [1:7587241409519876552:2110] self: [1:7587241409519876551:0] peer: [5:7587241407215203022:0] socket: 340 qp: -1
2025-12-24T02:11:40.986546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409519876567:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:40.986562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876559:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.986569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.986576Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409519876559:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.986753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241406888536358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
2025-12-24T02:11:40.986940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762507:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241407937762440:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.987009Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407937762510:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.987128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215203020:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241407215202938:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.987131Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536358:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987145Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041093:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536350:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041074:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987596Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241406888536350:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.987578Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241405659041074:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.987620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215203006:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987712Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241406888536279:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35508838ns) {Configure:104(35507751ns) {ProcessConfigured:135(34727443ns) {StartInitialHandshake:157(34726898ns) {DropHandshakes:470(421ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1637ns) {SwitchToState:179(172ns)}}}}}
2025-12-24T02:11:40.987846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876548:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409519876548:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.987576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407701819262:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.987622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407701819183:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.987332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241407937762508:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.987347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.987352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407937762508:2109] [node 5] ICP22 created new session: [3:7587241407937762523:2109]
2025-12-24T02:11:40.987356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407937762523:2109] [node 5] ICS09 handshake done sender: [3:7587241407937762510:2112] self: [3:7587241407937762509:0] peer: [5:7587241407215203021:0] socket: 341 qp: -1
2025-12-24T02:11:40.987362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407937762523:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:40.987382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.988355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762518:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.988423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241407937762516:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23715375ns) {Configure:104(23714239ns) {ProcessConfigured:135(23306152ns) {StartInitialHandshake:157(23305067ns) {DropHandshakes:470(950ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(211ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(1227ns) {SwitchToState:179(841ns)}}}}}
2025-12-24T02:11:40.988549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409519876559:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:40.988685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409519876557:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.988699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:40.988705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409519876557:2111] [node 4] ICP22 created new session: [1:7587241409519876571:2111]
2025-12-24T02:11:40.988712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409519876571:2111] [node 4] ICS09 handshake done sender: [1:7587241409519876559:2112] self: [1:7587241409519876558:0] peer: [4:7587241405659041096:0] socket: 351 qp: -1
2025-12-24T02:11:40.988719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409519876571:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:40.988726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.988821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241407701819265:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.988937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203026:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:40.989100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241407215203008:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.989216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241407215202939:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.989235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.989241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241407215202939:2050] [node 3] ICP22 created new session: [5:7587241407215203027:2050]
2025-12-24T02:11:40.989247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241407215203027:2050] [node 3] ICS09 handshake done sender: [5:7587241407215203008:2113] self: [5:7587241407215203021:0] peer: [3:7587241407937762509:0] socket: 344 qp: -1
2025-12-24T02:11:40.989254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241407215203027:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:40.989260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.989108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241407937762508:2109] [node 5] ICP18 handshake (actor [3:7587241407937762518:2115]) is too old
2025-12-24T02:11:40.990032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041088:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.990052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241405659041088:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.989883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409519876569:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-12-24T02:11:40.990092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536353:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.990107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241406888536353:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.990464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405659041074:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.990174Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241407215203025:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.991245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241405659041072:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.991264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.991271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405659041072:2102] [node 6] ICP22 created new session: [4:7587241405659041099:2102]
2025-12-24T02:11:40.991277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405659041099:2102] [node 6] ICS09 handshake done sender: [4:7587241405659041074:2104] self: [4:7587241405659041073:0] peer: [6:7587241407701819263:0] socket: 333 qp: -1
2025-12-24T02:11:40.991282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405659041099:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:40.991299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:40.991686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405659041100:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:40.991465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241407215203007:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.991770Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407937762528:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-12-24T02:11:40.993052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876569:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.993153Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409519876548:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:40.993354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409519876485:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.993377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:40.993383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409519876485:2049] [node 3] ICP22 created new session: [1:7587241409519876575:2049]
2025-12-24T02:11:40.993389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409519876575:2049] [node 3] ICS09 handshake done sender: [1:7587241409519876548:2109] self: [1:7587241409519876547:0] peer: [3:7587241407937762522:0] socket: 337 qp: -1
2025-12-24T02:11:40.993395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409519876575:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:40.993424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:40.993462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405659041092:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.993556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405659041014:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.993570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.993577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405659041014:2048] [node 1] ICP22 created new session: [4:7587241405659041103:2048]
2025-12-24T02:11:40.993584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405659041103:2048] [node 1] ICS09 handshake done sender: [4:7587241405659041092:2115] self: [4:7587241405659041096:0] peer: [1:7587241409519876558:0] socket: 356 qp: -1
2025-12-24T02:11:40.993589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405659041103:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.993595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.994438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241407701819265:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.994483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407701819182:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.994485Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241407937762440:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241407937762507:2111] held: no
2025-12-24T02:11:40.994799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241407215203001:2094] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:40.994916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407937762507:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:40.995009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241407937762440:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.995021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.995026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407937762440:2048] [node 1] ICP22 created new session: [3:7587241407937762529:2048]
2025-12-24T02:11:40.995033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241407937762529:2048] [node 1] ICS09 handshake done sender: [3:7587241407937762507:2111] self: [3:7587241407937762522:0] peer: [1:7587241409519876547:0] socket: 339 qp: -1
2025-12-24T02:11:40.995037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241407937762529:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.995043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.995100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407701819250:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407701819249:2108] held: no
2025-12-24T02:11:40.995256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241407215202937:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:40.995269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:40.995276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241407215202937:2048] [node 1] ICP22 created new session: [5:7587241407215203029:2048]
2025-12-24T02:11:40.995280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241407215203029:2048] [node 1] ICS09 handshake done sender: [5:7587241407215203007:2112] self: [5:7587241407215203022:0] peer: [1:7587241409519876551:0] socket: 343 qp: -1
2025-12-24T02:11:40.995283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241407215203029:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:40.995289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:40.995506Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241406888536350:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:40.995613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241406888536342:2105] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.995637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.995651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241406888536342:2105] [node 5] ICP22 created new session: [2:7587241406888536364:2105]
2025-12-24T02:11:40.995661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241406888536364:2105] [node 5] ICS09 handshake done sender: [2:7587241406888536350:2111] self: [2:7587241406888536349:0] peer: [5:7587241407215203023:0] socket: 360 qp: -1
2025-12-24T02:11:40.995666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241406888536364:2105] [node 5] ICS10 traffic start
2025-12-24T02:11:40.995682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.995630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876561:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:40.995642Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409519876561:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:40.996460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409519876561:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.996562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409519876554:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.996573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.996579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409519876554:2105] [node 6] ICP22 created new session: [1:7587241409519876580:2105]
2025-12-24T02:11:40.996585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241409519876580:2105] [node 6] ICS09 handshake done sender: [1:7587241409519876561:2113] self: [1:7587241409519876555:0] peer: [6:7587241407701819266:0] socket: 373 qp: -1
2025-12-24T02:11:40.996589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241409519876580:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:40.996595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.997959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241406888536368:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
2025-12-24T02:11:40.997977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241406888536369:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.998196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241409519876578:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-12-24T02:11:40.998294Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241409519876557:2111] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:40.998355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405659041088:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:40.998627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241405659041086:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.998647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:40.998652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405659041086:2109] [node 5] ICP22 created new session: [4:7587241405659041107:2109]
2025-12-24T02:11:40.998657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241405659041107:2109] [node 5] ICS09 handshake done sender: [4:7587241405659041088:2112] self: [4:7587241405659041087:0] peer: [5:7587241407215203024:0] socket: 338 qp: -1
2025-12-24T02:11:40.998661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241405659041107:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:40.998666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:40.999544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241406888536353:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:40.999681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241406888536345:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:40.999697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:40.999703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241406888536345:2107] [node 6] ICP22 created new session: [2:7587241406888536370:2107]
2025-12-24T02:11:40.999709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241406888536370:2107] [node 6] ICS09 handshake done sender: [2:7587241406888536353:2112] self: [2:7587241406888536346:0] peer: [6:7587241407701819264:0] socket: 368 qp: -1
2025-12-24T02:11:40.999713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241406888536370:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:40.999720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.001810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241407701819183:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241407701819262:2117] held: no
2025-12-24T02:11:41.001862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241405659041109:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:41.002276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241409519876563:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.002291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241409519876563:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.000282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536368:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.000310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241406888536278:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.007901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241409519876563:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.009043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241407937762531:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.009351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241409519876484:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.009382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.009389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241409519876484:2048] [node 2] ICP22 created new session: [1:7587241413814843879:2048]
2025-12-24T02:11:41.009396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413814843879:2048] [node 2] ICS09 handshake done sender: [1:7587241409519876563:2115] self: [1:7587241409519876546:0] peer: [2:7587241406888536372:0] socket: 382 qp: -1
2025-12-24T02:11:41.009403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413814843879:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:41.009411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.009163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241407215203001:2094] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31910) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241407215203006:2111] held: no
2025-12-24T02:11:41.009496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241407215202938:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63851) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241407215203020:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.011220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241411510170329:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:41.011413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241407215203006:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.011831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762531:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.011886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241407937762441:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.011608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241407215203001:2094] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.011623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.011632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241407215203001:2094] [node 4] ICP22 created new session: [5:7587241411510170330:2094]
2025-12-24T02:11:41.011638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241411510170330:2094] [node 4] ICS09 handshake done sender: [5:7587241407215203006:2111] self: [5:7587241407215203024:0] peer: [4:7587241405659041087:0] socket: 342 qp: -1
2025-12-24T02:11:41.011659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241411510170330:2094] [node 4] ICS10 traffic start
2025-12-24T02:11:41.011670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.012080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536355:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.012091Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241406888536355:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.015290Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786565:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.016512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241413814843881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26697
2025-12-24T02:11:41.015696Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536369:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.015780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241406888536355:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.016786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241406888536342:2105] [node 5] ICP35 event processing took too much time PendingConnection:279(21071605ns) {HandleHandshakeStatus:329(21070738ns) {DropIncomingHandshake:442(5007ns) DropOutgoingHandshake:457(91ns) SwitchToState:179(216ns) ProcessPendingSessionEvents:495(9349351ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9330187ns max# 9245583ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.017714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786564:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
2025-12-24T02:11:41.017865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786566:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
2025-12-24T02:11:41.017823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241411510170328:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37887
2025-12-24T02:11:41.019000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241407215203020:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.017982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241411183503671:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27641
2025-12-24T02:11:41.019624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241407215202938:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.019642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.019655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241407215202938:2049] [node 2] ICP22 created new session: [5:7587241411510170332:2049]
2025-12-24T02:11:41.019666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241411510170332:2049] [node 2] ICS09 handshake done sender: [5:7587241407215203020:2119] self: [5:7587241407215203023:0] peer: [2:7587241406888536349:0] socket: 365 qp: -1
2025-12-24T02:11:41.019674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241411510170332:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.019689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.020560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241405659041109:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.020658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405659041015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.020265Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241406888536278:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18792) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241406888536368:2118] held: no
2025-12-24T02:11:41.020436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241406888536368:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.021202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411996786564:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.021247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786568:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
2025-12-24T02:11:41.021281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407701819265:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.021507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241406888536360:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.021515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241406888536360:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.021969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241406888536279:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.021987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.021995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241406888536279:2049] [node 3] ICP22 created new session: [2:7587241411183503673:2049]
2025-12-24T02:11:41.022001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241411183503673:2049] [node 3] ICS09 handshake done sender: [2:7587241406888536355:2113] self: [2:7587241406888536343:0] peer: [3:7587241412232729828:0] socket: 390 qp: -1
2025-12-24T02:11:41.022007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241411183503673:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:41.022015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.023036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241406888536360:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.023103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407701819249:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.023427Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241412232729830:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-12-24T02:11:41.023208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411996786568:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.023220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407701819182:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.023231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.023236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407701819182:2048] [node 1] ICP22 created new session: [6:7587241411996786569:2048]
2025-12-24T02:11:41.023241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411996786569:2048] [node 1] ICS09 handshake done sender: [6:7587241407701819265:2118] self: [6:7587241407701819266:0] peer: [1:7587241409519876555:0] socket: 375 qp: -1
2025-12-24T02:11:41.023246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411996786569:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.023254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.023601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407701819250:2089] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.023610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.023615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407701819250:2089] [node 4] ICP22 created new session: [6:7587241411996786571:2089]
2025-12-24T02:11:41.023620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411996786571:2089] [node 4] ICS09 handshake done sender: [6:7587241407701819249:2108] self: [6:7587241407701819263:0] peer: [4:7587241405659041073:0] socket: 334 qp: -1
2025-12-24T02:11:41.023625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411996786571:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:41.023632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.023829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407701819184:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.024073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241407701819196:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.024184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786567:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.025209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409954008407:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:41.025238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409954008408:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:41.024977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241407701819262:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.025528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762519:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.025536Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241407937762519:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.026102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407937762531:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.026338Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241405659041109:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.026495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407701819183:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.026508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.026513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407701819183:2049] [node 2] ICP22 created new session: [6:7587241411996786576:2049]
2025-12-24T02:11:41.026519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411996786576:2049] [node 2] ICS09 handshake done sender: [6:7587241407701819262:2117] self: [6:7587241407701819264:0] peer: [2:7587241406888536346:0] socket: 369 qp: -1
2025-12-24T02:11:41.026524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411996786576:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.026531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.028244Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407937762519:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.028325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405659041015:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.028346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.028351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405659041015:2049] [node 2] ICP22 created new session: [4:7587241409954008409:2049]
2025-12-24T02:11:41.028356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409954008409:2049] [node 2] ICS09 handshake done sender: [4:7587241405659041109:2119] self: [4:7587241409954008406:0] peer: [2:7587241406888536347:0] socket: 370 qp: -1
2025-12-24T02:11:41.028362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409954008409:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.028371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.029300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241407937762505:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.029313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.029318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407937762505:2103] [node 6] ICP22 created new session: [3:7587241412232729832:2103]
2025-12-24T02:11:41.029324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241412232729832:2103] [node 6] ICS09 handshake done sender: [3:7587241407937762519:2116] self: [3:7587241407937762517:0] peer: [6:7587241411996786574:0] socket: 377 qp: -1
2025-12-24T02:11:41.029329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241412232729832:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:41.029335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.029692Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241407215203017:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.029705Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241407215203017:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.030462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241407215203017:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.030538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241407215203005:2101] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.030554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.030560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241407215203005:2101] [node 6] ICP22 created new session: [5:7587241411510170335:2101]
2025-12-24T02:11:41.030565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241411510170335:2101] [node 6] ICS09 handshake done sender: [5:7587241407215203017:2117] self: [5:7587241407215203011:0] peer: [6:7587241411996786575:0] socket: 392 qp: -1
2025-12-24T02:11:41.030570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241411510170335:2101] [node 6] ICS10 traffic start
2025-12-24T02:11:41.030576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.031267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241406888536278:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.031285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.031292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241406888536278:2048] [node 1] ICP22 created new session: [2:7587241411183503676:2048]
2025-12-24T02:11:41.031298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241411183503676:2048] [node 1] ICS09 handshake done sender: [2:7587241406888536368:2118] self: [2:7587241406888536372:0] peer: [1:7587241409519876546:0] socket: 383 qp: -1
2025-12-24T02:11:41.031303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241411183503676:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.031309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.031537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241406888536341:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.031547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.031550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241406888536341:2104] [node 4] ICP22 created new session: [2:7587241411183503678:2104]
2025-12-24T02:11:41.031554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241411183503678:2104] [node 4] ICS09 handshake done sender: [2:7587241406888536360:2117] self: [2:7587241406888536347:0] peer: [4:7587241409954008406:0] socket: 366 qp: -1
2025-12-24T02:11:41.031558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241411183503678:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:41.031562Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.032628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241407937762441:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.032644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.032649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407937762441:2049] [node 2] ICP22 created new session: [3:7587241412232729834:2049]
2025-12-24T02:11:41.032655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241412232729834:2049] [node 2] ICS09 handshake done sender: [3:7587241407937762531:2119] self: [3:7587241412232729828:0] peer: [2:7587241406888536343:0] socket: 393 qp: -1
2025-12-24T02:11:41.032661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241412232729834:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.032667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.035027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241412232729836:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35943
2025-12-24T02:11:41.034928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786578:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.039834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411996786579:2125] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20602
2025-12-24T02:11:41.040133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241412232729836:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.040168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241407937762505:2103] [node 6] ICP18 handshake (actor [3:7587241412232729836:2121]) is too old
2025-12-24T02:11:41.040073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411996786564:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.040518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411996786568:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.041358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241409954008407:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.041406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241405659041016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.041607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241407937762526:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.041614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241407937762526:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.042350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241407937762526:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.042435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241407937762516:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.042458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.042464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241407937762516:2107] [node 4] ICP22 created new session: [3:7587241412232729838:2107]
2025-12-24T02:11:41.042470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241412232729838:2107] [node 4] ICS09 handshake done sender: [3:7587241407937762526:2117] self: [3:7587241407937762520:0] peer: [4:7587241409954008411:0] socket: 363 qp: -1
2025-12-24T02:11:41.042476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241412232729838:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:41.042499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.042689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407701819184:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.042718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.042723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407701819184:2050] [node 3] ICP22 created new session: [6:7587241411996786580:2050]
2025-12-24T02:11:41.042728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411996786580:2050] [node 3] ICS09 handshake done sender: [6:7587241411996786564:2119] self: [6:7587241411996786574:0] peer: [3:7587241407937762517:0] socket: 380 qp: -1
2025-12-24T02:11:41.042733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411996786580:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.042740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.042919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241407701819184:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:41.043116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241407701819196:2060] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.043124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.043129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241407701819196:2060] [node 5] ICP22 created new session: [6:7587241411996786582:2060]
2025-12-24T02:11:41.043133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411996786582:2060] [node 5] ICS09 handshake done sender: [6:7587241411996786568:2123] self: [6:7587241411996786575:0] peer: [5:7587241407215203011:0] socket: 398 qp: -1
2025-12-24T02:11:41.043136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411996786582:2060] [node 5] ICS10 traffic start
2025-12-24T02:11:41.043142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.043277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241407701819196:2060] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:41.043668Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241409519876554:2105] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.044289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241409954008412:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42620
2025-12-24T02:11:41.045455Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241405659041016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241409954008407:2120] held: no
2025-12-24T02:11:41.046116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241409954008407:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.046177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241405659041016:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.046192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.046199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241405659041016:2050] [node 3] ICP22 created new session: [4:7587241409954008413:2050]
2025-12-24T02:11:41.046205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241409954008413:2050] [node 3] ICS09 handshake done sender: [4:7587241409954008407:2120] self: [4:7587241409954008411:0] peer: [3:7587241407937762520:0] socket: 400 qp: -1
2025-12-24T02:11:41.046210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241409954008413:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.046217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.064894Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:41.065008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241411183503676:2048] [node 1] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:41.065019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241406888536278:2048] [node 1] ICP30 unregister session Session# [2:7587241411183503676:2048] VirtualId# [2:7587241406888536372:0]
2025-12-24T02:11:41.065028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241411183503676:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.065064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.065087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.065507Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:41.065573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241407937762529:2048] [node 1] ICS01 socket: 339 reason# EndOfStream
2025-12-24T02:11:41.065578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241407937762440:2048] [node 1] ICP30 unregister session Session# [3:7587241407937762529:2048] VirtualId# [3:7587241407937762522:0]
2025-12-24T02:11:41.065583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241407937762529:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.065609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.065624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.090782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:41.090895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241407215203029:2048] [node 1] ICS07 socket disconnect 343 reason# EndOfStream
2025-12-24T02:11:41.090903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241407215203029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.090968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.090997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241407215203029:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:41.091272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:41.091346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411996786569:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-12-24T02:11:41.091353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407701819182:2048] [node 1] ICP30 unregister session Session# [6:7587241411996786569:2048] VirtualId# [6:7587241407701819266:0]
2025-12-24T02:11:41.091360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411996786569:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.091390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.091467Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:41.091543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241405659041103:2048] [node 1] ICS01 socket: 356 reason# EndOfStream
2025-12-24T02:11:41.091550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241405659041014:2048] [node 1] ICP30 unregister session Session# [4:7587241405659041103:2048] VirtualId# [4:7587241405659041096:0]
2025-12-24T02:11:41.091555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241405659041103:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.091587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.091610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.097594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241407215202937:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18792) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:41.097617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241407215203029:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:41.091420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.097623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241407215202937:2048] [node 1] ICP30 unregister session Session# [5:7587241407215203029:2048] VirtualId# [5:7587241407215203022:0]
2025-12-24T02:11:41.097689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.097699Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241407215202937:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:18792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.097704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18792) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.101275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:41.101383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241412232729834:2049] [node 2] ICS01 socket: 393 reason# EndOfStream
2025-12-24T02:11:41.101391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241407937762441:2049] [node 2] ICP30 unregister session Session# [3:7587241412232729834:2049] VirtualId# [3:7587241412232729828:0]
2025-12-24T02:11:41.101400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241412232729834:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.101442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.101465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.114071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:41.114194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241411510170332:2049] [node 2] ICS01 socket: 365 reason# EndOfStream
2025-12-24T02:11:41.114206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241407215202938:2049] [node 2] ICP30 unregister session Session# [5:7587241411510170332:2049] VirtualId# [5:7587241407215203023:0]
2025-12-24T02:11:41.114215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241411510170332:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.114279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.114306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.114521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241407215202937:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:41.114361Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:41.114426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411996786576:2049] [node 2] ICS01 socket: 369 reason# EndOfStream
2025-12-24T02:11:41.114431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407701819183:2049] [node 2] ICP30 unregister session Session# [6:7587241411996786576:2049] VirtualId# [6:7587241407701819264:0]
2025-12-24T02:11:41.114436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411996786576:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.114470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.114491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.114834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:41.114916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241409954008409:2049] [node 2] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:41.114922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241405659041015:2049] [node 2] ICP30 unregister session Session# [4:7587241409954008409:2049] VirtualId# [4:7587241409954008406:0]
2025-12-24T02:11:41.114927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241409954008409:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.114961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.114986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.137948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:41.138126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411996786580:2050] [node 3] ICS01 socket: 380 reason# EndOfStream
2025-12-24T02:11:41.138137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407701819184:2050] [node 3] ICP30 unregister session Session# [6:7587241411996786580:2050] VirtualId# [6:7587241411996786574:0]
2025-12-24T02:11:41.138146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411996786580:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.138227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.138256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.138308Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:41.138393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241407215203027:2050] [node 3] ICS07 socket disconnect 344 reason# EndOfStream
2025-12-24T02:11:41.138398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241407215203027:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.138429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.138452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241407215203027:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:41.139217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241407215202939:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:41.139228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241407215203027:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:41.139233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241407215202939:2050] [node 3] ICP30 unregister session Session# [5:7587241407215203027:2050] VirtualId# [5:7587241407215203021:0]
2025-12-24T02:11:41.139284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.139291Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241407215202939:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.139294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26508) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.151207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241407215202939:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:41.163392Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:41.163512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411996786571:2089] [node 4] ICS01 socket: 334 reason# EndOfStream
2025-12-24T02:11:41.163523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241407701819250:2089] [node 4] ICP30 unregister session Session# [6:7587241411996786571:2089] VirtualId# [6:7587241407701819263:0]
2025-12-24T02:11:41.163532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411996786571:2089] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.163606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:41.163634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:41.247025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241410495678119:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.247044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241410495678120:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.247136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241413941386613:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:41.247152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241413941386614:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.247289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241413881321969:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.247305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241413881321970:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.263232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411685998775:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.263252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411685998776:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.263259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411685998777:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.263605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412965085743:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.263624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412965085744:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.263631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412965085745:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.278889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241409925776224:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.278900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241409925776225:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.278904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241409925776226:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.331059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412965085777:2066] [node 4] ICP01 ready to work
2025-12-24T02:11:41.331164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412965085778:2068] [node 6] ICP01 ready to work
2025-12-24T02:11:41.331019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241413941386674:2106] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.335145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411685998811:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:41.335322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411685998815:2069] [node 6] ICP01 ready to work
2025-12-24T02:11:41.338267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998816:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.338500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998817:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.339116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322029:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.343413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241410495678184:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:41.343553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241410495678185:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:41.343715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678181:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.343812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678182:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.343824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678183:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.351240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241413941386687:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:41.351401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241413941386688:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:41.352244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241413941386686:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25880
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.352871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241413881322035:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:41.353017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241413881322036:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:41.353245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241413881322037:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:41.353302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322033:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
2025-12-24T02:11:41.353361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322034:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.355495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998816:2084] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.355551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411685998775:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.355643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998854:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.359827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322033:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.359925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241413881321969:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.366431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386678:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.366463Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241413941386678:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.367497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241409925776290:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:41.367861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241409925776291:2071] [node 5] ICP01 ready to work
2025-12-24T02:11:41.372910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241410495678198:2116] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.393513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085821:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.393576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.394904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678181:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.396976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386676:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.396994Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241413941386676:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.397563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386686:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.398317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412965085821:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.398466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412965085777:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.399036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241413941386678:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.403434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412965085822:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.403486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412965085744:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.403235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998817:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.403287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998823:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.403292Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241411685998823:2090] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.404246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998857:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.406075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411685998823:2090] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.406252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241411685998811:2064] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.406274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.406282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411685998811:2064] [node 5] ICP22 created new session: [4:7587241411685998861:2064]
2025-12-24T02:11:41.406290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241411685998861:2064] [node 5] ICS09 handshake done sender: [4:7587241411685998823:2090] self: [4:7587241411685998822:0] peer: [5:7587241412965085823:0] socket: 352 qp: -1
2025-12-24T02:11:41.406297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241411685998861:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:41.406311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.406600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411685998816:2084] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.404568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241409925776226:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34015492ns) {Configure:104(34014615ns) {ProcessConfigured:135(31364416ns) {StartInitialHandshake:157(31363788ns) {DropHandshakes:470(1051ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(239ns)} GenerateSessionVirtualId:988(154ns) PrepareNewSessionHandshake:477(1813ns) {SwitchToState:179(1037ns)}}}}}
2025-12-24T02:11:41.405137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776292:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.405209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776293:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.405258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776294:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.404406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241413941386676:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.407046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411685998775:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.407060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.407066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411685998775:2048] [node 1] ICP22 created new session: [4:7587241411685998863:2048]
2025-12-24T02:11:41.407071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241411685998863:2048] [node 1] ICS09 handshake done sender: [4:7587241411685998816:2084] self: [4:7587241411685998855:0] peer: [1:7587241413941386677:0] socket: 349 qp: -1
2025-12-24T02:11:41.407075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241411685998863:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.407082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.410768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322052:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.417718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322039:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.417736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241413881322039:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.418076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241409925776292:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.423789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678182:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.424032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678199:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.424070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241410495678119:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.424425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241413941386674:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.424457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.424464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241413941386674:2106] [node 4] ICP22 created new session: [1:7587241413941386696:2106]
2025-12-24T02:11:41.424471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413941386696:2106] [node 4] ICS09 handshake done sender: [1:7587241413941386678:2109] self: [1:7587241413941386677:0] peer: [4:7587241411685998855:0] socket: 343 qp: -1
2025-12-24T02:11:41.424479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413941386696:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:41.424491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.424606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998860:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.424701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998854:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.426198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411685998777:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.428885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678189:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.428897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241410495678189:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.429229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678183:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.426366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241412965085777:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241412965085821:2115] held: no
2025-12-24T02:11:41.426633Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085825:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.427140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412965085821:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.427269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412965085777:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.427282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.427289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412965085777:2066] [node 4] ICP22 created new session: [5:7587241412965085827:2066]
2025-12-24T02:11:41.427295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412965085827:2066] [node 4] ICS09 handshake done sender: [5:7587241412965085821:2115] self: [5:7587241412965085823:0] peer: [4:7587241411685998822:0] socket: 371 qp: -1
2025-12-24T02:11:41.427301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412965085827:2066] [node 4] ICS10 traffic start
2025-12-24T02:11:41.427311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.430609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322029:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.430944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241413881322033:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.431157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241413881321969:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.431177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.431183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241413881321969:2048] [node 1] ICP22 created new session: [2:7587241413881322056:2048]
2025-12-24T02:11:41.431190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241413881322056:2048] [node 1] ICS09 handshake done sender: [2:7587241413881322033:2111] self: [2:7587241413881322051:0] peer: [1:7587241413941386675:0] socket: 353 qp: -1
2025-12-24T02:11:41.431197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241413881322056:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.431206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.431499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
2025-12-24T02:11:41.431515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241413881322054:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15390
2025-12-24T02:11:41.432133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085829:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.432023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241409925776293:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.432104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241409925776294:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.432128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241409925776291:2071] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.432183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241409925776226:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.432194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241409925776225:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.432267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776305:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.432287Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412965085789:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.432293Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241412965085789:2088] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.433044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412965085789:2088] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.433118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241412965085778:2068] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.433132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.433138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412965085778:2068] [node 6] ICP22 created new session: [5:7587241412965085831:2068]
2025-12-24T02:11:41.433143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412965085831:2068] [node 6] ICS09 handshake done sender: [5:7587241412965085789:2088] self: [5:7587241412965085788:0] peer: [6:7587241409925776306:0] socket: 348 qp: -1
2025-12-24T02:11:41.433150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412965085831:2068] [node 6] ICS10 traffic start
2025-12-24T02:11:41.433158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.431309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241413941386613:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.431329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.431336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241413941386613:2048] [node 2] ICP22 created new session: [1:7587241413941386699:2048]
2025-12-24T02:11:41.431342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413941386699:2048] [node 2] ICS09 handshake done sender: [1:7587241413941386676:2108] self: [1:7587241413941386675:0] peer: [2:7587241413881322051:0] socket: 342 qp: -1
2025-12-24T02:11:41.431348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413941386699:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:41.431355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.434741Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322054:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.434796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241413881322039:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.434766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241410495678189:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.435007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322041:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.435016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241413881322041:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.435361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241413881322035:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.435371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.435379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241413881322035:2108] [node 5] ICP22 created new session: [2:7587241413881322061:2108]
2025-12-24T02:11:41.435384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241413881322061:2108] [node 5] ICS09 handshake done sender: [2:7587241413881322039:2113] self: [2:7587241413881322038:0] peer: [5:7587241412965085824:0] socket: 363 qp: -1
2025-12-24T02:11:41.435390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241413881322061:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:41.435399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.436822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241413881322035:2108] [node 5] ICP18 handshake (actor [2:7587241413881322054:2120]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.437417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998860:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.437632Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998868:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.437888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386680:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.437899Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241413941386680:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.439005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678199:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.442878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678187:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.442901Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241410495678187:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.443296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241410495678185:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.443320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.443329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241410495678185:2106] [node 4] ICP22 created new session: [3:7587241410495678206:2106]
2025-12-24T02:11:41.443338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241410495678206:2106] [node 4] ICS09 handshake done sender: [3:7587241410495678189:2113] self: [3:7587241410495678188:0] peer: [4:7587241411685998867:0] socket: 358 qp: -1
2025-12-24T02:11:41.443346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241410495678206:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:41.443361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.446737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998869:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.447441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241413881322041:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.448098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322034:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.448153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241413881322036:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.448171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.448179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241413881322036:2109] [node 6] ICP22 created new session: [2:7587241413881322063:2109]
2025-12-24T02:11:41.448185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241413881322063:2109] [node 6] ICS09 handshake done sender: [2:7587241413881322041:2114] self: [2:7587241413881322040:0] peer: [6:7587241409925776308:0] socket: 364 qp: -1
2025-12-24T02:11:41.448192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241413881322063:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:41.448204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.448777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412965085822:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.448212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241410495678120:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.448716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241413941386698:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25880
2025-12-24T02:11:41.448775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241413941386702:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25880
2025-12-24T02:11:41.448823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241413941386701:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25880
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.449061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241412965085744:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7740) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241412965085822:2116] held: no
2025-12-24T02:11:41.449172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412965085744:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.449182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.449187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412965085744:2049] [node 2] ICP22 created new session: [5:7587241412965085834:2049]
2025-12-24T02:11:41.449196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412965085834:2049] [node 2] ICS09 handshake done sender: [5:7587241412965085822:2116] self: [5:7587241412965085824:0] peer: [2:7587241413881322038:0] socket: 372 qp: -1
2025-12-24T02:11:41.449202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412965085834:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.449211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.449856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085836:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.447632Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241409925776305:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.447687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241409925776290:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.447806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776309:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.450892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241410495678187:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.451003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241413941386680:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.451726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998827:2092] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.451753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241411685998827:2092] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.452441Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411685998869:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.452548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411685998776:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.452594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411685998854:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.450110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322043:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.450120Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241413881322043:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.451520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241413881322043:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.451593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241413881321970:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.451608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.451614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241413881321970:2049] [node 3] ICP22 created new session: [2:7587241413881322067:2049]
2025-12-24T02:11:41.451621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241413881322067:2049] [node 3] ICS09 handshake done sender: [2:7587241413881322043:2115] self: [2:7587241413881322042:0] peer: [3:7587241410495678210:0] socket: 365 qp: -1
2025-12-24T02:11:41.451627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241413881322067:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:41.451634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.452797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241410495678184:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.452814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.452821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241410495678184:2103] [node 6] ICP22 created new session: [3:7587241410495678213:2103]
2025-12-24T02:11:41.452828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241410495678213:2103] [node 6] ICS09 handshake done sender: [3:7587241410495678187:2112] self: [3:7587241410495678186:0] peer: [6:7587241409925776307:0] socket: 357 qp: -1
2025-12-24T02:11:41.452834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241410495678213:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:41.452842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.454834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.454565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386701:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.454585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241413941386614:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.454597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.454605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241413941386614:2049] [node 3] ICP22 created new session: [1:7587241413941386706:2049]
2025-12-24T02:11:41.454612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413941386706:2049] [node 3] ICS09 handshake done sender: [1:7587241413941386680:2110] self: [1:7587241413941386679:0] peer: [3:7587241410495678203:0] socket: 344 qp: -1
2025-12-24T02:11:41.454616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413941386706:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:41.454624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.458291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085837:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.458652Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386698:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.461862Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322053:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.461919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241413881322047:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.461924Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241413881322047:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.462544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241413881322047:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.462616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241413881322037:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.462631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.462639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241413881322037:2110] [node 4] ICP22 created new session: [2:7587241413881322071:2110]
2025-12-24T02:11:41.462646Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241413881322071:2110] [node 4] ICS09 handshake done sender: [2:7587241413881322047:2117] self: [2:7587241413881322046:0] peer: [4:7587241411685998872:0] socket: 390 qp: -1
2025-12-24T02:11:41.462653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241413881322071:2110] [node 4] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.462966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678212:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.462664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.462892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241411685998777:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10267) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241411685998854:2116] held: no
2025-12-24T02:11:41.463018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411685998777:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.463028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.463035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411685998777:2050] [node 3] ICP22 created new session: [4:7587241411685998874:2050]
2025-12-24T02:11:41.463042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241411685998874:2050] [node 3] ICS09 handshake done sender: [4:7587241411685998854:2116] self: [4:7587241411685998867:0] peer: [3:7587241410495678188:0] socket: 366 qp: -1
2025-12-24T02:11:41.463049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241411685998874:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.463058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.463494Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411685998827:2092] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.464332Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241410495678120:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7740) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241410495678199:2117] held: no
2025-12-24T02:11:41.464471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241410495678182:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.465752Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241410495678199:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.466201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241409925776225:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7740) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241409925776294:2113] held: no
2025-12-24T02:11:41.466332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241409925776292:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.466166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241410495678119:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.466187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.466195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241410495678119:2048] [node 1] ICP22 created new session: [3:7587241410495678216:2048]
2025-12-24T02:11:41.466202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241410495678216:2048] [node 1] ICS09 handshake done sender: [3:7587241410495678182:2110] self: [3:7587241410495678203:0] peer: [1:7587241413941386679:0] socket: 355 qp: -1
2025-12-24T02:11:41.466211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241410495678216:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.466221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.466638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241410495678120:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.466646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.466650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241410495678120:2049] [node 2] ICP22 created new session: [3:7587241410495678218:2049]
2025-12-24T02:11:41.466654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241410495678218:2049] [node 2] ICS09 handshake done sender: [3:7587241410495678199:2117] self: [3:7587241410495678210:0] peer: [2:7587241413881322042:0] socket: 373 qp: -1
2025-12-24T02:11:41.466659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241410495678218:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.466664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.467574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412965085837:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.467616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412965085745:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.467940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241410495678201:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.467948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241410495678201:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.468967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085839:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.469075Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241410495678201:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.469246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412965085837:2120] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.469392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412965085745:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.469411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.469418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412965085745:2050] [node 3] ICP22 created new session: [5:7587241412965085840:2050]
2025-12-24T02:11:41.469424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412965085840:2050] [node 3] ICS09 handshake done sender: [5:7587241412965085837:2120] self: [5:7587241412965085838:0] peer: [3:7587241410495678200:0] socket: 405 qp: -1
2025-12-24T02:11:41.469430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412965085840:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.469438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.470662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241410495678198:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.470680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.470686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241410495678198:2116] [node 5] ICP22 created new session: [3:7587241410495678221:2116]
2025-12-24T02:11:41.470704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241410495678221:2116] [node 5] ICS09 handshake done sender: [3:7587241410495678201:2118] self: [3:7587241410495678200:0] peer: [5:7587241412965085838:0] socket: 404 qp: -1
2025-12-24T02:11:41.470710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241410495678221:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:41.470718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.471563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386702:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.471950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241411685998815:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.471972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241413941386614:2049] [node 3] ICP18 handshake (actor [1:7587241413941386702:2118]) is too old
2025-12-24T02:11:41.471973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.471977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411685998815:2069] [node 6] ICP22 created new session: [4:7587241411685998876:2069]
2025-12-24T02:11:41.471984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241411685998876:2069] [node 6] ICS09 handshake done sender: [4:7587241411685998827:2092] self: [4:7587241411685998826:0] peer: [6:7587241409925776311:0] socket: 350 qp: -1
2025-12-24T02:11:41.471988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241411685998876:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:41.471993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.472073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776310:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.472217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412965085836:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.472245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412965085743:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.472372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241409925776291:2071] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.472386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.472391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241409925776291:2071] [node 5] ICP22 created new session: [6:7587241409925776315:2071]
2025-12-24T02:11:41.472396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241409925776315:2071] [node 5] ICS09 handshake done sender: [6:7587241409925776292:2111] self: [6:7587241409925776306:0] peer: [5:7587241412965085788:0] socket: 368 qp: -1
2025-12-24T02:11:41.472401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241409925776315:2071] [node 5] ICS10 traffic start
2025-12-24T02:11:41.472415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.472852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241409925776294:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.472455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386690:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.472464Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241413941386690:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.473212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241413941386690:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.473298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241413941386687:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.473317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.473324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241413941386687:2111] [node 5] ICP22 created new session: [1:7587241413941386711:2111]
2025-12-24T02:11:41.473329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413941386711:2111] [node 5] ICS09 handshake done sender: [1:7587241413941386690:2114] self: [1:7587241413941386689:0] peer: [5:7587241412965085842:0] socket: 400 qp: -1
2025-12-24T02:11:41.473335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413941386711:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:41.473342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.473856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241409925776225:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.473872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.473878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241409925776225:2049] [node 2] ICP22 created new session: [6:7587241409925776317:2049]
2025-12-24T02:11:41.473883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241409925776317:2049] [node 2] ICS09 handshake done sender: [6:7587241409925776294:2113] self: [6:7587241409925776308:0] peer: [2:7587241413881322040:0] socket: 370 qp: -1
2025-12-24T02:11:41.473889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241409925776317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.473894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.474273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776314:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.474681Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241411685998776:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7740) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241411685998869:2121] held: no
2025-12-24T02:11:41.475786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241409925776305:2118] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.475899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411685998873:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23656
2025-12-24T02:11:41.476106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411685998869:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.476768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411685998776:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.476782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.476787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411685998776:2049] [node 2] ICP22 created new session: [4:7587241411685998878:2049]
2025-12-24T02:11:41.476796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241411685998878:2049] [node 2] ICS09 handshake done sender: [4:7587241411685998869:2121] self: [4:7587241411685998872:0] peer: [2:7587241413881322046:0] socket: 395 qp: -1
2025-12-24T02:11:41.476802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241411685998878:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.476809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.477316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241409925776290:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.477330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.477335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241409925776290:2067] [node 4] ICP22 created new session: [6:7587241409925776320:2067]
2025-12-24T02:11:41.477340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241409925776320:2067] [node 4] ICS09 handshake done sender: [6:7587241409925776305:2118] self: [6:7587241409925776311:0] peer: [4:7587241411685998826:0] socket: 347 qp: -1
2025-12-24T02:11:41.477345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241409925776320:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:41.477351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.477508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241409925776290:2067] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:41.477866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776312:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.478329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776313:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.478419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241409925776293:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.478006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241412965085743:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241412965085836:2119] held: no
2025-12-24T02:11:41.478579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412965085843:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6440
2025-12-24T02:11:41.478638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241409925776313:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.478650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241409925776226:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.478659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.478664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241409925776226:2050] [node 3] ICP22 created new session: [6:7587241409925776322:2050]
2025-12-24T02:11:41.478669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241409925776322:2050] [node 3] ICS09 handshake done sender: [6:7587241409925776293:2112] self: [6:7587241409925776307:0] peer: [3:7587241410495678186:0] socket: 369 qp: -1
2025-12-24T02:11:41.478673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241409925776322:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.478678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.479280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241409925776224:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.479395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241413941386692:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.479405Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241413941386692:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.480134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241413941386692:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.480211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241413941386688:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.480223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.480228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241413941386688:2112] [node 6] ICP22 created new session: [1:7587241413941386714:2112]
2025-12-24T02:11:41.480234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241413941386714:2112] [node 6] ICS09 handshake done sender: [1:7587241413941386692:2115] self: [1:7587241413941386691:0] peer: [6:7587241409925776324:0] socket: 401 qp: -1
2025-12-24T02:11:41.480238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241413941386714:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:41.480245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.483490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241410495678223:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6952
2025-12-24T02:11:41.484336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412965085836:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.484442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412965085743:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.484456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.484464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412965085743:2048] [node 1] ICP22 created new session: [5:7587241412965085844:2048]
2025-12-24T02:11:41.484470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412965085844:2048] [node 1] ICS09 handshake done sender: [5:7587241412965085836:2119] self: [5:7587241412965085842:0] peer: [1:7587241413941386689:0] socket: 383 qp: -1
2025-12-24T02:11:41.484477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412965085844:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.484486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.487210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241409925776224:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6245) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241409925776313:2122] held: no
2025-12-24T02:11:41.487465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241409925776325:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39468
2025-12-24T02:11:41.487660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241409925776313:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.487718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241409925776224:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.487745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.487752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241409925776224:2048] [node 1] ICP22 created new session: [6:7587241409925776326:2048]
2025-12-24T02:11:41.487758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241409925776326:2048] [node 1] ICS09 handshake done sender: [6:7587241409925776313:2122] self: [6:7587241409925776324:0] peer: [1:7587241413941386691:0] socket: 397 qp: -1
2025-12-24T02:11:41.487765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241409925776326:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.487773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.500954Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:41.501065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241410495678216:2048] [node 1] ICS01 socket: 355 reason# EndOfStream
2025-12-24T02:11:41.501077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241410495678119:2048] [node 1] ICP30 unregister session Session# [3:7587241410495678216:2048] VirtualId# [3:7587241410495678203:0]
2025-12-24T02:11:41.501085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241410495678216:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.501125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.501150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.502747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:41.502841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241413881322056:2048] [node 1] ICS01 socket: 353 reason# EndOfStream
2025-12-24T02:11:41.502848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241413881321969:2048] [node 1] ICP30 unregister session Session# [2:7587241413881322056:2048] VirtualId# [2:7587241413881322051:0]
2025-12-24T02:11:41.502855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241413881322056:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.502914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.502937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.503066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:41.503135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241409925776326:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-12-24T02:11:41.503141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241409925776224:2048] [node 1] ICP30 unregister session Session# [6:7587241409925776326:2048] VirtualId# [6:7587241409925776324:0]
2025-12-24T02:11:41.503145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241409925776326:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.503151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.503176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.504763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:41.504849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412965085844:2048] [node 1] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:41.504857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412965085743:2048] [node 1] ICP30 unregister session Session# [5:7587241412965085844:2048] VirtualId# [5:7587241412965085842:0]
2025-12-24T02:11:41.504864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412965085844:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.504913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.504937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.511367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:41.511460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241411685998863:2048] [node 1] ICS01 socket: 349 reason# EndOfStream
2025-12-24T02:11:41.511468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241411685998775:2048] [node 1] ICP30 unregister session Session# [4:7587241411685998863:2048] VirtualId# [4:7587241411685998855:0]
2025-12-24T02:11:41.511476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241411685998863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.511530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.511556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.515501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:41.515584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241410495678218:2049] [node 2] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:41.515591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241410495678120:2049] [node 2] ICP30 unregister session Session# [3:7587241410495678218:2049] VirtualId# [3:7587241410495678210:0]
2025-12-24T02:11:41.515597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241410495678218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.515625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.515646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.525403Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:41.525502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412965085834:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-12-24T02:11:41.525512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412965085744:2049] [node 2] ICP30 unregister session Session# [5:7587241412965085834:2049] VirtualId# [5:7587241412965085824:0]
2025-12-24T02:11:41.525521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412965085834:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.525572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.525598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.526787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:41.526879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241409925776317:2049] [node 2] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:41.526888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241409925776225:2049] [node 2] ICP30 unregister session Session# [6:7587241409925776317:2049] VirtualId# [6:7587241409925776308:0]
2025-12-24T02:11:41.526895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241409925776317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.526948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.526974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.527017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:41.527086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241411685998878:2049] [node 2] ICS01 socket: 395 reason# EndOfStream
2025-12-24T02:11:41.527091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241411685998776:2049] [node 2] ICP30 unregister session Session# [4:7587241411685998878:2049] VirtualId# [4:7587241411685998872:0]
2025-12-24T02:11:41.527096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241411685998878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.527123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.527145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.538880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:41.538992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241411685998874:2050] [node 3] ICS01 socket: 366 reason# EndOfStream
2025-12-24T02:11:41.539004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241411685998777:2050] [node 3] ICP30 unregister session Session# [4:7587241411685998874:2050] VirtualId# [4:7587241411685998867:0]
2025-12-24T02:11:41.539012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241411685998874:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.539055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.539080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.542484Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:41.542499Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:41.542592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412965085840:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:41.542601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412965085745:2050] [node 3] ICP30 unregister session Session# [5:7587241412965085840:2050] VirtualId# [5:7587241412965085838:0]
2025-12-24T02:11:41.542582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241409925776322:2050] [node 3] ICS01 socket: 369 reason# EndOfStream
2025-12-24T02:11:41.542590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241409925776226:2050] [node 3] ICP30 unregister session Session# [6:7587241409925776322:2050] VirtualId# [6:7587241409925776307:0]
2025-12-24T02:11:41.542597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241409925776322:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.542656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.542680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.542608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412965085840:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.542654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.542678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.558802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:41.558920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241409925776320:2067] [node 4] ICS01 socket: 347 reason# EndOfStream
2025-12-24T02:11:41.558934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241409925776290:2067] [node 4] ICP30 unregister session Session# [6:7587241409925776320:2067] VirtualId# [6:7587241409925776311:0]
2025-12-24T02:11:41.558944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241409925776320:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.559000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:41.559029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:41.622908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410553897574:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.622934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410553897575:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.622903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411877068186:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:41.622923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411877068187:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.634973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241413067813585:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.634986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241413067813586:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.643722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241411807937917:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.643752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241411807937918:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.634952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241413463454458:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.634965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241413463454459:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.643757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241411807937919:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.650044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412633437784:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.650062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412633437785:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.634972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241413463454460:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.652906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412633437783:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.675451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412633437842:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:41.676520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411877068249:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:41.676667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411877068250:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:41.676766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411877068251:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:41.678596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241413067813645:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23362
2025-12-24T02:11:41.679567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410553897639:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:41.679690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410553897640:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:41.679783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410553897641:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.685280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241413067813650:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:41.685385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241413067813651:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:41.685477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241413067813652:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.687543Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437848:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.687608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437849:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.689211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437848:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.689575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412633437783:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.687455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241411807937972:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:41.688869Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411877068265:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:778
2025-12-24T02:11:41.689730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068257:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.689735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411877068257:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.702466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807937986:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.703104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897653:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
2025-12-24T02:11:41.703165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897654:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
2025-12-24T02:11:41.704014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437849:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.704543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412633437785:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.703392Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813645:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.703499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241413067813585:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.704878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813658:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.704884Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241413067813658:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.705291Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241413067813666:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23362
2025-12-24T02:11:41.705498Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241413067813658:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.706224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807937987:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.705602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241413067813651:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.705623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.705632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241413067813651:2109] [node 5] ICP22 created new session: [3:7587241413067813668:2109]
2025-12-24T02:11:41.705640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241413067813668:2109] [node 5] ICS09 handshake done sender: [3:7587241413067813658:2113] self: [3:7587241413067813657:0] peer: [5:7587241412633437855:0] socket: 353 qp: -1
2025-12-24T02:11:41.705647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241413067813668:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:41.705657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.703027Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411877068257:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.703248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411877068250:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.703266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.703277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411877068250:2108] [node 5] ICP22 created new session: [1:7587241411877068267:2108]
2025-12-24T02:11:41.703288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241411877068267:2108] [node 5] ICS09 handshake done sender: [1:7587241411877068257:2112] self: [1:7587241411877068256:0] peer: [5:7587241412633437853:0] socket: 345 qp: -1
2025-12-24T02:11:41.703297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241411877068267:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:41.703309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.707313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897653:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.707356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897654:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.710722Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897655:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.711877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437856:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.711443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454528:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.711561Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454529:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.711623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454530:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.711666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454531:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.719252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454530:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.719380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241413463454460:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.719524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813666:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.723273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241413463454535:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:41.723445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241413463454536:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:41.725475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241411807937993:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:41.726307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411807937987:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.726748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241411807937919:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.727227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813660:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.727241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241413067813660:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.727999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241413067813660:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.728097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241413067813652:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.728116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.728123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241413067813652:2110] [node 6] ICP22 created new session: [3:7587241413067813673:2110]
2025-12-24T02:11:41.728131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241413067813673:2110] [node 6] ICS09 handshake done sender: [3:7587241413067813660:2114] self: [3:7587241413067813659:0] peer: [6:7587241411807937997:0] socket: 376 qp: -1
2025-12-24T02:11:41.728139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241413067813673:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:41.728150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.729329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897655:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.730299Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068253:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.730326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411877068253:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.730786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813656:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.730805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241413067813656:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.730847Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437857:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.734723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897656:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
2025-12-24T02:11:41.736384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897646:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.736391Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410553897646:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.736979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410553897646:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.737084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410553897641:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.737104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.737112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410553897641:2107] [node 4] ICP22 created new session: [2:7587241410553897661:2107]
2025-12-24T02:11:41.737120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241410553897661:2107] [node 4] ICS09 handshake done sender: [2:7587241410553897646:2112] self: [2:7587241410553897645:0] peer: [4:7587241413463454541:0] socket: 361 qp: -1
2025-12-24T02:11:41.737129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241410553897661:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:41.737156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.735042Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241411877068250:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31674770ns) {HandleHandshakeStatus:329(31673721ns) {DropIncomingHandshake:442(235ns) DropOutgoingHandshake:457(991ns) SwitchToState:179(367ns) ProcessPendingSessionEvents:495(24007889ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 23986691ns max# 23900389ns}}}
2025-12-24T02:11:41.735643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411877068269:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:778
2025-12-24T02:11:41.735688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411877068270:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:778
2025-12-24T02:11:41.735701Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411877068271:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:778
2025-12-24T02:11:41.736083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454531:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.736119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454529:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.736190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241413463454459:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.736203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241413463454458:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.737694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454543:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.737862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241413463454531:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.737919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241413463454459:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.737933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.737939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241413463454459:2049] [node 2] ICP22 created new session: [4:7587241413463454544:2049]
2025-12-24T02:11:41.737945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241413463454544:2049] [node 2] ICS09 handshake done sender: [4:7587241413463454531:2114] self: [4:7587241413463454541:0] peer: [2:7587241410553897645:0] socket: 382 qp: -1
2025-12-24T02:11:41.737951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241413463454544:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.737958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.739134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068271:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.739204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068269:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.741964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437858:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.742815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411807937986:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.742858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241411807937918:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.743282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068265:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.744773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807937999:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.744861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.745101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411807937987:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.745177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241411807937919:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.745196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.745202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241411807937919:2050] [node 3] ICP22 created new session: [6:7587241411807938001:2050]
2025-12-24T02:11:41.745211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411807938001:2050] [node 3] ICS09 handshake done sender: [6:7587241411807937987:2112] self: [6:7587241411807937997:0] peer: [3:7587241413067813659:0] socket: 377 qp: -1
2025-12-24T02:11:41.745219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411807938001:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.745228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.746718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241412633437860:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:41.746942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437857:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.747655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412633437848:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.750674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241410553897641:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:41.750775Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068255:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.750795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411877068255:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.751035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411877068253:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.750477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412633437849:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.751133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412633437783:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.751153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.751163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412633437783:2048] [node 1] ICP22 created new session: [5:7587241412633437865:2048]
2025-12-24T02:11:41.751171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412633437865:2048] [node 1] ICS09 handshake done sender: [5:7587241412633437848:2109] self: [5:7587241412633437853:0] peer: [1:7587241411877068256:0] socket: 348 qp: -1
2025-12-24T02:11:41.751181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412633437865:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.751191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.751352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412633437785:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.751363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.751371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412633437785:2050] [node 3] ICP22 created new session: [5:7587241412633437867:2050]
2025-12-24T02:11:41.751377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412633437867:2050] [node 3] ICS09 handshake done sender: [5:7587241412633437849:2110] self: [5:7587241412633437855:0] peer: [3:7587241413067813657:0] socket: 354 qp: -1
2025-12-24T02:11:41.751383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412633437867:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.751389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.751872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437861:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.750871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241413067813676:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23362
2025-12-24T02:11:41.750950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241413067813656:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.751279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241413067813650:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.751297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.751304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241413067813650:2108] [node 4] ICP22 created new session: [3:7587241413067813678:2108]
2025-12-24T02:11:41.751310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241413067813678:2108] [node 4] ICS09 handshake done sender: [3:7587241413067813656:2112] self: [3:7587241413067813655:0] peer: [4:7587241413463454532:0] socket: 352 qp: -1
2025-12-24T02:11:41.751315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241413067813678:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:41.751325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.752388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241413067813645:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.752687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241413067813585:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.752700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.752706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241413067813585:2048] [node 1] ICP22 created new session: [3:7587241413067813680:2048]
2025-12-24T02:11:41.752711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241413067813680:2048] [node 1] ICS09 handshake done sender: [3:7587241413067813645:2107] self: [3:7587241413067813665:0] peer: [1:7587241411877068252:0] socket: 346 qp: -1
2025-12-24T02:11:41.752717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241413067813680:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.752723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.759039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241413463454459:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20978677ns) {HandleHandshakeStatus:329(20977670ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(1722ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(19634817ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19606820ns max# 19497504ns}}}
2025-12-24T02:11:41.760895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897644:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.760914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410553897644:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.759235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454528:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.760353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241413463454460:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16987) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241413463454530:2113] held: no
2025-12-24T02:11:41.760706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454546:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.760752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454547:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.762912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411807938000:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.762959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241411807937917:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.762246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068270:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.762326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411877068187:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.762348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.762358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411877068187:2049] [node 3] ICP22 created new session: [1:7587241411877068277:2049]
2025-12-24T02:11:41.762365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241411877068277:2049] [node 3] ICS09 handshake done sender: [1:7587241411877068253:2110] self: [1:7587241411877068252:0] peer: [3:7587241413067813665:0] socket: 343 qp: -1
2025-12-24T02:11:41.762373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241411877068277:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:41.762383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.763330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410553897644:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.764153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410553897640:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.764174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.764180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410553897640:2110] [node 6] ICP22 created new session: [2:7587241410553897666:2110]
2025-12-24T02:11:41.764187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241410553897666:2110] [node 6] ICS09 handshake done sender: [2:7587241410553897644:2111] self: [2:7587241410553897642:0] peer: [6:7587241411807937998:0] socket: 360 qp: -1
2025-12-24T02:11:41.764193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241410553897666:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:41.764200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.763572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454546:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.763602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241413463454530:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.764640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241413463454460:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.764650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.764656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241413463454460:2050] [node 3] ICP22 created new session: [4:7587241413463454550:2050]
2025-12-24T02:11:41.764662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241413463454550:2050] [node 3] ICS09 handshake done sender: [4:7587241413463454530:2113] self: [4:7587241413463454532:0] peer: [3:7587241413067813655:0] socket: 381 qp: -1
2025-12-24T02:11:41.764667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241413463454550:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:41.764673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:41.766375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068259:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.766388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411877068259:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.767367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437861:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.767480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437870:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.767544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412633437784:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.768542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437870:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.768580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241412633437842:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.768357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241413463454554:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58176
2025-12-24T02:11:41.768743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454538:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.768747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241413463454538:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.770239Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241413463454529:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.770286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241413463454458:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.770297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.770304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241413463454458:2048] [node 1] ICP22 created new session: [4:7587241413463454555:2048]
2025-12-24T02:11:41.770310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241413463454555:2048] [node 1] ICS09 handshake done sender: [4:7587241413463454529:2112] self: [4:7587241413463454542:0] peer: [1:7587241411877068254:0] socket: 380 qp: -1
2025-12-24T02:11:41.770315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241413463454555:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.770321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.768695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241413067813682:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23362
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.768944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411877068255:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.769092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411877068259:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.769423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411877068249:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.769438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.769446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411877068249:2107] [node 4] ICP22 created new session: [1:7587241411877068283:2107]
2025-12-24T02:11:41.769453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241411877068283:2107] [node 4] ICS09 handshake done sender: [1:7587241411877068255:2111] self: [1:7587241411877068254:0] peer: [4:7587241413463454542:0] socket: 344 qp: -1
2025-12-24T02:11:41.769458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241411877068283:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:41.769467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.769675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411877068251:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.769685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.769695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411877068251:2109] [node 6] ICP22 created new session: [1:7587241411877068285:2109]
2025-12-24T02:11:41.769699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241411877068285:2109] [node 6] ICS09 handshake done sender: [1:7587241411877068259:2113] self: [1:7587241411877068258:0] peer: [6:7587241411807938004:0] socket: 392 qp: -1
2025-12-24T02:11:41.769703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241411877068285:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:41.769708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.770879Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241411807937918:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32397) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241411807937986:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.774633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897649:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.774649Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410553897649:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.776465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897668:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.778014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410553897649:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.778249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410553897639:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.778271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.778278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410553897639:2109] [node 5] ICP22 created new session: [2:7587241410553897670:2109]
2025-12-24T02:11:41.778285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241410553897670:2109] [node 5] ICS09 handshake done sender: [2:7587241410553897649:2113] self: [2:7587241410553897648:0] peer: [5:7587241412633437871:0] socket: 394 qp: -1
2025-12-24T02:11:41.778291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241410553897670:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:41.778303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.778926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938008:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.777857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.777891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241412633437874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30566
2025-12-24T02:11:41.780789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412633437861:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.781497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241413463454538:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.781664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241413463454535:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.781685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.781692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241413463454535:2115] [node 5] ICP22 created new session: [4:7587241413463454558:2115]
2025-12-24T02:11:41.781699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241413463454558:2115] [node 5] ICS09 handshake done sender: [4:7587241413463454538:2117] self: [4:7587241413463454537:0] peer: [5:7587241412633437872:0] socket: 403 qp: -1
2025-12-24T02:11:41.781705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241413463454558:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:41.781716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.791114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938007:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.792204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411807938008:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.792237Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241411807937917:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241411807938000:2118] held: no
2025-12-24T02:11:41.792339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411807938000:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.792485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938005:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.792535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938006:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.792745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241411807937917:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.792765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.792774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241411807937917:2048] [node 1] ICP22 created new session: [6:7587241411807938009:2048]
2025-12-24T02:11:41.792780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411807938009:2048] [node 1] ICS09 handshake done sender: [6:7587241411807938000:2118] self: [6:7587241411807938004:0] peer: [1:7587241411877068258:0] socket: 396 qp: -1
2025-12-24T02:11:41.792790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411807938009:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.792799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.793057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241411807937993:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.793122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411807937986:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.793248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241411807937918:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.793258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.793262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241411807937918:2049] [node 2] ICP22 created new session: [6:7587241411807938012:2049]
2025-12-24T02:11:41.793266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411807938012:2049] [node 2] ICS09 handshake done sender: [6:7587241411807937986:2111] self: [6:7587241411807937998:0] peer: [2:7587241410553897642:0] socket: 363 qp: -1
2025-12-24T02:11:41.793271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411807938012:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.793277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.791303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241413067813682:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.791360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241413067813586:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.795050Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241410553897639:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(16662613ns) {HandleHandshakeStatus:329(16661068ns) {DropIncomingHandshake:442(4983ns) DropOutgoingHandshake:457(357ns) SwitchToState:179(409ns) ProcessPendingSessionEvents:495(16567263ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16522938ns max# 16417589ns}}}
2025-12-24T02:11:41.795142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897668:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.795188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410553897652:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.795191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410553897652:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.795139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241413463454540:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.795153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241413463454540:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.796175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412633437784:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.796200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.796209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412633437784:2049] [node 2] ICP22 created new session: [5:7587241412633437875:2049]
2025-12-24T02:11:41.796217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412633437875:2049] [node 2] ICS09 handshake done sender: [5:7587241412633437861:2117] self: [5:7587241412633437871:0] peer: [2:7587241410553897648:0] socket: 398 qp: -1
2025-12-24T02:11:41.796224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412633437875:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.796234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.796517Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241412633437842:2105] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241412633437870:2119] held: no
2025-12-24T02:11:41.797573Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241413463454540:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.797686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241413463454536:2116] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.797709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.797717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241413463454536:2116] [node 6] ICP22 created new session: [4:7587241413463454561:2116]
2025-12-24T02:11:41.797723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241413463454561:2116] [node 6] ICS09 handshake done sender: [4:7587241413463454540:2118] self: [4:7587241413463454539:0] peer: [6:7587241411807938011:0] socket: 391 qp: -1
2025-12-24T02:11:41.797730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241413463454561:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:41.797742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.798224Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241411807938005:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.798267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938014:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
2025-12-24T02:11:41.798493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412633437870:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.798348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241411807937972:2101] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:41.798754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241412633437863:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.798763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241412633437863:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.798800Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241411807937993:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16611) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241411807938008:2122] held: no
2025-12-24T02:11:41.798958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411807938008:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:41.798876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241412633437842:2105] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.798896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.798904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412633437842:2105] [node 4] ICP22 created new session: [5:7587241412633437877:2105]
2025-12-24T02:11:41.798911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412633437877:2105] [node 4] ICS09 handshake done sender: [5:7587241412633437870:2119] self: [5:7587241412633437872:0] peer: [4:7587241413463454537:0] socket: 409 qp: -1
2025-12-24T02:11:41.798918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412633437877:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:41.798928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.807439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241410553897574:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.807731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241411807937993:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.807765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:41.807781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241411807937993:2114] [node 4] ICP22 created new session: [6:7587241411807938016:2114]
2025-12-24T02:11:41.807788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411807938016:2114] [node 4] ICS09 handshake done sender: [6:7587241411807938008:2122] self: [6:7587241411807938011:0] peer: [4:7587241413463454539:0] socket: 411 qp: -1
2025-12-24T02:11:41.807797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411807938016:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:41.807807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:41.808164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241411807938018:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47118
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.808228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411877068261:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:41.808359Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241413067813684:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23362
2025-12-24T02:11:41.808243Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411877068261:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:41.808366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241412633437863:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:41.808641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241411807938005:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:41.808832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241411807937972:2101] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.808848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:41.808854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241411807937972:2101] [node 5] ICP22 created new session: [6:7587241411807938019:2101]
2025-12-24T02:11:41.808860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241411807938019:2101] [node 5] ICS09 handshake done sender: [6:7587241411807938005:2119] self: [6:7587241411807938015:0] peer: [5:7587241412633437862:0] socket: 374 qp: -1
2025-12-24T02:11:41.808865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241411807938019:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:41.808872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:41.810610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241412633437860:2116] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.810631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:41.810639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241412633437860:2116] [node 6] ICP22 created new session: [5:7587241412633437880:2116]
2025-12-24T02:11:41.810645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241412633437880:2116] [node 6] ICS09 handshake done sender: [5:7587241412633437863:2118] self: [5:7587241412633437862:0] peer: [6:7587241411807938015:0] socket: 401 qp: -1
2025-12-24T02:11:41.810651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241412633437880:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:41.810662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:41.810864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410553897652:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:41.811916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410553897575:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.811938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:41.811943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410553897575:2049] [node 3] ICP22 created new session: [2:7587241410553897675:2049]
2025-12-24T02:11:41.811950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241410553897675:2049] [node 3] ICS09 handshake done sender: [2:7587241410553897652:2114] self: [2:7587241410553897643:0] peer: [3:7587241413067813683:0] socket: 405 qp: -1
2025-12-24T02:11:41.811955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241410553897675:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:41.811961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.812526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410553897674:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36222
2025-12-24T02:11:41.817214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411877068261:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.817685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410553897668:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:41.816630Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241413067813586:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32397) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241413067813682:2118] held: no
2025-12-24T02:11:41.816811Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241413067813682:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:41.816940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241413067813586:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.816952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.816959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241413067813586:2049] [node 2] ICP22 created new session: [3:7587241413067813685:2049]
2025-12-24T02:11:41.816965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241413067813685:2049] [node 2] ICS09 handshake done sender: [3:7587241413067813682:2118] self: [3:7587241413067813683:0] peer: [2:7587241410553897643:0] socket: 410 qp: -1
2025-12-24T02:11:41.816973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241413067813685:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:41.816981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.818255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411877068186:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:41.818278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:41.818286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411877068186:2048] [node 2] ICP22 created new session: [1:7587241411877068288:2048]
2025-12-24T02:11:41.818293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241411877068288:2048] [node 2] ICS09 handshake done sender: [1:7587241411877068261:2114] self: [1:7587241411877068260:0] peer: [2:7587241410553897673:0] socket: 407 qp: -1
2025-12-24T02:11:41.818298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241411877068288:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:41.818308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:41.817803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241410553897574:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:41.817823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:41.817829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410553897574:2048] [node 1] ICP22 created new session: [2:7587241410553897677:2048]
2025-12-24T02:11:41.817835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241410553897677:2048] [node 1] ICS09 handshake done sender: [2:7587241410553897668:2119] self: [2:7587241410553897673:0] peer: [1:7587241411877068260:0] socket: 389 qp: -1
2025-12-24T02:11:41.817840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241410553897677:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:41.817849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:41.836760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:41.836871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241410553897677:2048] [node 1] ICS01 socket: 389 reason# EndOfStream
2025-12-24T02:11:41.836882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241410553897574:2048] [node 1] ICP30 unregister session Session# [2:7587241410553897677:2048] VirtualId# [2:7587241410553897673:0]
2025-12-24T02:11:41.836889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241410553897677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.836940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.836959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.837481Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:41.837549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241413067813680:2048] [node 1] ICS07 socket disconnect 346 reason# EndOfStream
2025-12-24T02:11:41.837554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241413067813680:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.837577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.837593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241413067813680:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:41.839939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241413067813585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:41.839952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241413067813680:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:41.840217Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241413067813585:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:41.840228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241413067813680:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:41.840233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241413067813585:2048] [node 1] ICP30 unregister session Session# [3:7587241413067813680:2048] VirtualId# [3:7587241413067813665:0]
2025-12-24T02:11:41.840248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.840255Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241413067813585:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.840258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.846274Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:41.846369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241413463454555:2048] [node 1] ICS01 socket: 380 reason# EndOfStream
2025-12-24T02:11:41.846377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241413463454458:2048] [node 1] ICP30 unregister session Session# [4:7587241413463454555:2048] VirtualId# [4:7587241413463454542:0]
2025-12-24T02:11:41.846385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241413463454555:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.846447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.846475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.846773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:41.846865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241412633437865:2048] [node 1] ICS07 socket disconnect 348 reason# EndOfStream
2025-12-24T02:11:41.846872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412633437865:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.846916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.846937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241412633437865:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:41.847293Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:41.847364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411807938009:2048] [node 1] ICS01 socket: 396 reason# EndOfStream
2025-12-24T02:11:41.847370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241411807937917:2048] [node 1] ICP30 unregister session Session# [6:7587241411807938009:2048] VirtualId# [6:7587241411807938004:0]
2025-12-24T02:11:41.847376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411807938009:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.847403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:41.847424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.847669Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241412633437783:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:41.847680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412633437865:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:41.847685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412633437783:2048] [node 1] ICP30 unregister session Session# [5:7587241412633437865:2048] VirtualId# [5:7587241412633437853:0]
2025-12-24T02:11:41.847733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:41.847749Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241412633437783:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.847752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2563) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.850905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241413067813585:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:41.859058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:41.859160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241413067813685:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-12-24T02:11:41.859170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241413067813586:2049] [node 2] ICP30 unregister session Session# [3:7587241413067813685:2049] VirtualId# [3:7587241413067813683:0]
2025-12-24T02:11:41.859178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241413067813685:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.859217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.859238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.862630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241412633437783:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:41.865948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:41.866059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412633437875:2049] [node 2] ICS01 socket: 398 reason# EndOfStream
2025-12-24T02:11:41.866070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412633437784:2049] [node 2] ICP30 unregister session Session# [5:7587241412633437875:2049] VirtualId# [5:7587241412633437871:0]
2025-12-24T02:11:41.866078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412633437875:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.866131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.866158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.866200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:41.866270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411807938012:2049] [node 2] ICS01 socket: 363 reason# EndOfStream
2025-12-24T02:11:41.866276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241411807937918:2049] [node 2] ICP30 unregister session Session# [6:7587241411807938012:2049] VirtualId# [6:7587241411807937998:0]
2025-12-24T02:11:41.866280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411807938012:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.866308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.866329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.866899Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:41.866990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241413463454544:2049] [node 2] ICS07 socket disconnect 382 reason# EndOfStream
2025-12-24T02:11:41.866997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241413463454544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.867045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:41.867081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241413463454544:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:41.867561Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241413463454459:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32397) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:41.867573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241413463454544:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:41.867578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241413463454459:2049] [node 2] ICP30 unregister session Session# [4:7587241413463454544:2049] VirtualId# [4:7587241413463454541:0]
2025-12-24T02:11:41.867637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:41.867644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241413463454459:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.867647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32397) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:41.879390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241413463454459:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:41.882879Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:41.882984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241413463454550:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:41.882996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241413463454460:2050] [node 3] ICP30 unregister session Session# [4:7587241413463454550:2050] VirtualId# [4:7587241413463454532:0]
2025-12-24T02:11:41.883006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241413463454550:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.883056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.883082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.882914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:41.882990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241412633437867:2050] [node 3] ICS01 socket: 354 reason# EndOfStream
2025-12-24T02:11:41.882997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241412633437785:2050] [node 3] ICP30 unregister session Session# [5:7587241412633437867:2050] VirtualId# [5:7587241412633437855:0]
2025-12-24T02:11:41.883006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241412633437867:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.883056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.883081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.883116Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:41.883218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411807938001:2050] [node 3] ICS01 socket: 377 reason# EndOfStream
2025-12-24T02:11:41.883225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241411807937919:2050] [node 3] ICP30 unregister session Session# [6:7587241411807938001:2050] VirtualId# [6:7587241411807937997:0]
2025-12-24T02:11:41.883231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411807938001:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.883261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:41.883284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:41.905323Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:41.905438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241411807938016:2114] [node 4] ICS01 socket: 411 reason# EndOfStream
2025-12-24T02:11:41.905450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241411807937993:2114] [node 4] ICP30 unregister session Session# [6:7587241411807938016:2114] VirtualId# [6:7587241411807938011:0]
2025-12-24T02:11:41.905459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241411807938016:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:41.905517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:41.905544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:41.950961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411203208519:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:41.950972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411203208520:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.950961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410751433526:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.950983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410751433527:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:41.962896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241412852229776:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.962915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241412852229777:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.963247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411713210760:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.963255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411713210761:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.963259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411713210762:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.965380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241413575864147:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.965389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241413575864148:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.965394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241413575864149:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:41.969151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241410936049670:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:41.969161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241410936049671:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:41.969166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241410936049672:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.981863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411713210803:2087] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.982878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241410936049686:2062] [node 4] ICP01 ready to work
2025-12-24T02:11:41.983455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241413575864179:2073] [node 4] ICP01 ready to work
2025-12-24T02:11:41.983522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241413575864178:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:41.982084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411203208564:2088] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.985848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411203208569:2092] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61053
2025-12-24T02:11:41.986046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411203208572:2093] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.983202Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241412852229824:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
2025-12-24T02:11:41.980886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410751433556:2076] [node 6] ICP01 ready to work
2025-12-24T02:11:41.981160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410751433559:2078] [node 4] ICP01 ready to work
2025-12-24T02:11:41.981586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410751433560:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41576
2025-12-24T02:11:41.985785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241410751433565:2082] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41576
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:41.986948Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411713210816:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:41.986981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411713210817:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:41.986991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411713210818:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:41.986107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241413575864189:2082] [node 6] ICP01 ready to work
2025-12-24T02:11:41.988522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241410751433580:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:41.987718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241412852229826:2092] [node 4] ICP01 ready to work
2025-12-24T02:11:41.990990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241411713210824:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:41.991055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241411203208583:2095] [node 6] ICP01 ready to work
2025-12-24T02:11:41.991154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241412852229823:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.002055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241413575864222:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.003660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241411203208590:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61053
2025-12-24T02:11:42.003462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210818:2099] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.003529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411713210834:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:42.003565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241411713210835:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:42.003597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411713210761:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.005601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241413575864222:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.005718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241413575864148:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.008555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210834:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.006968Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410751433570:2086] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.006983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410751433570:2086] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.007383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410751433582:2093] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.007387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410751433582:2093] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.008015Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241415046400899:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41576
2025-12-24T02:11:42.010198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241417147197149:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:42.006656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241413575864189:2082] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20488719ns) {Configure:104(20488361ns) {ProcessConfigured:135(20285062ns) {StartInitialHandshake:157(20284867ns) {DropHandshakes:470(139ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(25ns)} GenerateSessionVirtualId:988(54ns) PrepareNewSessionHandshake:477(501ns) {SwitchToState:179(225ns)}}}}}
2025-12-24T02:11:42.008626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831521:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.008721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241413575864178:2078] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.008744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241413575864179:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.011011Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241410751433526:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23841819ns) {Configure:104(23841348ns) {ProcessConfigured:135(23558304ns) {StartInitialHandshake:157(23557977ns) {DropHandshakes:470(817ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(1426ns) {SwitchToState:179(1076ns)}}}}}
2025-12-24T02:11:42.011191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410751433570:2086] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.011683Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241410751433527:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25538095ns) {Configure:104(25537116ns) {ProcessConfigured:135(25256464ns) {StartInitialHandshake:157(25256224ns) {DropHandshakes:470(165ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(632ns) {SwitchToState:179(459ns)}}}}}
2025-12-24T02:11:42.011817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410751433582:2093] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.012527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410751433559:2078] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.012542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.012548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410751433559:2078] [node 4] ICP22 created new session: [2:7587241415046400904:2078]
2025-12-24T02:11:42.012555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241415046400904:2078] [node 4] ICS09 handshake done sender: [2:7587241410751433570:2086] self: [2:7587241410751433569:0] peer: [4:7587241411713210838:0] socket: 371 qp: -1
2025-12-24T02:11:42.012563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241415046400904:2078] [node 4] ICS10 traffic start
2025-12-24T02:11:42.012575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.013880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208590:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.013914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208569:2092] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.011909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241412852229841:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:42.014445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241410936049727:2074] [node 5] ICP01 ready to work
2025-12-24T02:11:42.017176Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241411713210824:2104] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25872535ns) {Configure:104(25871510ns) {ProcessConfigured:135(25459917ns) {StartInitialHandshake:157(25459322ns) {DropHandshakes:470(1035ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(254ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1308ns) {SwitchToState:179(735ns)}}}}}
2025-12-24T02:11:42.017362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210817:2098] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.017384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411713210760:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.018927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241417147197148:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
2025-12-24T02:11:42.018996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241412852229824:2091] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.019024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241412852229823:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.019499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241412852229776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.020810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208559:2086] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.020823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411203208559:2086] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.021390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241415498175896:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61053
2025-12-24T02:11:42.021477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411203208559:2086] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.021552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411203208520:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.021572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.021577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411203208520:2049] [node 3] ICP22 created new session: [1:7587241415498175898:2049]
2025-12-24T02:11:42.021584Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241415498175898:2049] [node 3] ICS09 handshake done sender: [1:7587241411203208559:2086] self: [1:7587241411203208558:0] peer: [3:7587241417147197154:0] socket: 351 qp: -1
2025-12-24T02:11:42.021589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241415498175898:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.021607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.019807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017040:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.019855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017041:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.018617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241413575864222:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.018785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241413575864148:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.018802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.018811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241413575864148:2049] [node 2] ICP22 created new session: [5:7587241417870831526:2049]
2025-12-24T02:11:42.018817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417870831526:2049] [node 2] ICS09 handshake done sender: [5:7587241413575864222:2112] self: [5:7587241417870831519:0] peer: [2:7587241410751433581:0] socket: 384 qp: -1
2025-12-24T02:11:42.018825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417870831526:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.018836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.022993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210805:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.023011Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241411713210805:2088] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.023162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208566:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.023183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411203208566:2090] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.023126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241417147197147:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
2025-12-24T02:11:42.024792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210835:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.024823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241411713210762:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.024936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241417147197148:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.027467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416008178143:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:42.030402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411713210818:2099] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.030924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241415046400899:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.030958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410751433560:2079] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.031088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410751433580:2092] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.031099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.031106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410751433580:2092] [node 5] ICP22 created new session: [2:7587241415046400909:2092]
2025-12-24T02:11:42.031112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241415046400909:2092] [node 5] ICS09 handshake done sender: [2:7587241410751433582:2093] self: [2:7587241410751433581:0] peer: [5:7587241417870831519:0] socket: 380 qp: -1
2025-12-24T02:11:42.031118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241415046400909:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:42.031125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.030500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411713210761:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.030523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.030531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411713210761:2049] [node 2] ICP22 created new session: [4:7587241416008178144:2049]
2025-12-24T02:11:42.030538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416008178144:2049] [node 2] ICS09 handshake done sender: [4:7587241411713210818:2099] self: [4:7587241411713210838:0] peer: [2:7587241410751433569:0] socket: 374 qp: -1
2025-12-24T02:11:42.030545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416008178144:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.030554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.031334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410751433565:2082] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.031514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241410751433526:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.034840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241417147197149:2099] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24327766ns) {Configure:104(24326820ns) {ProcessConfigured:135(23905583ns) {StartInitialHandshake:157(23904529ns) {DropHandshakes:470(1003ns) {DropIncomingHandshake:442(276ns) DropOutgoingHandshake:457(187ns)} GenerateSessionVirtualId:988(631ns) PrepareNewSessionHandshake:477(1606ns) {SwitchToState:179(1149ns)}}}}}
2025-12-24T02:11:42.034890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241412852229828:2093] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.034896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241412852229828:2093] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.035268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241412852229776:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241412852229824:2091] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.035643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241410936049671:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(21120052ns) {Configure:104(21119113ns) {ProcessConfigured:135(20678167ns) {StartInitialHandshake:157(20677818ns) {DropHandshakes:470(584ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1358ns) {SwitchToState:179(28ns)}}}}}
2025-12-24T02:11:42.035821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241415231017040:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.035842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241410936049671:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.036331Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416008178147:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:42.039364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241412852229828:2093] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.038846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241410751433562:2080] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.038858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241410751433562:2080] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.040941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831529:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.041527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417870831520:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.041530Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241417870831520:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.042181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417870831520:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.041320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241415231017041:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.041361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241410936049727:2074] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.042108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017048:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.042350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241415231017041:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.042390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241410936049727:2074] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.042405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.042412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241410936049727:2074] [node 5] ICP22 created new session: [6:7587241415231017049:2074]
2025-12-24T02:11:42.042419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241415231017049:2074] [node 5] ICS09 handshake done sender: [6:7587241415231017041:2114] self: [6:7587241415231017047:0] peer: [5:7587241413575864190:0] socket: 393 qp: -1
2025-12-24T02:11:42.042425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241415231017049:2074] [node 5] ICS10 traffic start
2025-12-24T02:11:42.042435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.043087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411713210835:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.043409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208568:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.043426Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411203208568:2091] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.046730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241413575864189:2082] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.046752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.046763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241413575864189:2082] [node 6] ICP22 created new session: [5:7587241417870831531:2082]
2025-12-24T02:11:42.046771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417870831531:2082] [node 6] ICS09 handshake done sender: [5:7587241417870831520:2113] self: [5:7587241413575864190:0] peer: [6:7587241415231017047:0] socket: 392 qp: -1
2025-12-24T02:11:42.046779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417870831531:2082] [node 6] ICS10 traffic start
2025-12-24T02:11:42.046789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.047670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411203208566:2090] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.047785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411203208564:2088] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.047801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.047807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411203208564:2088] [node 4] ICP22 created new session: [1:7587241415498175904:2088]
2025-12-24T02:11:42.047814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241415498175904:2088] [node 4] ICS09 handshake done sender: [1:7587241411203208566:2090] self: [1:7587241411203208565:0] peer: [4:7587241416008178141:0] socket: 370 qp: -1
2025-12-24T02:11:42.047820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241415498175904:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:42.047827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.049819Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241417147197147:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.049844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241412852229826:2092] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.049863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.049869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241412852229826:2092] [node 4] ICP22 created new session: [3:7587241417147197161:2092]
2025-12-24T02:11:42.049875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241417147197161:2092] [node 4] ICS09 handshake done sender: [3:7587241412852229828:2093] self: [3:7587241412852229827:0] peer: [4:7587241416008178142:0] socket: 379 qp: -1
2025-12-24T02:11:42.049881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241417147197161:2092] [node 4] ICS10 traffic start
2025-12-24T02:11:42.049890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.050514Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241410751433559:2078] [node 4] ICP35 event processing took too much time PendingConnection:279(37876704ns) {HandleHandshakeStatus:329(37875993ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(300ns) SwitchToState:179(64ns) ProcessPendingSessionEvents:495(26927763ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26902530ns max# 26822084ns}}}
2025-12-24T02:11:42.050671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241410751433559:2078] [node 4] ICP18 handshake (actor [2:7587241415046400899:2113]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.050927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017052:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.051888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241415498175907:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61053
2025-12-24T02:11:42.052354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241415498175908:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61053
2025-12-24T02:11:42.051624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241415046400914:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41576
2025-12-24T02:11:42.052299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410751433562:2080] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.054553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241415231017052:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.054591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241410936049670:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.054814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241417147197159:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
2025-12-24T02:11:42.055033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417870831529:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.055073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241413575864147:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.055160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411713210762:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.055184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.055194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411713210762:2050] [node 3] ICP22 created new session: [4:7587241416008178148:2050]
2025-12-24T02:11:42.055203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416008178148:2050] [node 3] ICS09 handshake done sender: [4:7587241411713210835:2113] self: [4:7587241416008178142:0] peer: [3:7587241412852229827:0] socket: 383 qp: -1
2025-12-24T02:11:42.055211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416008178148:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.055219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.055645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241411713210816:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.056901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241411713210762:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:42.056956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416008178150:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55395
2025-12-24T02:11:42.057416Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411713210817:2098] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.057528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241411713210805:2088] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.057780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241415498175896:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.057846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208592:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.057856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411203208592:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.057599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241411713210760:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.057616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.057622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411713210760:2048] [node 1] ICP22 created new session: [4:7587241416008178153:2048]
2025-12-24T02:11:42.057628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416008178153:2048] [node 1] ICS09 handshake done sender: [4:7587241411713210817:2098] self: [4:7587241416008178141:0] peer: [1:7587241411203208565:0] socket: 373 qp: -1
2025-12-24T02:11:42.057634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416008178153:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.057640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.058002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241411713210803:2087] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.058014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.058018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411713210803:2087] [node 5] ICP22 created new session: [4:7587241416008178155:2087]
2025-12-24T02:11:42.058023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416008178155:2087] [node 5] ICS09 handshake done sender: [4:7587241411713210805:2088] self: [4:7587241411713210804:0] peer: [5:7587241417870831522:0] socket: 353 qp: -1
2025-12-24T02:11:42.058028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416008178155:2087] [node 5] ICS10 traffic start
2025-12-24T02:11:42.058034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.058417Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241411203208575:2094] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.058428Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241411203208575:2094] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.058950Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241415498175908:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.059371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241411203208564:2088] [node 4] ICP18 handshake (actor [1:7587241415498175908:2119]) is too old
2025-12-24T02:11:42.063083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411203208592:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.063208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411203208575:2094] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.063278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411203208583:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.063293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.063300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411203208583:2095] [node 6] ICP22 created new session: [1:7587241415498175912:2095]
2025-12-24T02:11:42.063306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241415498175912:2095] [node 6] ICS09 handshake done sender: [1:7587241411203208592:2109] self: [1:7587241411203208591:0] peer: [6:7587241415231017053:0] socket: 408 qp: -1
2025-12-24T02:11:42.063311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241415498175912:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:42.063320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.063539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411203208572:2093] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.063548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.063552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411203208572:2093] [node 5] ICP22 created new session: [1:7587241415498175914:2093]
2025-12-24T02:11:42.063556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241415498175914:2093] [node 5] ICS09 handshake done sender: [1:7587241411203208575:2094] self: [1:7587241411203208574:0] peer: [5:7587241417870831533:0] socket: 355 qp: -1
2025-12-24T02:11:42.063559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241415498175914:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:42.063563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.064776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241415498175907:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.065120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241411203208568:2091] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.065199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241411203208519:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.065214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.065223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241411203208519:2048] [node 2] ICP22 created new session: [1:7587241415498175918:2048]
2025-12-24T02:11:42.065230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241415498175918:2048] [node 2] ICS09 handshake done sender: [1:7587241411203208568:2091] self: [1:7587241411203208567:0] peer: [2:7587241415046400913:0] socket: 356 qp: -1
2025-12-24T02:11:42.065236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241415498175918:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:42.065243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.063058Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241410751433580:2092] [node 5] ICP35 event processing took too much time PendingConnection:279(31842816ns) {HandleHandshakeStatus:329(31842256ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(404ns) SwitchToState:179(30ns) ProcessPendingSessionEvents:495(19586564ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19564601ns max# 19474398ns}}}
2025-12-24T02:11:42.063427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410751433556:2076] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.063436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.063442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410751433556:2076] [node 6] ICP22 created new session: [2:7587241415046400916:2076]
2025-12-24T02:11:42.063447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241415046400916:2076] [node 6] ICS09 handshake done sender: [2:7587241410751433562:2080] self: [2:7587241410751433561:0] peer: [6:7587241415231017046:0] socket: 354 qp: -1
2025-12-24T02:11:42.063453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241415046400916:2076] [node 6] ICS10 traffic start
2025-12-24T02:11:42.063461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.066969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241410936049670:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241415231017052:2118] held: no
2025-12-24T02:11:42.067351Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241413575864189:2082] [node 6] ICP35 event processing took too much time PendingConnection:279(20530623ns) {HandleHandshakeStatus:329(20529608ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(400ns) SwitchToState:179(69ns) ProcessPendingSessionEvents:495(20408380ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20383761ns max# 20311810ns}}}
2025-12-24T02:11:42.067436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241413575864147:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241417870831529:2118] held: no
2025-12-24T02:11:42.073318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241417147197160:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.074300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.074532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.079495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241417147197160:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.079614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241412852229824:2091] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.079803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241412852229776:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.079819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.079826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241412852229776:2048] [node 1] ICP22 created new session: [3:7587241417147197166:2048]
2025-12-24T02:11:42.079834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241417147197166:2048] [node 1] ICS09 handshake done sender: [3:7587241412852229824:2091] self: [3:7587241417147197154:0] peer: [1:7587241411203208558:0] socket: 361 qp: -1
2025-12-24T02:11:42.079841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241417147197166:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.079850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.080210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241412852229777:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.083553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831535:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.084090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831536:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.084123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831534:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.084444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241410751433526:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32238) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241410751433565:2082] held: no
2025-12-24T02:11:42.091292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241415231017052:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.091707Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241415231017040:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.092276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241410936049670:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.092298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.092306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241410936049670:2048] [node 1] ICP22 created new session: [6:7587241415231017058:2048]
2025-12-24T02:11:42.092312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241415231017058:2048] [node 1] ICS09 handshake done sender: [6:7587241415231017052:2118] self: [6:7587241415231017053:0] peer: [1:7587241411203208591:0] socket: 411 qp: -1
2025-12-24T02:11:42.092321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241415231017058:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.092331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.092570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241410936049671:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.092580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.092584Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241410936049671:2049] [node 2] ICP22 created new session: [6:7587241415231017060:2049]
2025-12-24T02:11:42.092589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241415231017060:2049] [node 2] ICS09 handshake done sender: [6:7587241415231017040:2113] self: [6:7587241415231017046:0] peer: [2:7587241410751433561:0] socket: 359 qp: -1
2025-12-24T02:11:42.092595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241415231017060:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.092600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.092758Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241410936049671:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:42.092947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017056:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.092986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017057:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.091438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241415046400914:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.091484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241410751433556:2076] [node 6] ICP18 handshake (actor [2:7587241415046400914:2118]) is too old
2025-12-24T02:11:42.095941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241415046400903:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.095957Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241415046400903:2115] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.099770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417870831529:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.100125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241415231017056:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.100237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241410936049686:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.099988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241415046400918:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41576
2025-12-24T02:11:42.100592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241413575864178:2078] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.100714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241410751433565:2082] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.100947Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241413575864179:2073] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25560) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241413575864178:2078] held: no
2025-12-24T02:11:42.101050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241413575864179:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.101062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.101068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241413575864179:2073] [node 4] ICP22 created new session: [5:7587241417870831537:2073]
2025-12-24T02:11:42.101077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417870831537:2073] [node 4] ICS09 handshake done sender: [5:7587241413575864178:2078] self: [5:7587241417870831522:0] peer: [4:7587241411713210804:0] socket: 362 qp: -1
2025-12-24T02:11:42.101085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417870831537:2073] [node 4] ICS10 traffic start
2025-12-24T02:11:42.101096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.101320Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417870831536:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.101328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241413575864147:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.101335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.101340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241413575864147:2048] [node 1] ICP22 created new session: [5:7587241417870831539:2048]
2025-12-24T02:11:42.101344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417870831539:2048] [node 1] ICS09 handshake done sender: [5:7587241417870831529:2118] self: [5:7587241417870831533:0] peer: [1:7587241411203208574:0] socket: 397 qp: -1
2025-12-24T02:11:42.101349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417870831539:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.101354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.101685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241413575864149:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.102105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241415231017057:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.102131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241410936049672:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.102024Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241417147197156:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.102037Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241417147197156:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.102232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241410751433526:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.102244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.102250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410751433526:2048] [node 1] ICP22 created new session: [2:7587241415046400919:2048]
2025-12-24T02:11:42.102256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241415046400919:2048] [node 1] ICS09 handshake done sender: [2:7587241410751433565:2082] self: [2:7587241415046400913:0] peer: [1:7587241411203208567:0] socket: 368 qp: -1
2025-12-24T02:11:42.102262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241415046400919:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.102268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.102018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416008178140:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.102032Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241416008178140:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.103299Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416008178140:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.103451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241411713210824:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.103477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.103491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241411713210824:2104] [node 6] ICP22 created new session: [4:7587241416008178159:2104]
2025-12-24T02:11:42.103497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416008178159:2104] [node 6] ICS09 handshake done sender: [4:7587241416008178140:2119] self: [4:7587241411713210828:0] peer: [6:7587241415231017062:0] socket: 395 qp: -1
2025-12-24T02:11:42.103509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416008178159:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:42.103523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.108560Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241415046400903:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.109784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017064:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.109933Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241410936049686:2062] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25560) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241415231017056:2121] held: no
2025-12-24T02:11:42.109983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241415231017056:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.110049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241410936049686:2062] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.110061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.110067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241410936049686:2062] [node 4] ICP22 created new session: [6:7587241415231017065:2062]
2025-12-24T02:11:42.110073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241415231017065:2062] [node 4] ICS09 handshake done sender: [6:7587241415231017056:2121] self: [6:7587241415231017062:0] peer: [4:7587241411713210828:0] socket: 377 qp: -1
2025-12-24T02:11:42.110080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241415231017065:2062] [node 4] ICS10 traffic start
2025-12-24T02:11:42.110089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.110847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241410751433527:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.110865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.110870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241410751433527:2049] [node 3] ICP22 created new session: [2:7587241415046400922:2049]
2025-12-24T02:11:42.110876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241415046400922:2049] [node 3] ICS09 handshake done sender: [2:7587241415046400903:2115] self: [2:7587241410751433571:0] peer: [3:7587241417147197168:0] socket: 400 qp: -1
2025-12-24T02:11:42.110881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241415046400922:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.110887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.109270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241417147197170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46436
2025-12-24T02:11:42.109489Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241417147197153:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.109499Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241417147197153:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.111256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241417147197156:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.112294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241417147197153:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.112431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417870831542:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6224
2025-12-24T02:11:42.112625Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417870831536:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.112374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241412852229841:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.112392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.112398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241412852229841:2096] [node 6] ICP22 created new session: [3:7587241417147197172:2096]
2025-12-24T02:11:42.112404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241417147197172:2096] [node 6] ICS09 handshake done sender: [3:7587241417147197153:2117] self: [3:7587241417147197152:0] peer: [6:7587241415231017063:0] socket: 417 qp: -1
2025-12-24T02:11:42.112409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241417147197172:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:42.112416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.112723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241413575864149:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.112738Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.112744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241413575864149:2050] [node 3] ICP22 created new session: [5:7587241417870831543:2050]
2025-12-24T02:11:42.112749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417870831543:2050] [node 3] ICS09 handshake done sender: [5:7587241417870831536:2121] self: [5:7587241417870831541:0] peer: [3:7587241417147197150:0] socket: 419 qp: -1
2025-12-24T02:11:42.112753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417870831543:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.112760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.114082Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241412852229777:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26786) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241417147197160:2120] held: no
2025-12-24T02:11:42.114222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241417147197160:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.119401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241410936049672:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25781) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241415231017057:2122] held: no
2025-12-24T02:11:42.117641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241417147197149:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.117664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.117670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241417147197149:2099] [node 5] ICP22 created new session: [3:7587241417147197174:2099]
2025-12-24T02:11:42.117676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241417147197174:2099] [node 5] ICS09 handshake done sender: [3:7587241417147197156:2118] self: [3:7587241417147197150:0] peer: [5:7587241417870831541:0] socket: 418 qp: -1
2025-12-24T02:11:42.117682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241417147197174:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:42.117689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.123027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241412852229777:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.123040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.123046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241412852229777:2049] [node 2] ICP22 created new session: [3:7587241417147197176:2049]
2025-12-24T02:11:42.123053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241417147197176:2049] [node 2] ICS09 handshake done sender: [3:7587241417147197160:2120] self: [3:7587241417147197168:0] peer: [2:7587241410751433571:0] socket: 409 qp: -1
2025-12-24T02:11:42.123058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241417147197176:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.123064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.132571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241415231017067:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62281
2025-12-24T02:11:42.133325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241415231017057:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.134606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241410936049672:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.134626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.134634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241410936049672:2050] [node 3] ICP22 created new session: [6:7587241415231017068:2050]
2025-12-24T02:11:42.134640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241415231017068:2050] [node 3] ICS09 handshake done sender: [6:7587241415231017057:2122] self: [6:7587241415231017063:0] peer: [3:7587241417147197152:0] socket: 399 qp: -1
2025-12-24T02:11:42.134648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241415231017068:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.134661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.146207Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:42.146314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241415046400919:2048] [node 1] ICS01 socket: 368 reason# EndOfStream
2025-12-24T02:11:42.146323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241410751433526:2048] [node 1] ICP30 unregister session Session# [2:7587241415046400919:2048] VirtualId# [2:7587241415046400913:0]
2025-12-24T02:11:42.146333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241415046400919:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.146362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.146383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.150546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:42.150637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241417147197166:2048] [node 1] ICS01 socket: 361 reason# EndOfStream
2025-12-24T02:11:42.150645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241412852229776:2048] [node 1] ICP30 unregister session Session# [3:7587241417147197166:2048] VirtualId# [3:7587241417147197154:0]
2025-12-24T02:11:42.150652Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241417147197166:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.150713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.150734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.152057Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:42.152138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241415231017058:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-12-24T02:11:42.152144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241410936049670:2048] [node 1] ICP30 unregister session Session# [6:7587241415231017058:2048] VirtualId# [6:7587241415231017053:0]
2025-12-24T02:11:42.152150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241415231017058:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.152183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.152203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.153176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:42.153266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241416008178153:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:42.153279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241411713210760:2048] [node 1] ICP30 unregister session Session# [4:7587241416008178153:2048] VirtualId# [4:7587241416008178141:0]
2025-12-24T02:11:42.153287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241416008178153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.153346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.153385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.153172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:42.153265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417870831539:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-12-24T02:11:42.153272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241413575864147:2048] [node 1] ICP30 unregister session Session# [5:7587241417870831539:2048] VirtualId# [5:7587241417870831533:0]
2025-12-24T02:11:42.153279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417870831539:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.153332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.153357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.165391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:42.165507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241417147197176:2049] [node 2] ICS01 socket: 409 reason# EndOfStream
2025-12-24T02:11:42.165518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241412852229777:2049] [node 2] ICP30 unregister session Session# [3:7587241417147197176:2049] VirtualId# [3:7587241417147197168:0]
2025-12-24T02:11:42.165527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241417147197176:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.165563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.165587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.170842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:42.171741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241415231017060:2049] [node 2] ICS01 socket: 359 reason# EndOfStream
2025-12-24T02:11:42.171768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241410936049671:2049] [node 2] ICP30 unregister session Session# [6:7587241415231017060:2049] VirtualId# [6:7587241415231017046:0]
2025-12-24T02:11:42.171773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241415231017060:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.171802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.171830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.171089Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:42.171196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417870831526:2049] [node 2] ICS01 socket: 384 reason# EndOfStream
2025-12-24T02:11:42.171201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241413575864148:2049] [node 2] ICP30 unregister session Session# [5:7587241417870831526:2049] VirtualId# [5:7587241417870831519:0]
2025-12-24T02:11:42.171206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417870831526:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.171243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.171265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.170843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:42.170945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241416008178144:2049] [node 2] ICS01 socket: 374 reason# EndOfStream
2025-12-24T02:11:42.170956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241411713210761:2049] [node 2] ICP30 unregister session Session# [4:7587241416008178144:2049] VirtualId# [4:7587241411713210838:0]
2025-12-24T02:11:42.170964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241416008178144:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.171018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.171045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.203002Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:42.203118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241415231017068:2050] [node 3] ICS01 socket: 399 reason# EndOfStream
2025-12-24T02:11:42.203128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241410936049672:2050] [node 3] ICP30 unregister session Session# [6:7587241415231017068:2050] VirtualId# [6:7587241415231017063:0]
2025-12-24T02:11:42.203137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241415231017068:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.203193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.203220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.203978Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:42.204051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417870831543:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:42.204056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241413575864149:2050] [node 3] ICP30 unregister session Session# [5:7587241417870831543:2050] VirtualId# [5:7587241417870831541:0]
2025-12-24T02:11:42.204062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417870831543:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.204088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.204111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.218766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:42.218877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241415231017065:2062] [node 4] ICS01 socket: 377 reason# EndOfStream
2025-12-24T02:11:42.218888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241410936049686:2062] [node 4] ICP30 unregister session Session# [6:7587241415231017065:2062] VirtualId# [6:7587241415231017062:0]
2025-12-24T02:11:42.218897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241415231017065:2062] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.218947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:42.218974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:42.273978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241416039479337:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.273987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241416039479338:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.275801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414417293275:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.278783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414417293274:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:42.275116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241417139410195:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.275129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241417139410196:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.279062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417941079261:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.279070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417941079262:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.279074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417941079263:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.290807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414384011688:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.290830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414384011689:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.298711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414384011687:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.310862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417629731291:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.310882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417629731292:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.310888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417629731293:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.313123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241417139410224:2074] [node 5] ICP01 ready to work
2025-12-24T02:11:42.313410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241417139410227:2077] [node 4] ICP01 ready to work
2025-12-24T02:11:42.313354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414417293308:2076] [node 4] ICP01 ready to work
2025-12-24T02:11:42.313475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414417293309:2079] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.315461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414417293316:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44866
2025-12-24T02:11:42.315559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241416039479355:2061] [node 6] ICP01 ready to work
2025-12-24T02:11:42.315983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417629731304:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:42.316286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241416039479365:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:42.316384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241416039479367:2067] [node 5] ICP01 ready to work
2025-12-24T02:11:42.315946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414417293324:2088] [node 5] ICP01 ready to work
2025-12-24T02:11:42.316226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417941079278:2061] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.326687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241417139410248:2080] [node 6] ICP01 ready to work
2025-12-24T02:11:42.326778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410247:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
2025-12-24T02:11:42.330774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417629731335:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:42.332313Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241417139410196:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19099061ns) {Configure:104(19097483ns) {ProcessConfigured:135(17592891ns) {StartInitialHandshake:157(17592304ns) {DropHandshakes:470(121ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(28ns)} GenerateSessionVirtualId:988(92ns) PrepareNewSessionHandshake:477(600ns) {SwitchToState:179(238ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.335038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414384011712:2067] [node 5] ICP01 ready to work
2025-12-24T02:11:42.333899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479394:2095] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.333922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479395:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.339189Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417941079278:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22899560ns) {Configure:104(22899205ns) {ProcessConfigured:135(22641140ns) {StartInitialHandshake:157(22640970ns) {DropHandshakes:470(265ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:477(1721ns) {SwitchToState:179(1100ns)}}}}}
2025-12-24T02:11:42.340010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079330:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.340035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079331:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.342166Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417629731304:2059] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26112393ns) {Configure:104(26111951ns) {ProcessConfigured:135(25769836ns) {StartInitialHandshake:157(25769638ns) {DropHandshakes:470(261ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(908ns) {SwitchToState:179(548ns)}}}}}
2025-12-24T02:11:42.342865Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241416039479365:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26424106ns) {Configure:104(26423698ns) {ProcessConfigured:135(26149988ns) {StartInitialHandshake:157(26149767ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(784ns) {SwitchToState:179(284ns)}}}}}
2025-12-24T02:11:42.343937Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241414384011688:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26006380ns) {Configure:104(26005991ns) {ProcessConfigured:135(25596861ns) {StartInitialHandshake:157(25596741ns) {DropHandshakes:470(218ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1179ns) {SwitchToState:179(497ns)}}}}}
2025-12-24T02:11:42.338829Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241414417293308:2076] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23926204ns) {Configure:104(23924016ns) {ProcessConfigured:135(23630504ns) {StartInitialHandshake:157(23629897ns) {DropHandshakes:470(422ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1590ns) {SwitchToState:179(764ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.346231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293316:2084] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.343470Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417941079262:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27709679ns) {Configure:104(27708882ns) {ProcessConfigured:135(27290293ns) {StartInitialHandshake:157(27289895ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(505ns) PrepareNewSessionHandshake:477(951ns) {SwitchToState:179(300ns)}}}}}
2025-12-24T02:11:42.343741Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417941079263:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(26615918ns) {Configure:104(26614595ns) {ProcessConfigured:135(26356555ns) {StartInitialHandshake:157(26356228ns) {DropHandshakes:470(541ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(738ns) {SwitchToState:179(339ns)}}}}}
2025-12-24T02:11:42.346126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079339:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.346742Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417629731292:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27920615ns) {Configure:104(27919531ns) {ProcessConfigured:135(27624892ns) {StartInitialHandshake:157(27624263ns) {DropHandshakes:470(432ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(864ns) {SwitchToState:179(332ns)}}}}}
2025-12-24T02:11:42.347236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417941079293:2066] [node 6] ICP01 ready to work
2025-12-24T02:11:42.347621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414384011721:2069] [node 6] ICP01 ready to work
2025-12-24T02:11:42.350490Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241414417293309:2079] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28220994ns) {Configure:104(28220243ns) {ProcessConfigured:135(27940988ns) {StartInitialHandshake:157(27940445ns) {DropHandshakes:470(230ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(877ns) {SwitchToState:179(369ns)}}}}}
2025-12-24T02:11:42.354321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079331:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.363152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479395:2096] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.365361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011764:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
No RDMA context found for GID: 2025-12-24T02:11:42.365436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011765:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.365449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011766:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.365545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241414384011689:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(43007880ns) {Configure:104(43007313ns) {ProcessConfigured:135(42721545ns) {StartInitialHandshake:157(42720680ns) {DropHandshakes:470(324ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(946ns)}}}}}
2025-12-24T02:11:42.365629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011764:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.365766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241416039479338:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51987562ns) {Configure:104(51986953ns) {ProcessConfigured:135(51713547ns) {StartInitialHandshake:157(51713398ns) {DropHandshakes:470(442ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(1111ns) {SwitchToState:179(541ns)}}}}}
2025-12-24T02:11:42.365778Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241416039479367:2067] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49107417ns) {Configure:104(30302748ns) {ProcessConfigured:135(29791101ns) {StartInitialHandshake:157(29790791ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(638ns) {SwitchToState:179(473ns)}}}}}
2025-12-24T02:11:42.365990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417629731291:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(47042948ns) {Configure:104(27687968ns) {ProcessConfigured:135(27363893ns) {StartInitialHandshake:157(27363659ns) {DropHandshakes:470(456ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(362ns) PrepareNewSessionHandshake:477(1068ns) {SwitchToState:179(179ns)}}}}}
2025-12-24T02:11:42.367094Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011763:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.371023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011763:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.371070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414384011688:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.371894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011770:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.372211Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241414384011763:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.372291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414384011688:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.372314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.372321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414384011688:2049] [node 2] ICP22 created new session: [4:7587241414384011771:2049]
2025-12-24T02:11:42.372328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241414384011771:2049] [node 2] ICS09 handshake done sender: [4:7587241414384011763:2113] self: [4:7587241414384011769:0] peer: [2:7587241417139410231:0] socket: 372 qp: -1
2025-12-24T02:11:42.372337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241414384011771:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.372357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.371227Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410246:2092] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.371242Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241417139410246:2092] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.372016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241417139410246:2092] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.372973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241417139410227:2077] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.372981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.372986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241417139410227:2077] [node 4] ICP22 created new session: [2:7587241417139410276:2077]
2025-12-24T02:11:42.372991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241417139410276:2077] [node 4] ICS09 handshake done sender: [2:7587241417139410246:2092] self: [2:7587241417139410231:0] peer: [4:7587241414384011769:0] socket: 363 qp: -1
2025-12-24T02:11:42.372996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241417139410276:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:42.373003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.373143Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417941079293:2066] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25755562ns) {Configure:104(25754751ns) {ProcessConfigured:135(25391568ns) {StartInitialHandshake:157(25390113ns) {DropHandshakes:470(630ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(260ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1684ns) {SwitchToState:179(1070ns)}}}}}
2025-12-24T02:11:42.374021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.375341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479419:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.375929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011765:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.375961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414384011689:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.376075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479407:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.376081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241416039479407:2107] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.376611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011775:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.376721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241416039479407:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.376889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241414384011765:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.376964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414384011689:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.376981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.376987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414384011689:2050] [node 3] ICP22 created new session: [4:7587241414384011776:2050]
2025-12-24T02:11:42.376992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241414384011776:2050] [node 3] ICS09 handshake done sender: [4:7587241414384011765:2115] self: [4:7587241414384011774:0] peer: [3:7587241416039479369:0] socket: 390 qp: -1
2025-12-24T02:11:42.376998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241414384011776:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.377005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.377763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241416039479365:2064] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.377776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.377782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241416039479365:2064] [node 4] ICP22 created new session: [3:7587241416039479421:2064]
2025-12-24T02:11:42.377786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241416039479421:2064] [node 4] ICS09 handshake done sender: [3:7587241416039479407:2107] self: [3:7587241416039479369:0] peer: [4:7587241414384011774:0] socket: 380 qp: -1
2025-12-24T02:11:42.377791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241416039479421:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:42.377796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.378032Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241414384011721:2069] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(30268422ns) {Configure:104(30267727ns) {ProcessConfigured:135(29896275ns) {StartInitialHandshake:157(29895272ns) {DropHandshakes:470(495ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(1273ns) {SwitchToState:179(367ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.379219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079343:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.379259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417941079262:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.379239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414417293357:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44866
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.379301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414417293356:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44866
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.382848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410255:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.382860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241417139410255:2096] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.382867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410279:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
2025-12-24T02:11:42.383130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410281:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.385536Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410279:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.385608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241417139410255:2096] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.383990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079345:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.390718Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414417293355:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44866
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.391182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731369:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.391223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731370:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.391237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731371:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.390758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410280:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
2025-12-24T02:11:42.390778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410281:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.391652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417941079343:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.391996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417941079262:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.392015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.392022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417941079262:2049] [node 2] ICP22 created new session: [5:7587241417941079346:2049]
2025-12-24T02:11:42.392029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417941079346:2049] [node 2] ICS09 handshake done sender: [5:7587241417941079343:2118] self: [5:7587241417941079344:0] peer: [2:7587241417139410233:0] socket: 383 qp: -1
2025-12-24T02:11:42.392037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417941079346:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.392047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.394028Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417629731370:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417629731292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.393554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241417139410224:2074] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.393578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.393586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241417139410224:2074] [node 5] ICP22 created new session: [2:7587241417139410285:2074]
2025-12-24T02:11:42.393592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241417139410285:2074] [node 5] ICS09 handshake done sender: [2:7587241417139410255:2096] self: [2:7587241417139410233:0] peer: [5:7587241417941079344:0] socket: 395 qp: -1
2025-12-24T02:11:42.393598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241417139410285:2074] [node 5] ICS10 traffic start
2025-12-24T02:11:42.393607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.391359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011766:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394400Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479424:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.394354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417629731369:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417629731293:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.394752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293356:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293355:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410250:2094] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241417139410250:2094] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.394817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479424:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394911Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241416039479365:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(17033823ns) {HandleHandshakeStatus:329(17033009ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(127ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(7817025ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7785388ns max# 7707428ns}}}
2025-12-24T02:11:42.394913Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479357:2062] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.394917Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241416039479357:2062] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.395838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241416039479357:2062] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:42.395890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241416039479355:2061] [node 6] ICP20 outgoing handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:42.395907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.395913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241416039479355:2061] [node 6] ICP22 created new session: [3:7587241416039479428:2061]
2025-12-24T02:11:42.395919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241416039479428:2061] [node 6] ICS09 handshake done sender: [3:7587241416039479357:2062] self: [3:7587241416039479356:0] peer: [6:7587241417629731373:0] socket: 359 qp: -1
2025-12-24T02:11:42.395924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241416039479428:2061] [node 6] ICS10 traffic start
2025-12-24T02:11:42.395939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.396251Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079349:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.395794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731374:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.398654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410288:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
2025-12-24T02:11:42.398825Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241417139410250:2094] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.396986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241417629731293:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241417629731369:2115] held: no
2025-12-24T02:11:42.397117Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417629731369:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.397304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417629731293:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.397312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.397317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417629731293:2050] [node 3] ICP22 created new session: [6:7587241417629731375:2050]
2025-12-24T02:11:42.397322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417629731375:2050] [node 3] ICS09 handshake done sender: [6:7587241417629731369:2115] self: [6:7587241417629731373:0] peer: [3:7587241416039479356:0] socket: 373 qp: -1
2025-12-24T02:11:42.397327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417629731375:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.397342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.398903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731378:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.399265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479425:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.399637Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479425:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.399669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241416039479338:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.400417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479432:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.400806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241416039479425:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.400858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241416039479338:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.400871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.400876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241416039479338:2049] [node 2] ICP22 created new session: [3:7587241416039479433:2049]
2025-12-24T02:11:42.400882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241416039479433:2049] [node 2] ICS09 handshake done sender: [3:7587241416039479425:2119] self: [3:7587241416039479431:0] peer: [2:7587241417139410234:0] socket: 381 qp: -1
2025-12-24T02:11:42.400887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241416039479433:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.400894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.399052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241417139410248:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.399064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.399071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241417139410248:2080] [node 6] ICP22 created new session: [2:7587241417139410290:2080]
2025-12-24T02:11:42.399077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241417139410290:2080] [node 6] ICS09 handshake done sender: [2:7587241417139410250:2094] self: [2:7587241417139410249:0] peer: [6:7587241417629731372:0] socket: 364 qp: -1
2025-12-24T02:11:42.399081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241417139410290:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:42.399088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.399774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410260:2101] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.399777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241417139410260:2101] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.400650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241417139410260:2101] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.401990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241417139410196:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.402008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.402013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241417139410196:2049] [node 3] ICP22 created new session: [2:7587241417139410293:2049]
2025-12-24T02:11:42.402019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241417139410293:2049] [node 3] ICS09 handshake done sender: [2:7587241417139410260:2101] self: [2:7587241417139410234:0] peer: [3:7587241416039479431:0] socket: 408 qp: -1
2025-12-24T02:11:42.402024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241417139410293:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.402030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.399375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417629731370:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.402664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417629731292:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.402680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.402686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417629731292:2049] [node 2] ICP22 created new session: [6:7587241417629731379:2049]
2025-12-24T02:11:42.402704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417629731379:2049] [node 2] ICS09 handshake done sender: [6:7587241417629731370:2116] self: [6:7587241417629731372:0] peer: [2:7587241417139410249:0] socket: 405 qp: -1
2025-12-24T02:11:42.402709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417629731379:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.402717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.405760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079349:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.405824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417941079263:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.409000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414417293360:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44866
2025-12-24T02:11:42.409225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293360:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.412338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479418:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.412352Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241416039479418:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.413066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241416039479418:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.413245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241416039479367:2067] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.413264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.413271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241416039479367:2067] [node 5] ICP22 created new session: [3:7587241416039479436:2067]
2025-12-24T02:11:42.413279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241416039479436:2067] [node 5] ICS09 handshake done sender: [3:7587241416039479418:2116] self: [3:7587241416039479401:0] peer: [5:7587241417941079350:0] socket: 370 qp: -1
2025-12-24T02:11:42.413286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241416039479436:2067] [node 5] ICS10 traffic start
2025-12-24T02:11:42.413297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.412129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241417139410248:2080] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:42.414678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241416039479394:2095] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.414873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417629731371:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.414917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417629731304:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.415059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241417941079263:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13292) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241417941079349:2120] held: no
2025-12-24T02:11:42.415262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241417139410247:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241417139410195:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.415288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.415412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241416039479337:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.415701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079339:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079330:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417941079278:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.415784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417941079261:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.415938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293328:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414417293328:2091] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.416034Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417941079349:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.416082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417941079263:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.416094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.416099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417941079263:2050] [node 3] ICP22 created new session: [5:7587241417941079354:2050]
2025-12-24T02:11:42.416105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417941079354:2050] [node 3] ICS09 handshake done sender: [5:7587241417941079349:2120] self: [5:7587241417941079350:0] peer: [3:7587241416039479401:0] socket: 410 qp: -1
2025-12-24T02:11:42.416109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417941079354:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.416117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.416452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731383:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.416668Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417629731371:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.416766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417629731304:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.416781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.416785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417629731304:2059] [node 4] ICP22 created new session: [6:7587241417629731384:2059]
2025-12-24T02:11:42.416791Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417629731384:2059] [node 4] ICS09 handshake done sender: [6:7587241417629731371:2117] self: [6:7587241417629731382:0] peer: [4:7587241414384011762:0] socket: 406 qp: -1
2025-12-24T02:11:42.416795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417629731384:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:42.416805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.415503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011778:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415515Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241414384011778:2120] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.415933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011758:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.415936Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241414384011758:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.416336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241414384011778:2120] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.421036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241414384011758:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.421119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241414384011721:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.421137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.421142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414384011721:2069] [node 6] ICP22 created new session: [4:7587241414384011783:2069]
2025-12-24T02:11:42.421148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241414384011783:2069] [node 6] ICS09 handshake done sender: [4:7587241414384011778:2120] self: [4:7587241414384011762:0] peer: [6:7587241417629731382:0] socket: 400 qp: -1
2025-12-24T02:11:42.421154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241414384011783:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:42.421162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.422249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241414384011712:2067] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.422263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.422268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414384011712:2067] [node 5] ICP22 created new session: [4:7587241414384011785:2067]
2025-12-24T02:11:42.422272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241414384011785:2067] [node 5] ICS09 handshake done sender: [4:7587241414384011758:2112] self: [4:7587241414384011757:0] peer: [5:7587241417941079352:0] socket: 384 qp: -1
2025-12-24T02:11:42.422277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241414384011785:2067] [node 5] ICS10 traffic start
2025-12-24T02:11:42.422284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.422937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293314:2083] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.422952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414417293314:2083] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.422998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293319:2085] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.423004Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414417293319:2085] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.424340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293357:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.425073Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241417941079278:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241417941079339:2116] held: no
2025-12-24T02:11:42.425429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414417293328:2091] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.425641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414417293324:2088] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.425655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.425661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414417293324:2088] [node 5] ICP22 created new session: [1:7587241414417293365:2088]
2025-12-24T02:11:42.425666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414417293365:2088] [node 5] ICS09 handshake done sender: [1:7587241414417293328:2091] self: [1:7587241414417293327:0] peer: [5:7587241417941079353:0] socket: 360 qp: -1
2025-12-24T02:11:42.425670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414417293365:2088] [node 5] ICS10 traffic start
2025-12-24T02:11:42.425696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.426159Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731386:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.429438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079357:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.429579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417941079339:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.429660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414417293314:2083] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.429723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417941079358:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12374
2025-12-24T02:11:42.429794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414417293319:2085] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.429283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241416039479439:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60467
2025-12-24T02:11:42.429928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241416039479394:2095] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.430001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241416039479337:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.430015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.430021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241416039479337:2048] [node 1] ICP22 created new session: [3:7587241416039479440:2048]
2025-12-24T02:11:42.430027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241416039479440:2048] [node 1] ICS09 handshake done sender: [3:7587241416039479394:2095] self: [3:7587241416039479438:0] peer: [1:7587241414417293318:0] socket: 369 qp: -1
2025-12-24T02:11:42.430032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241416039479440:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.430040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.430945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241417139410296:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16161
2025-12-24T02:11:42.431194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417941079330:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.431213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241417139410247:2093] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.431293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414417293274:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.431319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.431324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414417293274:2048] [node 2] ICP22 created new session: [1:7587241414417293369:2048]
2025-12-24T02:11:42.431331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414417293369:2048] [node 2] ICS09 handshake done sender: [1:7587241414417293314:2083] self: [1:7587241414417293312:0] peer: [2:7587241417139410295:0] socket: 357 qp: -1
2025-12-24T02:11:42.431336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414417293369:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:42.431342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.431724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417941079278:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.431737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.431742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417941079278:2061] [node 4] ICP22 created new session: [5:7587241417941079359:2061]
2025-12-24T02:11:42.431757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417941079359:2061] [node 4] ICS09 handshake done sender: [5:7587241417941079339:2116] self: [5:7587241417941079352:0] peer: [4:7587241414384011757:0] socket: 385 qp: -1
2025-12-24T02:11:42.431762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417941079359:2061] [node 4] ICS10 traffic start
2025-12-24T02:11:42.431768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.432181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417629731386:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.432231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417629731335:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.431327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241417139410195:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.431344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.431349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241417139410195:2048] [node 1] ICP22 created new session: [2:7587241417139410297:2048]
2025-12-24T02:11:42.431355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241417139410297:2048] [node 1] ICS09 handshake done sender: [2:7587241417139410247:2093] self: [2:7587241417139410295:0] peer: [1:7587241414417293312:0] socket: 362 qp: -1
2025-12-24T02:11:42.431360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241417139410297:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.431369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.432525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417941079341:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.432532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241417941079341:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.432980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417941079261:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.432991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.432996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417941079261:2048] [node 1] ICP22 created new session: [5:7587241417941079361:2048]
2025-12-24T02:11:42.433000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417941079361:2048] [node 1] ICS09 handshake done sender: [5:7587241417941079330:2111] self: [5:7587241417941079353:0] peer: [1:7587241414417293327:0] socket: 374 qp: -1
2025-12-24T02:11:42.433006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417941079361:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.433012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.434182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414417293275:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.434198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.434203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414417293275:2049] [node 3] ICP22 created new session: [1:7587241414417293371:2049]
2025-12-24T02:11:42.434208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414417293371:2049] [node 3] ICS09 handshake done sender: [1:7587241414417293319:2085] self: [1:7587241414417293318:0] peer: [3:7587241416039479438:0] socket: 358 qp: -1
2025-12-24T02:11:42.434212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414417293371:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.434217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.435536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731388:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.435696Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417941079341:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.435847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417629731386:2121] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.436337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731389:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.436579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417629731335:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.436595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.436600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417629731335:2068] [node 5] ICP22 created new session: [6:7587241417629731390:2068]
2025-12-24T02:11:42.436604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417629731390:2068] [node 5] ICS09 handshake done sender: [6:7587241417629731386:2121] self: [6:7587241417629731387:0] peer: [5:7587241417941079340:0] socket: 418 qp: -1
2025-12-24T02:11:42.436609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417629731390:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:42.436617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.435023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011787:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.436468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241414384011787:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.436499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414384011687:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.437569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241417941079293:2066] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.437586Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.437592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417941079293:2066] [node 6] ICP22 created new session: [5:7587241417941079364:2066]
2025-12-24T02:11:42.437596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417941079364:2066] [node 6] ICS09 handshake done sender: [5:7587241417941079341:2117] self: [5:7587241417941079340:0] peer: [6:7587241417629731387:0] socket: 416 qp: -1
2025-12-24T02:11:42.437601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417941079364:2066] [node 6] ICS10 traffic start
2025-12-24T02:11:42.437608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.438756Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293351:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.438765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414417293351:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.439865Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414417293351:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.441100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414417293308:2076] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.441115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.441120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414417293308:2076] [node 4] ICP22 created new session: [1:7587241414417293375:2076]
2025-12-24T02:11:42.441125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414417293375:2076] [node 4] ICS09 handshake done sender: [1:7587241414417293351:2113] self: [1:7587241414417293313:0] peer: [4:7587241414384011788:0] socket: 417 qp: -1
2025-12-24T02:11:42.441128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414417293375:2076] [node 4] ICS10 traffic start
2025-12-24T02:11:42.441134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.441618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417629731389:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.441653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417629731291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.439389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241414384011789:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33355
2025-12-24T02:11:42.440240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241414384011787:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.440307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414384011687:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.440319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.440325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414384011687:2048] [node 1] ICP22 created new session: [4:7587241414384011790:2048]
2025-12-24T02:11:42.440330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241414384011790:2048] [node 1] ICS09 handshake done sender: [4:7587241414384011787:2121] self: [4:7587241414384011788:0] peer: [1:7587241414417293313:0] socket: 365 qp: -1
2025-12-24T02:11:42.440336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241414384011790:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.440343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.442881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414417293352:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.442891Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414417293352:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.446867Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414417293352:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.446963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414417293309:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.446993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.447002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414417293309:2079] [node 6] ICP22 created new session: [1:7587241414417293378:2079]
2025-12-24T02:11:42.447009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414417293378:2079] [node 6] ICS09 handshake done sender: [1:7587241414417293352:2114] self: [1:7587241414417293315:0] peer: [6:7587241417629731392:0] socket: 371 qp: -1
2025-12-24T02:11:42.447016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414417293378:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:42.447023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.448343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241417629731291:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17071) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241417629731389:2123] held: no
2025-12-24T02:11:42.448546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417629731393:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52840
2025-12-24T02:11:42.448806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417629731389:2123] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.448886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417629731291:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.448902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.448907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417629731291:2048] [node 1] ICP22 created new session: [6:7587241417629731394:2048]
2025-12-24T02:11:42.448914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417629731394:2048] [node 1] ICS09 handshake done sender: [6:7587241417629731389:2123] self: [6:7587241417629731392:0] peer: [1:7587241414417293315:0] socket: 424 qp: -1
2025-12-24T02:11:42.448921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417629731394:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.448944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.460852Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:42.460952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241417139410297:2048] [node 1] ICS07 socket disconnect 362 reason# EndOfStream
2025-12-24T02:11:42.460959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241417139410297:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.460997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.461012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241417139410297:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:42.462813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:42.462916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241416039479440:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-12-24T02:11:42.462925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241416039479337:2048] [node 1] ICP30 unregister session Session# [3:7587241416039479440:2048] VirtualId# [3:7587241416039479438:0]
2025-12-24T02:11:42.462933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241416039479440:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.462968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.462994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.467388Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241417139410195:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17071) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:42.467405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241417139410297:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:42.468142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241417139410195:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17071) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:42.468154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241417139410297:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:42.468159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241417139410195:2048] [node 1] ICP30 unregister session Session# [2:7587241417139410297:2048] VirtualId# [2:7587241417139410295:0]
2025-12-24T02:11:42.468213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.468221Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241417139410195:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17071) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:42.468224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17071) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:42.470621Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:42.470722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417629731394:2048] [node 1] ICS01 socket: 424 reason# EndOfStream
2025-12-24T02:11:42.470734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417629731291:2048] [node 1] ICP30 unregister session Session# [6:7587241417629731394:2048] VirtualId# [6:7587241417629731392:0]
2025-12-24T02:11:42.470741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417629731394:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.467655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:42.467788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241414384011790:2048] [node 1] ICS01 socket: 365 reason# EndOfStream
2025-12-24T02:11:42.467796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241414384011687:2048] [node 1] ICP30 unregister session Session# [4:7587241414384011790:2048] VirtualId# [4:7587241414384011788:0]
2025-12-24T02:11:42.467804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241414384011790:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.467844Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.467868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.470914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:42.470974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417941079361:2048] [node 1] ICS01 socket: 374 reason# EndOfStream
2025-12-24T02:11:42.470978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417941079261:2048] [node 1] ICP30 unregister session Session# [5:7587241417941079361:2048] VirtualId# [5:7587241417941079353:0]
2025-12-24T02:11:42.470983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417941079361:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.471012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.471031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.470788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.470811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.486944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:42.487060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241416039479433:2049] [node 2] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:42.487071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241416039479338:2049] [node 2] ICP30 unregister session Session# [3:7587241416039479433:2049] VirtualId# [3:7587241416039479431:0]
2025-12-24T02:11:42.487078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241416039479433:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.487113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.487133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.498400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:42.498506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241414384011771:2049] [node 2] ICS01 socket: 372 reason# EndOfStream
2025-12-24T02:11:42.498518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241414384011688:2049] [node 2] ICP30 unregister session Session# [4:7587241414384011771:2049] VirtualId# [4:7587241414384011769:0]
2025-12-24T02:11:42.498527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241414384011771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.498598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.498626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.499246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:42.499322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417941079346:2049] [node 2] ICS01 socket: 383 reason# EndOfStream
2025-12-24T02:11:42.499330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417941079262:2049] [node 2] ICP30 unregister session Session# [5:7587241417941079346:2049] VirtualId# [5:7587241417941079344:0]
2025-12-24T02:11:42.499335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417941079346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.499385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.499408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.506636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:42.506755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417629731379:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:42.506772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417629731292:2049] [node 2] ICP30 unregister session Session# [6:7587241417629731379:2049] VirtualId# [6:7587241417629731372:0]
2025-12-24T02:11:42.506779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417629731379:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.506833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.506861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.524747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:42.524857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417629731375:2050] [node 3] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:42.524868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417629731293:2050] [node 3] ICP30 unregister session Session# [6:7587241417629731375:2050] VirtualId# [6:7587241417629731373:0]
2025-12-24T02:11:42.524876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417629731375:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.524919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.524944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.524982Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:42.525044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241414384011776:2050] [node 3] ICS01 socket: 390 reason# EndOfStream
2025-12-24T02:11:42.525048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241414384011689:2050] [node 3] ICP30 unregister session Session# [4:7587241414384011776:2050] VirtualId# [4:7587241414384011774:0]
2025-12-24T02:11:42.525055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241414384011776:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.525078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.525096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.525658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:42.525732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417941079354:2050] [node 3] ICS01 socket: 410 reason# EndOfStream
2025-12-24T02:11:42.525739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417941079263:2050] [node 3] ICP30 unregister session Session# [5:7587241417941079354:2050] VirtualId# [5:7587241417941079350:0]
2025-12-24T02:11:42.525744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417941079354:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.525787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.525809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.555082Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:42.555184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417941079359:2061] [node 4] ICS01 socket: 385 reason# EndOfStream
2025-12-24T02:11:42.555195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417941079278:2061] [node 4] ICP30 unregister session Session# [5:7587241417941079359:2061] VirtualId# [5:7587241417941079352:0]
2025-12-24T02:11:42.555212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417941079359:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.555248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:42.555269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:42.555303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:42.555377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417629731384:2059] [node 4] ICS01 socket: 406 reason# EndOfStream
2025-12-24T02:11:42.555382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417629731304:2059] [node 4] ICP30 unregister session Session# [6:7587241417629731384:2059] VirtualId# [6:7587241417629731382:0]
2025-12-24T02:11:42.555387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417629731384:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.555408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:42.555425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:42.579949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:42.580059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417629731390:2068] [node 5] ICS01 socket: 418 reason# EndOfStream
2025-12-24T02:11:42.580070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417629731335:2068] [node 5] ICP30 unregister session Session# [6:7587241417629731390:2068] VirtualId# [6:7587241417629731387:0]
2025-12-24T02:11:42.580078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417629731390:2068] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.580128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:42.580151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:42.618571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414482598076:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:42.618578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414482598077:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.618798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241414558511652:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.618806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241414558511653:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.622949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414378054123:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.622966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414378054124:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.626297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241416087510338:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.626317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241416087510339:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.626322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241416087510340:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.630800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417764034831:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.630818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417764034832:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.630824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417764034833:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.646391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241416087510376:2084] [node 5] ICP01 ready to work
2025-12-24T02:11:42.646896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241416087510381:2087] [node 6] ICP01 ready to work
2025-12-24T02:11:42.646817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241414558511697:2091] [node 6] ICP01 ready to work
2025-12-24T02:11:42.647134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414482598123:2093] [node 5] ICP01 ready to work
2025-12-24T02:11:42.651006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417764034860:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:42.651322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414378054184:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:42.655180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241414558511711:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:42.654311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414482598129:2094] [node 4] ICP01 ready to work
2025-12-24T02:11:42.655229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414378054186:2105] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.660411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510402:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.663017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241416087510339:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17393258ns) {Configure:104(17389288ns) {ProcessConfigured:135(30876ns) {StartInitialHandshake:157(30258ns) {DropHandshakes:470(579ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(331ns) {SwitchToState:179(112ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.664931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511725:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.664312Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241416087510338:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17283968ns) {Configure:104(17276817ns) {ProcessConfigured:135(25169ns) {StartInitialHandshake:157(24322ns) {DropHandshakes:470(386ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(392ns) {SwitchToState:179(177ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.669955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510402:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.670073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241416087510338:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.670097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510413:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.669621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414378054184:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18225696ns) {Configure:104(18225165ns) {ProcessConfigured:135(17932846ns) {StartInitialHandshake:157(17932374ns) {DropHandshakes:470(373ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(1074ns) {SwitchToState:179(663ns)}}}}}
2025-12-24T02:11:42.669920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054194:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
2025-12-24T02:11:42.669934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054195:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
2025-12-24T02:11:42.670868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598132:2097] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.670882Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414482598132:2097] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.672177Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598147:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.672188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414482598147:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.670636Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511725:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.670674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241414558511652:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.674911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241414558511705:2095] [node 4] ICP01 ready to work
2025-12-24T02:11:42.672296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054195:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.682819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414378054123:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26737180ns) {Configure:104(26736625ns) {ProcessConfigured:135(26475610ns) {StartInitialHandshake:157(26474866ns) {DropHandshakes:470(333ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(198ns) PrepareNewSessionHandshake:477(1975ns) {SwitchToState:179(1292ns)}}}}}
2025-12-24T02:11:42.646243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417430682022:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.646264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417430682023:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.646275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417430682024:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.646417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417430682038:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:42.661063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241417430682077:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:42.666232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417430682023:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17608134ns) {Configure:104(17606793ns) {ProcessConfigured:135(17319035ns) {StartInitialHandshake:157(17318742ns) {DropHandshakes:470(285ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(37ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1113ns) {SwitchToState:179(441ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.683652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414482598153:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7247
2025-12-24T02:11:42.683676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414482598154:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7247
2025-12-24T02:11:42.683801Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417430682038:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(32814595ns) {Configure:104(25937985ns) {ProcessConfigured:135(25584271ns) {StartInitialHandshake:157(25583926ns) {DropHandshakes:470(300ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(860ns) {SwitchToState:179(613ns)}}}}}
2025-12-24T02:11:42.685674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417764034833:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31094001ns) {Configure:104(31093380ns) {ProcessConfigured:135(30789465ns) {StartInitialHandshake:157(30789075ns) {DropHandshakes:470(360ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(1399ns) {SwitchToState:179(1095ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.685904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034905:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.687267Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417764034831:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28325314ns) {Configure:104(28324722ns) {ProcessConfigured:135(28025339ns) {StartInitialHandshake:157(28024983ns) {DropHandshakes:470(460ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(223ns) PrepareNewSessionHandshake:477(1121ns) {SwitchToState:179(36ns)}}}}}
2025-12-24T02:11:42.687335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034902:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.687410Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034903:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.687445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034904:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.689628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034902:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.690025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034903:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.690837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241414482598136:2095] [node 6] ICP01 ready to work
2025-12-24T02:11:42.690906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414482598132:2097] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.688620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414378054189:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:42.690862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510418:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.690974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241417764034881:2090] [node 6] ICP01 ready to work
2025-12-24T02:11:42.702236Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241414558511653:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55203014ns) {Configure:104(55195032ns) {ProcessConfigured:135(53775894ns) {StartInitialHandshake:157(53775438ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(216ns)} GenerateSessionVirtualId:988(369ns) PrepareNewSessionHandshake:477(1331ns) {SwitchToState:179(200ns)}}}}}
2025-12-24T02:11:42.702451Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241416087510340:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(56682270ns) {Configure:104(56681457ns) {ProcessConfigured:135(38603111ns) {StartInitialHandshake:157(38602291ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(768ns) {SwitchToState:179(28ns)}}}}}
2025-12-24T02:11:42.702854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414482598129:2094] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.702880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.702888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414482598129:2094] [node 4] ICP22 created new session: [1:7587241414482598157:2094]
2025-12-24T02:11:42.702894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414482598157:2094] [node 4] ICS09 handshake done sender: [1:7587241414482598132:2097] self: [1:7587241414482598131:0] peer: [4:7587241416087510414:0] socket: 360 qp: -1
2025-12-24T02:11:42.702903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414482598157:2094] [node 4] ICS10 traffic start
2025-12-24T02:11:42.702915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.706354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054201:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.706847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241417430682024:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(45197560ns) {Configure:104(45197078ns) {ProcessConfigured:135(44972058ns) {StartInitialHandshake:157(44971597ns) {DropHandshakes:470(211ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(300ns) PrepareNewSessionHandshake:477(1245ns) {SwitchToState:179(918ns)}}}}}
2025-12-24T02:11:42.707114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682101:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.707590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417430682101:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.707639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417430682038:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.709390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414482598147:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.709871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414482598077:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.709886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.709894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414482598077:2049] [node 3] ICP22 created new session: [1:7587241414482598161:2049]
2025-12-24T02:11:42.709901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414482598161:2049] [node 3] ICS09 handshake done sender: [1:7587241414482598147:2110] self: [1:7587241414482598125:0] peer: [3:7587241414558511727:0] socket: 371 qp: -1
2025-12-24T02:11:42.709907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414482598161:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.709916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.709770Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414378054124:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(61060777ns) {Configure:104(61059968ns) {ProcessConfigured:135(60701011ns) {StartInitialHandshake:157(60699948ns) {DropHandshakes:470(421ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1270ns) {SwitchToState:179(611ns)}}}}}
2025-12-24T02:11:42.711257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417764034860:2070] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(60178518ns) {Configure:104(60177937ns) {ProcessConfigured:135(59855034ns) {StartInitialHandshake:157(59854702ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(105ns) PrepareNewSessionHandshake:477(1388ns) {SwitchToState:179(665ns)}}}}}
2025-12-24T02:11:42.711375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417764034860:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.713821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598153:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.713852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241414482598077:2049] [node 3] ICP18 handshake (actor [1:7587241414482598153:2114]) is too old
2025-12-24T02:11:42.714389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598149:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.714406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414482598149:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.714449Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510405:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.714468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241416087510405:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.714782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682078:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.716176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414482598163:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7247
2025-12-24T02:11:42.715684Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414378054186:2105] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(60373530ns) {Configure:104(60373137ns) {ProcessConfigured:135(60102018ns) {StartInitialHandshake:157(60101636ns) {DropHandshakes:470(344ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(1029ns) {SwitchToState:179(504ns)}}}}}
2025-12-24T02:11:42.715972Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414378054189:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27215960ns) {Configure:104(27215298ns) {ProcessConfigured:135(26833478ns) {StartInitialHandshake:157(26833077ns) {DropHandshakes:470(886ns) {DropIncomingHandshake:442(189ns) DropOutgoingHandshake:457(217ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(518ns) {SwitchToState:179(323ns)}}}}}
2025-12-24T02:11:42.716255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054201:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.716277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241414378054123:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.718949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510420:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.718769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511732:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.718800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511733:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.719200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416087510402:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.719034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241414482598136:2095] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27963545ns) {Configure:104(27962981ns) {ProcessConfigured:135(27546980ns) {StartInitialHandshake:157(27546606ns) {DropHandshakes:470(574ns) {DropIncomingHandshake:442(84ns) DropOutgoingHandshake:457(193ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(746ns) {SwitchToState:179(484ns)}}}}}
2025-12-24T02:11:42.719130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598151:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.719134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414482598151:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.719123Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241417764034881:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28007459ns) {Configure:104(28006999ns) {ProcessConfigured:135(27601163ns) {StartInitialHandshake:157(27600994ns) {DropHandshakes:470(414ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(772ns) {SwitchToState:179(453ns)}}}}}
2025-12-24T02:11:42.719281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241416087510338:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.719305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.719314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241416087510338:2048] [node 1] ICP22 created new session: [4:7587241416087510421:2048]
2025-12-24T02:11:42.719322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416087510421:2048] [node 1] ICS09 handshake done sender: [4:7587241416087510402:2108] self: [4:7587241416087510414:0] peer: [1:7587241414482598131:0] socket: 367 qp: -1
2025-12-24T02:11:42.719330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416087510421:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.719348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.719666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417764034831:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.719700Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241417764034831:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(29632045ns) {IncomingHandshake:261(29631440ns) {DropIncomingHandshake:442(41ns) PrepareNewSessionHandshake:477(345ns) {SwitchToState:179(120ns)} GenerateSessionVirtualId:988(240ns) IssueIncomingHandshakeReply:193(29627012ns) {SwitchToState:179(571ns)}}}
2025-12-24T02:11:42.719457Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598163:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.719821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241414482598164:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7247
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.722155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241414558511725:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.724005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598164:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.723953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682103:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.724332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417430682103:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.724357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417430682022:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.722301Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510401:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.722312Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241416087510401:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.724507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416087510401:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.724625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241416087510376:2084] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.724638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.724644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241416087510376:2084] [node 5] ICP22 created new session: [4:7587241416087510424:2084]
2025-12-24T02:11:42.724650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416087510424:2084] [node 5] ICS09 handshake done sender: [4:7587241416087510401:2107] self: [4:7587241416087510400:0] peer: [5:7587241417764034913:0] socket: 368 qp: -1
2025-12-24T02:11:42.724657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416087510424:2084] [node 5] ICS10 traffic start
2025-12-24T02:11:42.724664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.725388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511726:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.725397Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241414558511726:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.725819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241414558511652:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.725833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.725838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241414558511652:2048] [node 1] ICP22 created new session: [3:7587241414558511735:2048]
2025-12-24T02:11:42.725845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241414558511735:2048] [node 1] ICS09 handshake done sender: [3:7587241414558511725:2111] self: [3:7587241414558511727:0] peer: [1:7587241414482598125:0] socket: 375 qp: -1
2025-12-24T02:11:42.725852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241414558511735:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.725859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.726061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241414558511652:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:42.726722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417430682078:2098] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.726759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417430682024:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.727088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241414558511726:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.724404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034904:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.724441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417764034833:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.727234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511712:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.727245Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241414558511712:2101] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.728265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510418:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.727275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598165:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.727286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241414482598165:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.727566Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414482598149:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.735385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414482598123:2093] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.735414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.735421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414482598123:2093] [node 5] ICP22 created new session: [1:7587241414482598170:2093]
2025-12-24T02:11:42.735428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414482598170:2093] [node 5] ICS09 handshake done sender: [1:7587241414482598149:2111] self: [1:7587241414482598127:0] peer: [5:7587241417764034909:0] socket: 372 qp: -1
2025-12-24T02:11:42.735434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414482598170:2093] [node 5] ICS10 traffic start
2025-12-24T02:11:42.735447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.734814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682106:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.737143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417430682101:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.738055Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034905:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.737383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417430682038:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.737402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.737408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417430682038:2060] [node 4] ICP22 created new session: [6:7587241417430682107:2060]
2025-12-24T02:11:42.737414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417430682107:2060] [node 4] ICS09 handshake done sender: [6:7587241417430682101:2116] self: [6:7587241417430682102:0] peer: [4:7587241416087510404:0] socket: 376 qp: -1
2025-12-24T02:11:42.737421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417430682107:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:42.737441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.738660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054207:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
2025-12-24T02:11:42.736532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416087510405:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.737465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241416087510381:2087] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.737478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.737483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241416087510381:2087] [node 6] ICP22 created new session: [4:7587241416087510429:2087]
2025-12-24T02:11:42.737488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416087510429:2087] [node 6] ICS09 handshake done sender: [4:7587241416087510405:2109] self: [4:7587241416087510404:0] peer: [6:7587241417430682102:0] socket: 373 qp: -1
2025-12-24T02:11:42.737493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416087510429:2087] [node 6] ICS10 traffic start
2025-12-24T02:11:42.737500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.738661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510413:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.739189Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241414482598154:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.740130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241417764034831:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241417764034903:2112] held: no
2025-12-24T02:11:42.741160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511734:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.739806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241416087510339:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.743182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414482598151:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.743722Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054207:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.746229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034917:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.746423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034918:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.746660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034919:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.746815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417764034904:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.747091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417764034903:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.746112Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241414558511705:2095] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(70972083ns) {Configure:104(70971307ns) {ProcessConfigured:135(70564863ns) {StartInitialHandshake:157(70564107ns) {DropHandshakes:470(743ns) {DropIncomingHandshake:442(180ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(210ns)}}}}}
2025-12-24T02:11:42.746141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241414558511711:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.746151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.746158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241414558511711:2099] [node 5] ICP22 created new session: [3:7587241414558511739:2099]
2025-12-24T02:11:42.746164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241414558511739:2099] [node 5] ICS09 handshake done sender: [3:7587241414558511726:2112] self: [3:7587241414558511715:0] peer: [5:7587241417764034915:0] socket: 381 qp: -1
2025-12-24T02:11:42.746171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241414558511739:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:42.746186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.746509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511732:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.746628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241414558511711:2099] [node 5] ICP18 handshake (actor [3:7587241414558511732:2115]) is too old
2025-12-24T02:11:42.747135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417764034902:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.747159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417764034833:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.747171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.747178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417764034833:2050] [node 3] ICP22 created new session: [5:7587241417764034920:2050]
2025-12-24T02:11:42.747185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417764034920:2050] [node 3] ICS09 handshake done sender: [5:7587241417764034904:2113] self: [5:7587241417764034915:0] peer: [3:7587241414558511715:0] socket: 390 qp: -1
2025-12-24T02:11:42.747191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417764034920:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.747199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.748023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417764034831:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.748036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.748041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417764034831:2048] [node 1] ICP22 created new session: [5:7587241417764034923:2048]
2025-12-24T02:11:42.748046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417764034923:2048] [node 1] ICS09 handshake done sender: [5:7587241417764034903:2112] self: [5:7587241417764034909:0] peer: [1:7587241414482598127:0] socket: 389 qp: -1
2025-12-24T02:11:42.748052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417764034923:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.748058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.748089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417764034860:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.748099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.748105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417764034860:2070] [node 4] ICP22 created new session: [5:7587241417764034925:2070]
2025-12-24T02:11:42.748110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417764034925:2070] [node 4] ICS09 handshake done sender: [5:7587241417764034902:2111] self: [5:7587241417764034913:0] peer: [4:7587241416087510400:0] socket: 387 qp: -1
2025-12-24T02:11:42.748116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417764034925:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:42.748133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.747984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510432:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.750647Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241416087510376:2084] [node 5] ICP35 event processing took too much time PendingConnection:279(25906942ns) {HandleHandshakeStatus:329(25905965ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(45ns) ProcessPendingSessionEvents:495(25800033ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25526936ns max# 25444111ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.750761Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054194:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.751105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682110:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.752052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241414482598165:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.752718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414482598136:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.752733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.752739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414482598136:2095] [node 6] ICP22 created new session: [1:7587241414482598175:2095]
2025-12-24T02:11:42.752744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414482598175:2095] [node 6] ICS09 handshake done sender: [1:7587241414482598165:2118] self: [1:7587241414482598156:0] peer: [6:7587241417430682104:0] socket: 408 qp: -1
2025-12-24T02:11:42.752750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414482598175:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:42.752783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.753776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417430682110:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.753868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417430682077:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.753927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682111:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.753969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682112:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.754320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417430682103:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:42.753321Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241416087510338:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(33934495ns) {HandleHandshakeStatus:329(33933766ns) {DropIncomingHandshake:442(238ns) DropOutgoingHandshake:457(5412ns) SwitchToState:179(53ns) ProcessPendingSessionEvents:495(12137938ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12113351ns max# 12039729ns}}}
2025-12-24T02:11:42.754090Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510433:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.755038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241414558511712:2101] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.754477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034914:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.754485Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241417764034914:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.755189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417764034914:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:42.755256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241417764034881:2090] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.755268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.755272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417764034881:2090] [node 6] ICP22 created new session: [5:7587241417764034929:2090]
2025-12-24T02:11:42.755277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417764034929:2090] [node 6] ICS09 handshake done sender: [5:7587241417764034914:2118] self: [5:7587241417764034911:0] peer: [6:7587241417430682113:0] socket: 422 qp: -1
2025-12-24T02:11:42.755280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417764034929:2090] [node 6] ICS10 traffic start
2025-12-24T02:11:42.755294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.758960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417430682078:2098] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.766044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054193:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.766063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241414378054193:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.770379Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241414378054123:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20252) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241414378054201:2114] held: no
2025-12-24T02:11:42.766464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241414482598076:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.766499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.766507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241414482598076:2048] [node 2] ICP22 created new session: [1:7587241414482598177:2048]
2025-12-24T02:11:42.766515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241414482598177:2048] [node 2] ICS09 handshake done sender: [1:7587241414482598151:2112] self: [1:7587241414482598128:0] peer: [2:7587241414378054206:0] socket: 402 qp: -1
2025-12-24T02:11:42.766522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241414482598177:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:42.766530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.766579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417430682022:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.766598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.766604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417430682022:2048] [node 1] ICP22 created new session: [6:7587241417430682114:2048]
2025-12-24T02:11:42.766610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417430682114:2048] [node 1] ICS09 handshake done sender: [6:7587241417430682103:2117] self: [6:7587241417430682104:0] peer: [1:7587241414482598156:0] socket: 411 qp: -1
2025-12-24T02:11:42.766615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417430682114:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.766621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.766863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417430682024:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.766869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.766874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417430682024:2050] [node 3] ICP22 created new session: [6:7587241417430682116:2050]
2025-12-24T02:11:42.766878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417430682116:2050] [node 3] ICS09 handshake done sender: [6:7587241417430682078:2098] self: [6:7587241417430682105:0] peer: [3:7587241414558511702:0] socket: 370 qp: -1
2025-12-24T02:11:42.766882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417430682116:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.766886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.771829Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241417430682077:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8385) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241417430682110:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.772171Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
2025-12-24T02:11:42.772186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241414378054212:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8730
2025-12-24T02:11:42.772779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682118:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.772844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682119:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.773071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417430682110:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.773172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417430682077:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.773185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.773194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417430682077:2066] [node 5] ICP22 created new session: [6:7587241417430682120:2066]
2025-12-24T02:11:42.773200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417430682120:2066] [node 5] ICS09 handshake done sender: [6:7587241417430682110:2119] self: [6:7587241417430682113:0] peer: [5:7587241417764034911:0] socket: 420 qp: -1
2025-12-24T02:11:42.773206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417430682120:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:42.773216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:42.773515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241414378054201:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.775858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.776010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054212:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.776986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241414378054123:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.776996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:42.777003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414378054123:2048] [node 1] ICP22 created new session: [2:7587241414378054214:2048]
2025-12-24T02:11:42.777009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241414378054214:2048] [node 1] ICS09 handshake done sender: [2:7587241414378054201:2114] self: [2:7587241414378054206:0] peer: [1:7587241414482598128:0] socket: 403 qp: -1
2025-12-24T02:11:42.777016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241414378054214:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:42.777041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:42.776664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511734:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.776701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241414558511653:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.776766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241414558511697:2091] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.776779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.776787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241414558511697:2091] [node 6] ICP22 created new session: [3:7587241414558511746:2091]
2025-12-24T02:11:42.776794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241414558511746:2091] [node 6] ICS09 handshake done sender: [3:7587241414558511712:2101] self: [3:7587241414558511702:0] peer: [6:7587241417430682105:0] socket: 363 qp: -1
2025-12-24T02:11:42.776800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241414558511746:2091] [node 6] ICS10 traffic start
2025-12-24T02:11:42.776810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:42.777068Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241414558511697:2091] [node 6] ICP35 event processing took too much time PendingConnection:279(21616791ns) {HandleHandshakeStatus:329(21616614ns) {DropIncomingHandshake:442(657ns) DropOutgoingHandshake:457(180ns) SwitchToState:179(67ns) ProcessPendingSessionEvents:495(124721ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 101805ns max# 17194ns}}}
2025-12-24T02:11:42.777477Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241417430682119:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.777613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241417430682023:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.777703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511744:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.777834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054202:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.777841Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241414378054202:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.778234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054204:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.778237Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241414378054204:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.778855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241414378054202:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.778912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241414378054204:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.778995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241414558511749:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2326
2025-12-24T02:11:42.779080Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241414482598136:2095] [node 6] ICP35 event processing took too much time PendingConnection:279(26252538ns) {HandleHandshakeStatus:329(26251424ns) {DropIncomingHandshake:442(3704ns) DropOutgoingHandshake:457(278ns) SwitchToState:179(116ns) ProcessPendingSessionEvents:495(15657649ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15636621ns max# 15569728ns}}}
2025-12-24T02:11:42.779111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241414378054186:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.779129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:42.779134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414378054186:2105] [node 6] ICP22 created new session: [2:7587241414378054218:2105]
2025-12-24T02:11:42.779139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241414378054218:2105] [node 6] ICS09 handshake done sender: [2:7587241414378054204:2116] self: [2:7587241414378054187:0] peer: [6:7587241417430682122:0] socket: 383 qp: -1
2025-12-24T02:11:42.779144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241414378054218:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:42.779159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.779857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241417430682023:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241417430682119:2123] held: no
2025-12-24T02:11:42.780138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241414558511734:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.780293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241414378054193:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.780414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241414378054184:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.780435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.780441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414378054184:2104] [node 4] ICP22 created new session: [2:7587241414378054222:2104]
2025-12-24T02:11:42.780447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241414378054222:2104] [node 4] ICS09 handshake done sender: [2:7587241414378054193:2110] self: [2:7587241414378054185:0] peer: [4:7587241416087510431:0] socket: 379 qp: -1
2025-12-24T02:11:42.780454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241414378054222:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:42.780461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:42.780234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241414558511653:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.780246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.780251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241414558511653:2049] [node 2] ICP22 created new session: [3:7587241414558511750:2049]
2025-12-24T02:11:42.780255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241414558511750:2049] [node 2] ICS09 handshake done sender: [3:7587241414558511734:2117] self: [3:7587241414558511745:0] peer: [2:7587241414378054183:0] socket: 407 qp: -1
2025-12-24T02:11:42.780260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241414558511750:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.780266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.779231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241416087510433:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.779275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241416087510340:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.779554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034931:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.787472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241414558511738:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.787482Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241414558511738:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.787705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241414378054124:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.787722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.787727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414378054124:2049] [node 3] ICP22 created new session: [2:7587241414378054224:2049]
2025-12-24T02:11:42.787732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241414378054224:2049] [node 3] ICS09 handshake done sender: [2:7587241414378054202:2115] self: [2:7587241414378054183:0] peer: [3:7587241414558511745:0] socket: 404 qp: -1
2025-12-24T02:11:42.787738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241414378054224:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:42.787743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.786948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241417764034881:2090] [node 6] ICP35 event processing took too much time PendingConnection:279(31568538ns) {HandleHandshakeStatus:329(31567479ns) {DropIncomingHandshake:442(3218ns) DropOutgoingHandshake:457(99ns) SwitchToState:179(80ns) ProcessPendingSessionEvents:495(20612965ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20594990ns max# 20518650ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.789539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510435:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.789836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416087510413:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.790713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241416087510339:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.790732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.790739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241416087510339:2049] [node 2] ICP22 created new session: [4:7587241416087510436:2049]
2025-12-24T02:11:42.790746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416087510436:2049] [node 2] ICS09 handshake done sender: [4:7587241416087510413:2112] self: [4:7587241416087510431:0] peer: [2:7587241414378054185:0] socket: 380 qp: -1
2025-12-24T02:11:42.790752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416087510436:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.790760Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.795504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241417430682123:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14867
2025-12-24T02:11:42.798012Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241417430682119:2123] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.795068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241417764034931:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.795119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241417764034832:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:42.798090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241417430682023:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.798100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.798105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241417430682023:2049] [node 2] ICP22 created new session: [6:7587241417430682124:2049]
2025-12-24T02:11:42.798111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241417430682124:2049] [node 2] ICS09 handshake done sender: [6:7587241417430682119:2123] self: [6:7587241417430682122:0] peer: [2:7587241414378054187:0] socket: 386 qp: -1
2025-12-24T02:11:42.798115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241417430682124:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.798125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.797600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241414558511738:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:42.797907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241414558511705:2095] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.797920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:42.797927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241414558511705:2095] [node 4] ICP22 created new session: [3:7587241414558511753:2095]
2025-12-24T02:11:42.797934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241414558511753:2095] [node 4] ICS09 handshake done sender: [3:7587241414558511738:2118] self: [3:7587241414558511730:0] peer: [4:7587241416087510434:0] socket: 394 qp: -1
2025-12-24T02:11:42.797939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241414558511753:2095] [node 4] ICS10 traffic start
2025-12-24T02:11:42.797946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.801867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241414378054205:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:42.801879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241414378054205:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:42.801665Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241414482598076:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(47903416ns) {HandleHandshakeStatus:329(47902916ns) {DropIncomingHandshake:442(8459ns) DropOutgoingHandshake:457(399ns) SwitchToState:179(66ns) ProcessPendingSessionEvents:495(25224294ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25203348ns max# 25099028ns}}}
2025-12-24T02:11:42.801313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241416087510438:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32563
2025-12-24T02:11:42.802274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241416087510433:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:42.802368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241416087510340:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.802382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:42.802388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241416087510340:2050] [node 3] ICP22 created new session: [4:7587241416087510439:2050]
2025-12-24T02:11:42.802394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241416087510439:2050] [node 3] ICS09 handshake done sender: [4:7587241416087510433:2120] self: [4:7587241416087510434:0] peer: [3:7587241414558511730:0] socket: 426 qp: -1
2025-12-24T02:11:42.802400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241416087510439:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:42.802405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:42.803351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241414378054205:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:42.803441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241414378054189:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:42.803458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:42.803465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414378054189:2106] [node 5] ICP22 created new session: [2:7587241414378054227:2106]
2025-12-24T02:11:42.803471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241414378054227:2106] [node 5] ICS09 handshake done sender: [2:7587241414378054205:2117] self: [2:7587241414378054200:0] peer: [5:7587241417764034932:0] socket: 385 qp: -1
2025-12-24T02:11:42.803476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241414378054227:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:42.803482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:42.807986Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241417764034832:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6690) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241417764034931:2122] held: no
2025-12-24T02:11:42.808019Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241417764034933:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49440
2025-12-24T02:11:42.810314Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241417764034931:2122] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:42.810407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241417764034832:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:42.810420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:42.810427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241417764034832:2049] [node 2] ICP22 created new session: [5:7587241417764034934:2049]
2025-12-24T02:11:42.810433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241417764034934:2049] [node 2] ICS09 handshake done sender: [5:7587241417764034931:2122] self: [5:7587241417764034932:0] peer: [2:7587241414378054200:0] socket: 405 qp: -1
2025-12-24T02:11:42.810437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241417764034934:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:42.810446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:42.832623Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:42.832727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241414378054214:2048] [node 1] ICS01 socket: 403 reason# EndOfStream
2025-12-24T02:11:42.832738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241414378054123:2048] [node 1] ICP30 unregister session Session# [2:7587241414378054214:2048] VirtualId# [2:7587241414378054206:0]
2025-12-24T02:11:42.832746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241414378054214:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.832792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.832811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.834369Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:42.834467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241414558511735:2048] [node 1] ICS01 socket: 375 reason# EndOfStream
2025-12-24T02:11:42.834476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241414558511652:2048] [node 1] ICP30 unregister session Session# [3:7587241414558511735:2048] VirtualId# [3:7587241414558511727:0]
2025-12-24T02:11:42.834483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241414558511735:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.834517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.834540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.842866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:42.842964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241416087510421:2048] [node 1] ICS07 socket disconnect 367 reason# EndOfStream
2025-12-24T02:11:42.842971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241416087510421:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.843016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.843036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241416087510421:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:42.843810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241416087510338:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20252) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:42.843832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241416087510421:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:42.843837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241416087510338:2048] [node 1] ICP30 unregister session Session# [4:7587241416087510421:2048] VirtualId# [4:7587241416087510414:0]
2025-12-24T02:11:42.843889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.843896Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241416087510338:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:42.843899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20252) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:42.846828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:42.846896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417430682114:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-12-24T02:11:42.846901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417430682022:2048] [node 1] ICP30 unregister session Session# [6:7587241417430682114:2048] VirtualId# [6:7587241417430682104:0]
2025-12-24T02:11:42.846907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417430682114:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.846949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.846972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.847108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:42.847166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417764034923:2048] [node 1] ICS01 socket: 389 reason# EndOfStream
2025-12-24T02:11:42.847170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417764034831:2048] [node 1] ICP30 unregister session Session# [5:7587241417764034923:2048] VirtualId# [5:7587241417764034909:0]
2025-12-24T02:11:42.847174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417764034923:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.847197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:42.847217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:42.853125Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:42.853199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241414558511750:2049] [node 2] ICS01 socket: 407 reason# EndOfStream
2025-12-24T02:11:42.853205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241414558511653:2049] [node 2] ICP30 unregister session Session# [3:7587241414558511750:2049] VirtualId# [3:7587241414558511745:0]
2025-12-24T02:11:42.853210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241414558511750:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.853236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.853253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.854991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241416087510338:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:42.858839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:42.858914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241416087510436:2049] [node 2] ICS01 socket: 380 reason# EndOfStream
2025-12-24T02:11:42.858920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241416087510339:2049] [node 2] ICP30 unregister session Session# [4:7587241416087510436:2049] VirtualId# [4:7587241416087510431:0]
2025-12-24T02:11:42.858928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241416087510436:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.858976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.858995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.863456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:42.863525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417764034934:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:42.863530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417764034832:2049] [node 2] ICP30 unregister session Session# [5:7587241417764034934:2049] VirtualId# [5:7587241417764034932:0]
2025-12-24T02:11:42.863535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417764034934:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.863578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.863601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.863942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:42.864004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417430682124:2049] [node 2] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:42.864008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417430682023:2049] [node 2] ICP30 unregister session Session# [6:7587241417430682124:2049] VirtualId# [6:7587241417430682122:0]
2025-12-24T02:11:42.864012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417430682124:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.864040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:42.864060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:42.882865Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:42.882955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417430682116:2050] [node 3] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:42.882965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417430682024:2050] [node 3] ICP30 unregister session Session# [6:7587241417430682116:2050] VirtualId# [6:7587241417430682105:0]
2025-12-24T02:11:42.882972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417430682116:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.883022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.883043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.883081Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:42.883137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241417764034920:2050] [node 3] ICS01 socket: 390 reason# EndOfStream
2025-12-24T02:11:42.883142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241417764034833:2050] [node 3] ICP30 unregister session Session# [5:7587241417764034920:2050] VirtualId# [5:7587241417764034915:0]
2025-12-24T02:11:42.883146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241417764034920:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.883172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:42.883192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:42.902995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:42.903098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417430682107:2060] [node 4] ICS01 socket: 376 reason# EndOfStream
2025-12-24T02:11:42.903110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417430682038:2060] [node 4] ICP30 unregister session Session# [6:7587241417430682107:2060] VirtualId# [6:7587241417430682102:0]
2025-12-24T02:11:42.903118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417430682107:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.903169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:42.903197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:42.914792Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:42.914894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241417430682120:2066] [node 5] ICS01 socket: 420 reason# EndOfStream
2025-12-24T02:11:42.914909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241417430682077:2066] [node 5] ICP30 unregister session Session# [6:7587241417430682120:2066] VirtualId# [6:7587241417430682113:0]
2025-12-24T02:11:42.914916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241417430682120:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:42.914950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:42.914972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:42.958776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241418082853367:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.958786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241418082853368:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.950895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241415895699607:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:42.950904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241415895699608:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.963489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414251608653:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.963502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241414251608654:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:42.966900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414256745274:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.966913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414256745275:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.966918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241414256745276:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.967094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241416737701343:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.967104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241416737701344:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.967109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241416737701345:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:42.987003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241414907970918:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:42.987022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241414907970919:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:42.987027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241414907970920:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.045367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241420190666964:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.046776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576008:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.047081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820721:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.051189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241419202938264:2096] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.058070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418546576012:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:43.058199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418546576013:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:43.058285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576011:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
2025-12-24T02:11:43.065836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241419202938280:2101] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.074877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422377820729:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:43.074978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422377820730:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:43.072848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241420190666977:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:43.073071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241420190666978:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:43.073204Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241420190666976:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-12-24T02:11:43.091051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422377820738:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.091871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938292:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.095737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241421032668704:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:43.095835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668701:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.091996Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241414251608654:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33663975ns) {Configure:104(33662908ns) {ProcessConfigured:135(33188679ns) {StartInitialHandshake:157(33188078ns) {DropHandshakes:470(1275ns) {DropIncomingHandshake:442(175ns) DropOutgoingHandshake:457(205ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(1489ns) {SwitchToState:179(1067ns)}}}}}
2025-12-24T02:11:43.111435Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241418546576012:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19347970ns) {Configure:104(19346882ns) {ProcessConfigured:135(18812638ns) {StartInitialHandshake:157(18811954ns) {DropHandshakes:470(382ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(2310ns) {SwitchToState:179(676ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.115033Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820743:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.115079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820744:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.112438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418546576024:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:43.113259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576025:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
2025-12-24T02:11:43.116802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241420190666977:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42465357ns) {Configure:104(42464286ns) {ProcessConfigured:135(42038933ns) {StartInitialHandshake:157(42038458ns) {DropHandshakes:470(1854ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(150ns) PrepareNewSessionHandshake:477(1288ns) {SwitchToState:179(851ns)}}}}}
2025-12-24T02:11:43.117662Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576011:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.120096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241418551712640:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:43.120264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241418551712641:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:43.120565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712636:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.120580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712637:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.120628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712638:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.120638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712639:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.124686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241421032668715:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:43.124809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668701:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.130809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712637:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.134499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241420190666983:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.134545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241420190666984:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.138854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241419202938292:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.138899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241414907970920:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.141402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820735:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.141417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422377820735:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.139129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576025:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.139222Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712636:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.139253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414256745274:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.142147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712638:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.142171Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712643:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.142174Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241418551712643:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.141070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668716:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.141101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668717:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.142025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668717:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.142072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241421032668704:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.142839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241418551712643:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.142938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422377820735:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.143064Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820743:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.143077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422377820729:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.143087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.143094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422377820729:2109] [node 6] ICP22 created new session: [3:7587241422377820746:2109]
2025-12-24T02:11:43.143099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422377820746:2109] [node 6] ICS09 handshake done sender: [3:7587241422377820735:2113] self: [3:7587241422377820734:0] peer: [6:7587241419202938294:0] socket: 385 qp: -1
2025-12-24T02:11:43.143105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422377820746:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:43.143112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.145216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241421032668704:2108] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31578) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241421032668717:2116] held: no
2025-12-24T02:11:43.145484Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668722:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.145625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668723:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.145716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241421032668717:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.142932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241418551712640:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.142947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.142955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241418551712640:2108] [node 5] ICP22 created new session: [4:7587241418551712656:2108]
2025-12-24T02:11:43.142961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241418551712656:2108] [node 5] ICS09 handshake done sender: [4:7587241418551712643:2115] self: [4:7587241418551712642:0] peer: [5:7587241421032668721:0] socket: 401 qp: -1
2025-12-24T02:11:43.142968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241418551712656:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:43.142980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.143519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938293:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.143616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938295:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.144882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241419202938293:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.144901Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241419202938292:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.145467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241414907970920:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.145481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.145487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241414907970920:2050] [node 3] ICP22 created new session: [6:7587241419202938296:2050]
2025-12-24T02:11:43.145492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241419202938296:2050] [node 3] ICS09 handshake done sender: [6:7587241419202938292:2115] self: [6:7587241419202938294:0] peer: [3:7587241422377820734:0] socket: 386 qp: -1
2025-12-24T02:11:43.145499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241419202938296:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.145507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.145890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241419202938280:2101] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.145778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241421032668704:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.145791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.145796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241421032668704:2108] [node 4] ICP22 created new session: [5:7587241421032668725:2108]
2025-12-24T02:11:43.145801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241421032668725:2108] [node 4] ICS09 handshake done sender: [5:7587241421032668717:2116] self: [5:7587241421032668721:0] peer: [4:7587241418551712642:0] socket: 404 qp: -1
2025-12-24T02:11:43.145805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241421032668725:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:43.145811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.147288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712645:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.147298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241418551712645:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.147725Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241418551712645:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.147809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241418551712641:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.147825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.147831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241418551712641:2109] [node 6] ICP22 created new session: [4:7587241418551712660:2109]
2025-12-24T02:11:43.147836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241418551712660:2109] [node 6] ICS09 handshake done sender: [4:7587241418551712645:2116] self: [4:7587241418551712644:0] peer: [6:7587241419202938298:0] socket: 402 qp: -1
2025-12-24T02:11:43.147840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241418551712660:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:43.147847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.148651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666983:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.148705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666976:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.148724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666968:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.148727Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241420190666968:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.149110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666984:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.149281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241420190666985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.149468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712639:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.149589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414256745275:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.149636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712662:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.148501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576032:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
2025-12-24T02:11:43.148535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576033:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
2025-12-24T02:11:43.150564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820744:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.150592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820721:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.150665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241418082853368:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.150676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241418082853367:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.150680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668723:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.150793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241416737701345:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.150818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820750:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.150935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241420190666968:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.150902Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820737:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.150907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422377820737:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.151275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820750:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.151350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241418551712636:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.151609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414256745274:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.151623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.151628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414256745274:2048] [node 1] ICP22 created new session: [4:7587241418551712665:2048]
2025-12-24T02:11:43.151633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241418551712665:2048] [node 1] ICS09 handshake done sender: [4:7587241418551712636:2111] self: [4:7587241418551712653:0] peer: [1:7587241420190666967:0] socket: 376 qp: -1
2025-12-24T02:11:43.151639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241418551712665:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.151646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.152138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422377820737:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.152274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576032:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.152300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576008:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.152208Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666970:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.152216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241420190666970:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.156473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668728:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.158752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576019:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.158761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418546576019:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.159233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576017:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.159236Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418546576017:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.159651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576033:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.159816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241414251608653:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.160015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241419202938280:2101] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31578) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241419202938293:2116] held: no
2025-12-24T02:11:43.160226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938300:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.160537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422377820730:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.160557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.160563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422377820730:2110] [node 5] ICP22 created new session: [3:7587241422377820756:2110]
2025-12-24T02:11:43.160569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422377820756:2110] [node 5] ICS09 handshake done sender: [3:7587241422377820737:2114] self: [3:7587241422377820736:0] peer: [5:7587241421032668727:0] socket: 410 qp: -1
2025-12-24T02:11:43.160575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422377820756:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:43.160582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.161041Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241419202938293:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.161101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241419202938280:2101] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.161113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.161118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241419202938280:2101] [node 4] ICP22 created new session: [6:7587241419202938301:2101]
2025-12-24T02:11:43.161122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241419202938301:2101] [node 4] ICS09 handshake done sender: [6:7587241419202938293:2116] self: [6:7587241419202938298:0] peer: [4:7587241418551712644:0] socket: 405 qp: -1
2025-12-24T02:11:43.161127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241419202938301:2101] [node 4] ICS10 traffic start
2025-12-24T02:11:43.161133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.161592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820755:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.162973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241420190666964:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.163001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.163008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241420190666964:2107] [node 4] ICP22 created new session: [1:7587241420190666990:2107]
2025-12-24T02:11:43.163015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241420190666990:2107] [node 4] ICS09 handshake done sender: [1:7587241420190666968:2109] self: [1:7587241420190666967:0] peer: [4:7587241418551712653:0] socket: 367 qp: -1
2025-12-24T02:11:43.163020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241420190666990:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:43.163027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.163185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668716:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.163224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418546576019:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.163854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418546576017:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.163892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241416737701344:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.163942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418546576012:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.163960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.163970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418546576012:2107] [node 4] ICP22 created new session: [2:7587241418546576040:2107]
2025-12-24T02:11:43.163983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418546576040:2107] [node 4] ICS09 handshake done sender: [2:7587241418546576019:2113] self: [2:7587241418546576018:0] peer: [4:7587241418551712664:0] socket: 393 qp: -1
2025-12-24T02:11:43.163995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418546576040:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:43.164008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.164006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422377820744:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.164072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241418082853368:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.164075Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241416737701345:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15470) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241421032668723:2119] held: no
2025-12-24T02:11:43.164081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.164092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241418082853368:2049] [node 2] ICP22 created new session: [3:7587241422377820759:2049]
2025-12-24T02:11:43.164097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422377820759:2049] [node 2] ICS09 handshake done sender: [3:7587241422377820744:2117] self: [3:7587241422377820751:0] peer: [2:7587241418546576015:0] socket: 397 qp: -1
2025-12-24T02:11:43.164103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422377820759:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.164110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.164268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241414251608654:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.164285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.164291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414251608654:2049] [node 3] ICP22 created new session: [2:7587241418546576042:2049]
2025-12-24T02:11:43.164300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418546576042:2049] [node 3] ICS09 handshake done sender: [2:7587241418546576017:2112] self: [2:7587241418546576015:0] peer: [3:7587241422377820751:0] socket: 394 qp: -1
2025-12-24T02:11:43.164304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418546576042:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.164314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.165374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241421032668723:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.165367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712667:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.167456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666966:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.167474Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241420190666966:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.168291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576021:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.168305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418546576021:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.168349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241416737701345:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.168367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.168374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241416737701345:2050] [node 3] ICP22 created new session: [5:7587241421032668731:2050]
2025-12-24T02:11:43.168380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241421032668731:2050] [node 3] ICS09 handshake done sender: [5:7587241421032668723:2119] self: [5:7587241421032668727:0] peer: [3:7587241422377820736:0] socket: 413 qp: -1
2025-12-24T02:11:43.168387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241421032668731:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.168395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.168535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666985:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.168907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241420190666970:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.169200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241420190666966:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.169249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712668:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
2025-12-24T02:11:43.169305Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241414256745275:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5592) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241418551712639:2114] held: no
2025-12-24T02:11:43.169680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241418551712668:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.169701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241414256745276:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.169735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241418551712639:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.169813Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418546576021:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.169315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241415895699608:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.169333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.169338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241415895699608:2049] [node 3] ICP22 created new session: [1:7587241420190666996:2049]
2025-12-24T02:11:43.169342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241420190666996:2049] [node 3] ICS09 handshake done sender: [1:7587241420190666970:2110] self: [1:7587241420190666969:0] peer: [3:7587241422377820752:0] socket: 368 qp: -1
2025-12-24T02:11:43.169347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241420190666996:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.169353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.169542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241415895699607:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.169547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.169550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241415895699607:2048] [node 2] ICP22 created new session: [1:7587241420190666998:2048]
2025-12-24T02:11:43.169553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241420190666998:2048] [node 2] ICS09 handshake done sender: [1:7587241420190666966:2108] self: [1:7587241420190666965:0] peer: [2:7587241418546576038:0] socket: 366 qp: -1
2025-12-24T02:11:43.169555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241420190666998:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:43.169559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.169847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414256745275:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.169859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.169864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414256745275:2049] [node 2] ICP22 created new session: [4:7587241418551712670:2049]
2025-12-24T02:11:43.169869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241418551712670:2049] [node 2] ICS09 handshake done sender: [4:7587241418551712639:2114] self: [4:7587241418551712664:0] peer: [2:7587241418546576018:0] socket: 399 qp: -1
2025-12-24T02:11:43.169874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241418551712670:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.169880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.169200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422377820761:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28220
2025-12-24T02:11:43.169493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422377820721:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.169596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241418082853367:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.169607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.169616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241418082853367:2048] [node 1] ICP22 created new session: [3:7587241422377820762:2048]
2025-12-24T02:11:43.169621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422377820762:2048] [node 1] ICS09 handshake done sender: [3:7587241422377820721:2105] self: [3:7587241422377820752:0] peer: [1:7587241420190666969:0] socket: 370 qp: -1
2025-12-24T02:11:43.169627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422377820762:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.169633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.170613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422377820740:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.170624Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422377820740:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.168669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.169963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938304:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.171571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422377820740:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.171277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418546576013:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.171296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.171303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418546576013:2108] [node 5] ICP22 created new session: [2:7587241418546576048:2108]
2025-12-24T02:11:43.171309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418546576048:2108] [node 5] ICS09 handshake done sender: [2:7587241418546576021:2114] self: [2:7587241418546576020:0] peer: [5:7587241421032668729:0] socket: 395 qp: -1
2025-12-24T02:11:43.171314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418546576048:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:43.171321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.171822Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241418551712672:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23163
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.171890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241420190667000:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61435
2025-12-24T02:11:43.171982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241418551712668:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.171709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422377820738:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.171724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.171730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422377820738:2111] [node 4] ICP22 created new session: [3:7587241422377820765:2111]
2025-12-24T02:11:43.171735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422377820765:2111] [node 4] ICS09 handshake done sender: [3:7587241422377820740:2115] self: [3:7587241422377820739:0] peer: [4:7587241418551712669:0] socket: 396 qp: -1
2025-12-24T02:11:43.171739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422377820765:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:43.171746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.172098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668733:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.172167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668734:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.173774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666982:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.173782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241420190666982:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.174332Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241420190666982:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.170977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241419202938303:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.171014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241419202938264:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.172267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938306:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.173691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241419202938306:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.173715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241414907970918:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.174449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938308:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.174928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241421032668716:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.175027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241416737701344:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.175042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.175048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241416737701344:2049] [node 2] ICP22 created new session: [5:7587241421032668735:2049]
2025-12-24T02:11:43.175055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241421032668735:2049] [node 2] ICS09 handshake done sender: [5:7587241421032668716:2115] self: [5:7587241421032668729:0] peer: [2:7587241418546576020:0] socket: 403 qp: -1
2025-12-24T02:11:43.175060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241421032668735:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.175068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.176075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418546576047:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55317
2025-12-24T02:11:43.182009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241414256745276:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.182043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.182052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241414256745276:2050] [node 3] ICP22 created new session: [4:7587241418551712673:2050]
2025-12-24T02:11:43.182060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241418551712673:2050] [node 3] ICS09 handshake done sender: [4:7587241418551712668:2121] self: [4:7587241418551712669:0] peer: [3:7587241422377820739:0] socket: 428 qp: -1
2025-12-24T02:11:43.182071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241418551712673:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.182083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.182902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241419202938306:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.182789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668719:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.182817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241421032668719:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.186103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241414256745276:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:43.184257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241420190666978:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.184289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.184298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241420190666978:2112] [node 6] ICP22 created new session: [1:7587241420190667003:2112]
2025-12-24T02:11:43.184306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241420190667003:2112] [node 6] ICS09 handshake done sender: [1:7587241420190666982:2115] self: [1:7587241420190666981:0] peer: [6:7587241419202938307:0] socket: 435 qp: -1
2025-12-24T02:11:43.184312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241420190667003:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:43.184325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.183408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241421032668733:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.186853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418546576008:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.189470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241416737701344:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:43.189598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241421032668719:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.187110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241414251608653:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.187130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.187136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241414251608653:2048] [node 1] ICP22 created new session: [2:7587241418546576050:2048]
2025-12-24T02:11:43.187142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418546576050:2048] [node 1] ICS09 handshake done sender: [2:7587241418546576008:2106] self: [2:7587241418546576038:0] peer: [1:7587241420190666965:0] socket: 369 qp: -1
2025-12-24T02:11:43.187148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418546576050:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.187156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.186737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241414907970918:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.186763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.186771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241414907970918:2048] [node 1] ICP22 created new session: [6:7587241419202938309:2048]
2025-12-24T02:11:43.186778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241419202938309:2048] [node 1] ICS09 handshake done sender: [6:7587241419202938306:2121] self: [6:7587241419202938307:0] peer: [1:7587241420190666981:0] socket: 421 qp: -1
2025-12-24T02:11:43.186786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241419202938309:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.186796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.189705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241421032668715:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.189727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.189735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241421032668715:2114] [node 6] ICP22 created new session: [5:7587241421032668738:2114]
2025-12-24T02:11:43.189741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241421032668738:2114] [node 6] ICS09 handshake done sender: [5:7587241421032668719:2117] self: [5:7587241421032668718:0] peer: [6:7587241419202938305:0] socket: 388 qp: -1
2025-12-24T02:11:43.189747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241421032668738:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:43.189756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.190012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241416737701343:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.194871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190667000:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.195175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241420190666980:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.195181Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241420190666980:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.196002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241415895699607:2048] [node 2] ICP18 handshake (actor [1:7587241420190667000:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.196481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241421032668741:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12825
2025-12-24T02:11:43.197671Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241420190666980:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.197892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241421032668733:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.197955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241416737701343:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.197980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.197992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241416737701343:2048] [node 1] ICP22 created new session: [5:7587241421032668742:2048]
2025-12-24T02:11:43.197999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241421032668742:2048] [node 1] ICS09 handshake done sender: [5:7587241421032668733:2121] self: [5:7587241421032668740:0] peer: [1:7587241420190666979:0] socket: 384 qp: -1
2025-12-24T02:11:43.198006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241421032668742:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.198019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.196356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241419202938304:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.199812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241420190666977:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.199839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.199849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241420190666977:2111] [node 5] ICP22 created new session: [1:7587241420190667006:2111]
2025-12-24T02:11:43.199855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241420190667006:2111] [node 5] ICS09 handshake done sender: [1:7587241420190666980:2114] self: [1:7587241420190666979:0] peer: [5:7587241421032668740:0] socket: 416 qp: -1
2025-12-24T02:11:43.199861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241420190667006:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:43.199869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.201260Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418546576027:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.201275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418546576027:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.202060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418546576027:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.202144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418546576024:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.202164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.202170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418546576024:2110] [node 6] ICP22 created new session: [2:7587241418546576053:2110]
2025-12-24T02:11:43.202178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418546576053:2110] [node 6] ICS09 handshake done sender: [2:7587241418546576027:2117] self: [2:7587241418546576026:0] peer: [6:7587241419202938314:0] socket: 431 qp: -1
2025-12-24T02:11:43.202182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418546576053:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:43.202187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.199924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938311:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.200389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241419202938264:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6450) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241419202938303:2119] held: no
2025-12-24T02:11:43.200578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241419202938303:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.200643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241419202938264:2096] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.200653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.200658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241419202938264:2096] [node 5] ICP22 created new session: [6:7587241419202938312:2096]
2025-12-24T02:11:43.200663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241419202938312:2096] [node 5] ICS09 handshake done sender: [6:7587241419202938303:2119] self: [6:7587241419202938305:0] peer: [5:7587241421032668718:0] socket: 427 qp: -1
2025-12-24T02:11:43.200669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241419202938312:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:43.200675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.201068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241414907970919:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.203649Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241414907970919:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5592) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241419202938304:2120] held: no
2025-12-24T02:11:43.203855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241419202938315:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12831
2025-12-24T02:11:43.203975Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241419202938304:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.204017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241414907970919:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.204025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.204031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241414907970919:2049] [node 2] ICP22 created new session: [6:7587241419202938316:2049]
2025-12-24T02:11:43.204036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241419202938316:2049] [node 2] ICS09 handshake done sender: [6:7587241419202938304:2120] self: [6:7587241419202938314:0] peer: [2:7587241418546576026:0] socket: 432 qp: -1
2025-12-24T02:11:43.204041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241419202938316:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.204047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.215712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:43.215833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241418546576050:2048] [node 1] ICS01 socket: 369 reason# EndOfStream
2025-12-24T02:11:43.215844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241414251608653:2048] [node 1] ICP30 unregister session Session# [2:7587241418546576050:2048] VirtualId# [2:7587241418546576038:0]
2025-12-24T02:11:43.215853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241418546576050:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.215893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.215927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.218634Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:43.218741Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241422377820762:2048] [node 1] ICS01 socket: 370 reason# EndOfStream
2025-12-24T02:11:43.218748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241418082853367:2048] [node 1] ICP30 unregister session Session# [3:7587241422377820762:2048] VirtualId# [3:7587241422377820752:0]
2025-12-24T02:11:43.218755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241422377820762:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.218802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.218834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.223432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:43.223520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241418551712665:2048] [node 1] ICS01 socket: 376 reason# EndOfStream
2025-12-24T02:11:43.223528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241414256745274:2048] [node 1] ICP30 unregister session Session# [4:7587241418551712665:2048] VirtualId# [4:7587241418551712653:0]
2025-12-24T02:11:43.223535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241418551712665:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.223580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.223607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.225704Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:43.225788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241419202938309:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-12-24T02:11:43.225796Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241414907970918:2048] [node 1] ICP30 unregister session Session# [6:7587241419202938309:2048] VirtualId# [6:7587241419202938307:0]
2025-12-24T02:11:43.225803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241419202938309:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.225851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.225873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.226793Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:43.226861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241421032668742:2048] [node 1] ICS01 socket: 384 reason# EndOfStream
2025-12-24T02:11:43.226867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241416737701343:2048] [node 1] ICP30 unregister session Session# [5:7587241421032668742:2048] VirtualId# [5:7587241421032668740:0]
2025-12-24T02:11:43.226872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241421032668742:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.226907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.226930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.234889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:43.234982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241422377820759:2049] [node 2] ICS01 socket: 397 reason# EndOfStream
2025-12-24T02:11:43.234992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241418082853368:2049] [node 2] ICP30 unregister session Session# [3:7587241422377820759:2049] VirtualId# [3:7587241422377820751:0]
2025-12-24T02:11:43.234999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241422377820759:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.235045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.235070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.237656Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:43.237761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241418551712670:2049] [node 2] ICS01 socket: 399 reason# EndOfStream
2025-12-24T02:11:43.237770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241414256745275:2049] [node 2] ICP30 unregister session Session# [4:7587241418551712670:2049] VirtualId# [4:7587241418551712664:0]
2025-12-24T02:11:43.237776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241418551712670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.237823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.237848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.238770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:43.238851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241421032668735:2049] [node 2] ICS01 socket: 403 reason# EndOfStream
2025-12-24T02:11:43.238859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241416737701344:2049] [node 2] ICP30 unregister session Session# [5:7587241421032668735:2049] VirtualId# [5:7587241421032668729:0]
2025-12-24T02:11:43.238865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241421032668735:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.238905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.238929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.238967Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:43.239036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241419202938316:2049] [node 2] ICS01 socket: 432 reason# EndOfStream
2025-12-24T02:11:43.239042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241414907970919:2049] [node 2] ICP30 unregister session Session# [6:7587241419202938316:2049] VirtualId# [6:7587241419202938314:0]
2025-12-24T02:11:43.239046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241419202938316:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.239079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.239101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.251076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:43.251179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241419202938296:2050] [node 3] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:43.251191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241414907970920:2050] [node 3] ICP30 unregister session Session# [6:7587241419202938296:2050] VirtualId# [6:7587241419202938294:0]
2025-12-24T02:11:43.251200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241419202938296:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.251251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.251274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.251317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:43.251386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241421032668731:2050] [node 3] ICS01 socket: 413 reason# EndOfStream
2025-12-24T02:11:43.251392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241416737701345:2050] [node 3] ICP30 unregister session Session# [5:7587241421032668731:2050] VirtualId# [5:7587241421032668727:0]
2025-12-24T02:11:43.251397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241421032668731:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.251421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.251442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.274988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:43.275097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241419202938301:2101] [node 4] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:43.275110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241419202938280:2101] [node 4] ICP30 unregister session Session# [6:7587241419202938301:2101] VirtualId# [6:7587241419202938298:0]
2025-12-24T02:11:43.275118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241419202938301:2101] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.275170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:43.275193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:43.315088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419260416568:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:43.315097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419260416569:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:43.332513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422363109144:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.332525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422363109145:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.332531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422363109146:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.333135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422539268578:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.333145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422539268579:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.358813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420979591920:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.358883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420979591921:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:43.359061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241422518538380:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.359078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241422518538381:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.359365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241420440522951:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.359381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241420440522952:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.359461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241420440522950:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.362709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241422518538379:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.392849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268638:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.394671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422363109170:2072] [node 6] ICP01 ready to work
2025-12-24T02:11:43.395041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241422518538404:2061] [node 4] ICP01 ready to work
2025-12-24T02:11:43.395173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241422518538405:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:43.394949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241420440522974:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:43.395040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241420440522976:2062] [node 6] ICP01 ready to work
2025-12-24T02:11:43.395418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419260416632:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.409766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538453:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.410953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979591980:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
2025-12-24T02:11:43.411080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523027:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.416135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241422518538453:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.416173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241422518538405:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.424374Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241419260416568:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28880837ns) {Configure:104(28880333ns) {ProcessConfigured:135(28563907ns) {StartInitialHandshake:157(28563243ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(106ns) PrepareNewSessionHandshake:477(1871ns) {SwitchToState:179(808ns)}}}}}
2025-12-24T02:11:43.424787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419260416638:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:43.425001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419260416640:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:43.425736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419260416639:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9561
2025-12-24T02:11:43.426047Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241419260416632:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30054991ns) {Configure:104(30052647ns) {ProcessConfigured:135(29832028ns) {StartInitialHandshake:157(29831881ns) {DropHandshakes:470(259ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(673ns) {SwitchToState:179(303ns)}}}}}
2025-12-24T02:11:43.427276Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422363109146:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32799960ns) {Configure:104(32798757ns) {ProcessConfigured:135(32178229ns) {StartInitialHandshake:157(32177843ns) {DropHandshakes:470(674ns) {DropIncomingHandshake:442(140ns) DropOutgoingHandshake:457(124ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(1358ns) {SwitchToState:179(911ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.431013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979591981:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
2025-12-24T02:11:43.431360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420979591986:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:43.431934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523018:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.431953Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241420440523018:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.432548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523029:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.432675Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241422539268578:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17284143ns) {Configure:104(17282993ns) {ProcessConfigured:135(16844166ns) {StartInitialHandshake:157(16843711ns) {DropHandshakes:470(1144ns) {DropIncomingHandshake:442(344ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(1626ns) {SwitchToState:179(909ns)}}}}}
2025-12-24T02:11:43.432752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268638:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.432872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422539268644:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:43.433525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241422539268578:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.434151Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241420440523018:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.434651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420979591983:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:43.434892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420979591988:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:43.434974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422539268640:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:43.435110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422539268654:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.437354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422363109216:2075] [node 5] ICP01 ready to work
2025-12-24T02:11:43.437483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109211:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.439120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416629:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.439134Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419260416629:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.439630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419260416648:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9561
2025-12-24T02:11:43.439643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419260416649:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9561
2025-12-24T02:11:43.439153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109211:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.439938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422363109170:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(45187197ns) {Configure:104(45186672ns) {ProcessConfigured:135(44591773ns) {StartInitialHandshake:157(44591471ns) {DropHandshakes:470(216ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(42ns) PrepareNewSessionHandshake:477(878ns) {SwitchToState:179(470ns)}}}}}
2025-12-24T02:11:43.440495Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419260416629:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.441196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419260416569:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.441212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.441219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419260416569:2049] [node 3] ICP22 created new session: [1:7587241419260416651:2049]
2025-12-24T02:11:43.441226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419260416651:2049] [node 3] ICS09 handshake done sender: [1:7587241419260416629:2107] self: [1:7587241419260416628:0] peer: [3:7587241422539268649:0] socket: 372 qp: -1
2025-12-24T02:11:43.441234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419260416651:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.441254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.445286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523027:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.445308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241420440522976:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.445318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.445325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241420440522976:2062] [node 6] ICP22 created new session: [5:7587241420440523032:2062]
2025-12-24T02:11:43.445331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241420440523032:2062] [node 6] ICS09 handshake done sender: [5:7587241420440523018:2110] self: [5:7587241420440523017:0] peer: [6:7587241422518538458:0] socket: 379 qp: -1
2025-12-24T02:11:43.445338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241420440523032:2062] [node 6] ICS10 traffic start
2025-12-24T02:11:43.445358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.445701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241420440522976:2062] [node 6] ICP18 handshake (actor [5:7587241420440523027:2114]) is too old
2025-12-24T02:11:43.445728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523031:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.446588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591981:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.447454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591980:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.447549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979591989:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
2025-12-24T02:11:43.447589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979591990:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
2025-12-24T02:11:43.447600Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979591991:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.449231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268660:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.449286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268661:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.449296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268658:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.449307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268659:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.449556Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241420979591920:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18135668ns) {Configure:104(18134333ns) {ProcessConfigured:135(17670122ns) {StartInitialHandshake:157(17669814ns) {DropHandshakes:470(1130ns) {DropIncomingHandshake:442(328ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(783ns) {SwitchToState:179(18ns)}}}}}
2025-12-24T02:11:43.454938Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268660:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.455914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523029:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.455972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241420440522950:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.458476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241422518538405:2068] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64978) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241422518538453:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.458881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523036:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.459306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591989:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.461816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538461:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.458380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416642:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.458398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419260416642:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.458803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416639:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.467980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422539268638:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.471563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241420979591920:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.471177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241422539268578:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.471206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.471215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422539268578:2048] [node 1] ICP22 created new session: [3:7587241422539268663:2048]
2025-12-24T02:11:43.471222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422539268663:2048] [node 1] ICS09 handshake done sender: [3:7587241422539268638:2107] self: [3:7587241422539268649:0] peer: [1:7587241419260416628:0] socket: 373 qp: -1
2025-12-24T02:11:43.471233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422539268663:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.471262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.472039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241419260416569:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(30717863ns) {HandleHandshakeStatus:329(30717123ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(430ns) SwitchToState:179(86ns) ProcessPendingSessionEvents:495(21154525ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21132264ns max# 21032843ns}}}
2025-12-24T02:11:43.472481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538459:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.473109Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268658:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.473140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268661:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.473262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241422539268579:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.472444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591991:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.473484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591990:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.475537Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591997:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.475544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241420979591997:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.476072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241420979591997:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.476291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241420979591921:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.476317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.476326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420979591921:2049] [node 3] ICP22 created new session: [2:7587241420979592010:2049]
2025-12-24T02:11:43.476333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241420979592010:2049] [node 3] ICS09 handshake done sender: [2:7587241420979591997:2117] self: [2:7587241420979591996:0] peer: [3:7587241422539268668:0] socket: 409 qp: -1
2025-12-24T02:11:43.476341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241420979592010:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.476350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.476710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268669:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.477097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422539268661:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.477143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241422539268579:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.477154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.477162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422539268579:2049] [node 2] ICP22 created new session: [3:7587241422539268670:2049]
2025-12-24T02:11:43.477168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422539268670:2049] [node 2] ICS09 handshake done sender: [3:7587241422539268661:2118] self: [3:7587241422539268668:0] peer: [2:7587241420979591996:0] socket: 414 qp: -1
2025-12-24T02:11:43.477174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422539268670:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.477180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.479216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523036:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.479255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241420440522951:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.479486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538460:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.479574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241422518538453:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.479668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241422518538405:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.479683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.479688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241422518538405:2068] [node 5] ICP22 created new session: [6:7587241422518538463:2068]
2025-12-24T02:11:43.479693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241422518538463:2068] [node 5] ICS09 handshake done sender: [6:7587241422518538453:2111] self: [6:7587241422518538458:0] peer: [5:7587241420440523017:0] socket: 381 qp: -1
2025-12-24T02:11:43.479700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241422518538463:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:43.479707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.479939Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.481388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422539268673:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49974
2025-12-24T02:11:43.481486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241422518538462:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.481538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241422518538381:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.481615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268673:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.481638Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268662:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.481641Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422539268662:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.482757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422539268662:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.482888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422539268654:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.482908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.482915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422539268654:2113] [node 6] ICP22 created new session: [3:7587241422539268676:2113]
2025-12-24T02:11:43.482920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422539268676:2113] [node 6] ICS09 handshake done sender: [3:7587241422539268662:2119] self: [3:7587241422539268656:0] peer: [6:7587241422518538466:0] socket: 394 qp: -1
2025-12-24T02:11:43.482925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422539268676:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:43.482931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.483408Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416649:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.487163Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523038:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.488134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241420440523029:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.488233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241420440522950:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.488252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.488258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241420440522950:2048] [node 1] ICP22 created new session: [5:7587241420440523039:2048]
2025-12-24T02:11:43.488265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241420440523039:2048] [node 1] ICS09 handshake done sender: [5:7587241420440523029:2116] self: [5:7587241420440523035:0] peer: [1:7587241419260416641:0] socket: 391 qp: -1
2025-12-24T02:11:43.488271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241420440523039:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.488280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.486278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416637:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.486294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419260416637:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.486786Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241419260416569:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:43.487032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419260416642:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.489518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109224:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
2025-12-24T02:11:43.489672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109225:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.489894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538467:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.490014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591993:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.490029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241420979591993:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.490667Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419260416637:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.490939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109224:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.490977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422363109146:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.491197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241422518538462:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.491256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523031:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.491279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241420440522952:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.491365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109225:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.491526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268648:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.491537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422539268648:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.492081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109228:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.492232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523043:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.492254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523044:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.492342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422539268648:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.492497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241420440523031:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.492550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241420440522952:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.492567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.492574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241420440522952:2050] [node 3] ICP22 created new session: [5:7587241420440523045:2050]
2025-12-24T02:11:43.492580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241420440523045:2050] [node 3] ICS09 handshake done sender: [5:7587241420440523031:2117] self: [5:7587241420440523042:0] peer: [3:7587241422539268647:0] socket: 405 qp: -1
2025-12-24T02:11:43.492587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241420440523045:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.492593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.493439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422539268644:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.493457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.493463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422539268644:2109] [node 5] ICP22 created new session: [3:7587241422539268679:2109]
2025-12-24T02:11:43.493468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422539268679:2109] [node 5] ICS09 handshake done sender: [3:7587241422539268648:2112] self: [3:7587241422539268647:0] peer: [5:7587241420440523042:0] socket: 395 qp: -1
2025-12-24T02:11:43.493472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422539268679:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:43.493479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.503172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419260416568:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.503201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.503212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419260416568:2048] [node 2] ICP22 created new session: [1:7587241419260416658:2048]
2025-12-24T02:11:43.503221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419260416658:2048] [node 2] ICS09 handshake done sender: [1:7587241419260416637:2111] self: [1:7587241419260416633:0] peer: [2:7587241420979592006:0] socket: 388 qp: -1
2025-12-24T02:11:43.503229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419260416658:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:43.503241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.503889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422539268657:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.503911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422539268657:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.505722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241422518538381:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.505747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.505758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241422518538381:2050] [node 3] ICP22 created new session: [6:7587241422518538468:2050]
2025-12-24T02:11:43.505765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241422518538468:2050] [node 3] ICS09 handshake done sender: [6:7587241422518538462:2119] self: [6:7587241422518538466:0] peer: [3:7587241422539268656:0] socket: 377 qp: -1
2025-12-24T02:11:43.505771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241422518538468:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.505779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.506158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241422518538460:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.506873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109229:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
2025-12-24T02:11:43.504896Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241420979591993:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.505090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419260416638:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.505113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.505120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419260416638:2109] [node 5] ICP22 created new session: [1:7587241419260416660:2109]
2025-12-24T02:11:43.505128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419260416660:2109] [node 5] ICS09 handshake done sender: [1:7587241419260416642:2113] self: [1:7587241419260416641:0] peer: [5:7587241420440523035:0] socket: 389 qp: -1
2025-12-24T02:11:43.505134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419260416660:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:43.505142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.505360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422539268657:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.507642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422363109224:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.507722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241420440523036:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.507392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241420979592012:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27203
2025-12-24T02:11:43.507571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241422518538381:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:43.507604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241422518538380:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.507694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241422518538461:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.507742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241422518538379:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.508102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422539268640:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.508122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.508129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422539268640:2108] [node 4] ICP22 created new session: [3:7587241422539268682:2108]
2025-12-24T02:11:43.508134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422539268682:2108] [node 4] ICS09 handshake done sender: [3:7587241422539268657:2114] self: [3:7587241422539268655:0] peer: [4:7587241422363109226:0] socket: 393 qp: -1
2025-12-24T02:11:43.508139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422539268682:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:43.508147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.508951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422363109146:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.508970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.508977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422363109146:2050] [node 3] ICP22 created new session: [4:7587241422363109230:2050]
2025-12-24T02:11:43.508983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422363109230:2050] [node 3] ICS09 handshake done sender: [4:7587241422363109224:2116] self: [4:7587241422363109226:0] peer: [3:7587241422539268655:0] socket: 425 qp: -1
2025-12-24T02:11:43.508990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422363109230:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.508997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.509133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416644:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.509143Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419260416644:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.511072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241420440522951:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.511093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.511101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241420440522951:2049] [node 2] ICP22 created new session: [5:7587241420440523047:2049]
2025-12-24T02:11:43.511108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241420440523047:2049] [node 2] ICS09 handshake done sender: [5:7587241420440523036:2118] self: [5:7587241420440523037:0] peer: [2:7587241420979591992:0] socket: 417 qp: -1
2025-12-24T02:11:43.511115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241420440523047:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.511122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.511226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419260416644:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.511230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591995:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.511246Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241420979591995:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.514084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523049:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.511665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241420979591986:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.511688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.511695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420979591986:2111] [node 5] ICP22 created new session: [2:7587241420979592015:2111]
2025-12-24T02:11:43.511701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241420979592015:2111] [node 5] ICS09 handshake done sender: [2:7587241420979591993:2115] self: [2:7587241420979591992:0] peer: [5:7587241420440523037:0] socket: 410 qp: -1
2025-12-24T02:11:43.511707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241420979592015:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:43.511714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.512016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241420979591989:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.513935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538472:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.515060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241422518538461:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.515178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241422518538379:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.515197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.515217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241422518538379:2048] [node 1] ICP22 created new session: [6:7587241422518538473:2048]
2025-12-24T02:11:43.515223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241422518538473:2048] [node 1] ICS09 handshake done sender: [6:7587241422518538461:2118] self: [6:7587241422518538471:0] peer: [1:7587241419260416643:0] socket: 420 qp: -1
2025-12-24T02:11:43.515230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241422518538473:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.515241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.515841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419260416640:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.515858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.515866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419260416640:2110] [node 6] ICP22 created new session: [1:7587241419260416664:2110]
2025-12-24T02:11:43.515873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419260416664:2110] [node 6] ICS09 handshake done sender: [1:7587241419260416644:2114] self: [1:7587241419260416643:0] peer: [6:7587241422518538471:0] socket: 418 qp: -1
2025-12-24T02:11:43.515880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419260416664:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:43.515890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.516514Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419260416666:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9561
2025-12-24T02:11:43.516439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109233:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
2025-12-24T02:11:43.516532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109228:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.516564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422363109145:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.516880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538475:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.516744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241420979591920:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.516765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.516771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420979591920:2048] [node 1] ICP22 created new session: [2:7587241420979592019:2048]
2025-12-24T02:11:43.516778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241420979592019:2048] [node 1] ICS09 handshake done sender: [2:7587241420979591989:2112] self: [2:7587241420979592006:0] peer: [1:7587241419260416633:0] socket: 397 qp: -1
2025-12-24T02:11:43.516786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241420979592019:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.516793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.517175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241420979591999:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.517180Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241420979591999:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.517792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241420979591995:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.517364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109233:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.517393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422363109144:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.517497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419260416647:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.517504Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419260416647:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.518077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419260416647:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.518157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419260416632:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.518174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.518179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419260416632:2108] [node 4] ICP22 created new session: [1:7587241419260416668:2108]
2025-12-24T02:11:43.518185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419260416668:2108] [node 4] ICS09 handshake done sender: [1:7587241419260416647:2115] self: [1:7587241419260416634:0] peer: [4:7587241422363109235:0] socket: 436 qp: -1
2025-12-24T02:11:43.518191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419260416668:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:43.518197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.518944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241422518538460:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.519006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241422518538380:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.519020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.519025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241422518538380:2049] [node 2] ICP22 created new session: [6:7587241422518538476:2049]
2025-12-24T02:11:43.519029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241422518538476:2049] [node 2] ICS09 handshake done sender: [6:7587241422518538460:2117] self: [6:7587241422518538470:0] peer: [2:7587241420979591994:0] socket: 419 qp: -1
2025-12-24T02:11:43.519035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241422518538476:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.519042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.520434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241420979591999:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.520746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241420979591988:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.520769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.520777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420979591988:2110] [node 4] ICP22 created new session: [2:7587241420979592022:2110]
2025-12-24T02:11:43.520783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241420979592022:2110] [node 4] ICS09 handshake done sender: [2:7587241420979591999:2118] self: [2:7587241420979591998:0] peer: [4:7587241422363109234:0] socket: 407 qp: -1
2025-12-24T02:11:43.520788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241420979592022:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:43.520796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.518791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109236:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
2025-12-24T02:11:43.518819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422363109237:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40572
2025-12-24T02:11:43.527754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241422363109144:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22821) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241422363109233:2120] held: no
2025-12-24T02:11:43.528156Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241422363109145:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17258) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241422363109228:2118] held: no
2025-12-24T02:11:43.528321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241420440523049:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.528370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241420440522974:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.528568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241420979591983:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.528599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.528606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420979591983:2109] [node 6] ICP22 created new session: [2:7587241420979592024:2109]
2025-12-24T02:11:43.528613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241420979592024:2109] [node 6] ICS09 handshake done sender: [2:7587241420979591995:2116] self: [2:7587241420979591994:0] peer: [6:7587241422518538470:0] socket: 385 qp: -1
2025-12-24T02:11:43.528621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241420979592024:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:43.528634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.530818Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422363109228:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.531021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422363109233:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.533213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241420440523051:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54013
2025-12-24T02:11:43.531787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109218:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.531801Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241422363109218:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.532312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422363109144:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.532329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.532337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422363109144:2048] [node 1] ICP22 created new session: [4:7587241422363109240:2048]
2025-12-24T02:11:43.532343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422363109240:2048] [node 1] ICS09 handshake done sender: [4:7587241422363109233:2120] self: [4:7587241422363109235:0] peer: [1:7587241419260416634:0] socket: 437 qp: -1
2025-12-24T02:11:43.532352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422363109240:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.532361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.533638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422363109218:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.533841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241420440523049:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.533452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538478:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.534935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241420440522974:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.534956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.534966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241420440522974:2060] [node 4] ICP22 created new session: [5:7587241420440523052:2060]
2025-12-24T02:11:43.534972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241420440523052:2060] [node 4] ICS09 handshake done sender: [5:7587241420440523049:2122] self: [5:7587241420440523050:0] peer: [4:7587241422363109217:0] socket: 387 qp: -1
2025-12-24T02:11:43.534980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241420440523052:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:43.534989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.533963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422363109145:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.533980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.533987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422363109145:2049] [node 2] ICP22 created new session: [4:7587241422363109242:2049]
2025-12-24T02:11:43.533995Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422363109242:2049] [node 2] ICS09 handshake done sender: [4:7587241422363109228:2118] self: [4:7587241422363109234:0] peer: [2:7587241420979591998:0] socket: 433 qp: -1
2025-12-24T02:11:43.534002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422363109242:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.534011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.534197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241422363109216:2075] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.534208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.534213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422363109216:2075] [node 5] ICP22 created new session: [4:7587241422363109244:2075]
2025-12-24T02:11:43.534219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422363109244:2075] [node 5] ICS09 handshake done sender: [4:7587241422363109218:2114] self: [4:7587241422363109217:0] peer: [5:7587241420440523050:0] socket: 398 qp: -1
2025-12-24T02:11:43.534223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422363109244:2075] [node 5] ICS10 traffic start
2025-12-24T02:11:43.534230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.537160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241422518538478:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.537203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241422518538404:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.537404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422363109222:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.537412Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241422363109222:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.538188Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241422518538480:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13626
2025-12-24T02:11:43.538315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422363109222:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.538478Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241422518538478:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.538562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241422518538404:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.538578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.538583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241422518538404:2061] [node 4] ICP22 created new session: [6:7587241422518538481:2061]
2025-12-24T02:11:43.538589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241422518538481:2061] [node 4] ICS09 handshake done sender: [6:7587241422518538478:2123] self: [6:7587241422518538479:0] peer: [4:7587241422363109177:0] socket: 399 qp: -1
2025-12-24T02:11:43.538595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241422518538481:2061] [node 4] ICS10 traffic start
2025-12-24T02:11:43.538603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.539559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241422363109170:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.539577Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.539583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422363109170:2072] [node 6] ICP22 created new session: [4:7587241422363109247:2072]
2025-12-24T02:11:43.539587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422363109247:2072] [node 6] ICS09 handshake done sender: [4:7587241422363109222:2115] self: [4:7587241422363109177:0] peer: [6:7587241422518538479:0] socket: 375 qp: -1
2025-12-24T02:11:43.539592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422363109247:2072] [node 6] ICS10 traffic start
2025-12-24T02:11:43.539598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.553561Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:43.553680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241420979592019:2048] [node 1] ICS01 socket: 397 reason# EndOfStream
2025-12-24T02:11:43.553691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241420979591920:2048] [node 1] ICP30 unregister session Session# [2:7587241420979592019:2048] VirtualId# [2:7587241420979592006:0]
2025-12-24T02:11:43.553700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241420979592019:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.553739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.553762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.554262Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:43.554342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241422539268663:2048] [node 1] ICS01 socket: 373 reason# EndOfStream
2025-12-24T02:11:43.554347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241422539268578:2048] [node 1] ICP30 unregister session Session# [3:7587241422539268663:2048] VirtualId# [3:7587241422539268649:0]
2025-12-24T02:11:43.554352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241422539268663:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.554391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.554413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.566917Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:43.567031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241422363109240:2048] [node 1] ICS01 socket: 437 reason# EndOfStream
2025-12-24T02:11:43.567041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241422363109144:2048] [node 1] ICP30 unregister session Session# [4:7587241422363109240:2048] VirtualId# [4:7587241422363109235:0]
2025-12-24T02:11:43.567049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241422363109240:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.567108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.567134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.567181Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:43.567255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241420440523039:2048] [node 1] ICS01 socket: 391 reason# EndOfStream
2025-12-24T02:11:43.567260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241420440522950:2048] [node 1] ICP30 unregister session Session# [5:7587241420440523039:2048] VirtualId# [5:7587241420440523035:0]
2025-12-24T02:11:43.567265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241420440523039:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.567292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.567311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.567346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:43.567404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241422518538473:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-12-24T02:11:43.567409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241422518538379:2048] [node 1] ICP30 unregister session Session# [6:7587241422518538473:2048] VirtualId# [6:7587241422518538471:0]
2025-12-24T02:11:43.567413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241422518538473:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.567437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.567455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.574545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:43.574642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241422539268670:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-12-24T02:11:43.574652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241422539268579:2049] [node 2] ICP30 unregister session Session# [3:7587241422539268670:2049] VirtualId# [3:7587241422539268668:0]
2025-12-24T02:11:43.574660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241422539268670:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.574709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.574731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.582752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:43.582861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241420440523047:2049] [node 2] ICS07 socket disconnect 417 reason# EndOfStream
2025-12-24T02:11:43.582869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241420440523047:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.582918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.582943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241420440523047:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:43.583330Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:43.583411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241422518538476:2049] [node 2] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:43.583418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241422518538380:2049] [node 2] ICP30 unregister session Session# [6:7587241422518538476:2049] VirtualId# [6:7587241422518538470:0]
2025-12-24T02:11:43.583424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241422518538476:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.583453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.583477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.583636Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:43.583705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241422363109242:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:43.583711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241422363109145:2049] [node 2] ICP30 unregister session Session# [4:7587241422363109242:2049] VirtualId# [4:7587241422363109234:0]
2025-12-24T02:11:43.583716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241422363109242:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.583741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.583772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.588476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241420440522951:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17258) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:43.588497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241420440523047:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:43.588502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241420440522951:2049] [node 2] ICP30 unregister session Session# [5:7587241420440523047:2049] VirtualId# [5:7587241420440523037:0]
2025-12-24T02:11:43.588559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.588568Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241420440522951:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17258) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:43.588572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17258) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:43.598192Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:43.598351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241422518538468:2050] [node 3] ICS01 socket: 377 reason# EndOfStream
2025-12-24T02:11:43.598363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241422518538381:2050] [node 3] ICP30 unregister session Session# [6:7587241422518538468:2050] VirtualId# [6:7587241422518538466:0]
2025-12-24T02:11:43.598399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241422518538468:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.598453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.598480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.598581Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:43.598645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241420440523045:2050] [node 3] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:43.598650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241420440522952:2050] [node 3] ICP30 unregister session Session# [5:7587241420440523045:2050] VirtualId# [5:7587241420440523042:0]
2025-12-24T02:11:43.598655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241420440523045:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.598703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.598725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.598798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241420440522951:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:43.615023Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:43.615149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241422518538481:2061] [node 4] ICS01 socket: 399 reason# EndOfStream
2025-12-24T02:11:43.615161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241422518538404:2061] [node 4] ICP30 unregister session Session# [6:7587241422518538481:2061] VirtualId# [6:7587241422518538479:0]
2025-12-24T02:11:43.615170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241422518538481:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.615234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:43.615261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:43.650035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241420411415979:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.650045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241420411415980:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.650051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241420411415981:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.648665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419188970290:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:43.648674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419188970291:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:43.654862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420325873762:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.654880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420325873763:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.656009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419923255051:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.656027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419923255052:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.649070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241422514606073:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.649077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241422514606074:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:43.649082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241422514606075:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:43.659464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419923255050:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.648808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418765254764:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.648819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418765254765:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:43.687452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241422514606098:2068] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.689182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255090:2087] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
2025-12-24T02:11:43.689291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419923255091:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:43.689407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419923255094:2089] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.690601Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873822:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.690205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241422514606112:2077] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.693366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873823:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.693528Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254825:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.693551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254824:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.696588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419188970359:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:43.700677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241420411416020:2087] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.712608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419188970357:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:43.715082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241420411416047:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:43.715338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418765254833:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:43.711845Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241422514606073:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21085718ns) {Configure:104(21085328ns) {ProcessConfigured:135(20845119ns) {StartInitialHandshake:157(20844820ns) {DropHandshakes:470(215ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(944ns) {SwitchToState:179(756ns)}}}}}
2025-12-24T02:11:43.714105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606143:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.715494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241422514606143:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.715533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241422514606073:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.715850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241419188970365:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.723059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606152:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.723678Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241422514606143:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.723841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241422514606073:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.723863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.723873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241422514606073:2048] [node 1] ICP22 created new session: [5:7587241422514606153:2048]
2025-12-24T02:11:43.723880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241422514606153:2048] [node 1] ICS09 handshake done sender: [5:7587241422514606143:2107] self: [5:7587241422514606150:0] peer: [1:7587241419188970360:0] socket: 396 qp: -1
2025-12-24T02:11:43.723887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241422514606153:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.723926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.722814Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241420411415979:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23868022ns) {Configure:104(23866472ns) {ProcessConfigured:135(23515801ns) {StartInitialHandshake:157(23515508ns) {DropHandshakes:470(564ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(371ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(589ns)}}}}}
2025-12-24T02:11:43.722298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970361:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.722315Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419188970361:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.726903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606155:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.723435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419188970361:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.724161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419188970359:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.724170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.724175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419188970359:2110] [node 5] ICP22 created new session: [1:7587241419188970369:2110]
2025-12-24T02:11:43.724180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419188970369:2110] [node 5] ICS09 handshake done sender: [1:7587241419188970361:2111] self: [1:7587241419188970360:0] peer: [5:7587241422514606150:0] socket: 391 qp: -1
2025-12-24T02:11:43.724184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419188970369:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:43.724191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.724429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418765254834:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:43.724530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254824:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.726355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241418765254764:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.731233Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241419923255094:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(41767815ns) {Configure:104(41767165ns) {ProcessConfigured:135(41035207ns) {StartInitialHandshake:157(41034297ns) {DropHandshakes:470(237ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(23ns)} GenerateSessionVirtualId:988(58ns) PrepareNewSessionHandshake:477(1150ns) {SwitchToState:179(423ns)}}}}}
2025-12-24T02:11:43.731292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255090:2087] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.736012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241422514606155:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.736053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241422514606074:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.737081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416056:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.737126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416055:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.739291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420325873828:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:43.739586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420325873829:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:43.740430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241418765254832:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:43.738913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241419188970357:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23875260ns) {Configure:104(23874139ns) {ProcessConfigured:135(23450444ns) {StartInitialHandshake:157(23449579ns) {DropHandshakes:470(1365ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(288ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(1140ns) {SwitchToState:179(764ns)}}}}}
2025-12-24T02:11:43.738989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970353:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.738998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419188970353:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.740326Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255128:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
2025-12-24T02:11:43.742370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241419923255091:2086] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29466467ns) {Configure:104(29464719ns) {ProcessConfigured:135(28927795ns) {StartInitialHandshake:157(28926866ns) {DropHandshakes:470(1021ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(348ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(1496ns) {SwitchToState:179(980ns)}}}}}
2025-12-24T02:11:43.744004Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255128:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.744013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241422514606073:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20091371ns) {HandleHandshakeStatus:329(20090195ns) {DropIncomingHandshake:442(273ns) DropOutgoingHandshake:457(4203ns) SwitchToState:179(161ns) ProcessPendingSessionEvents:495(13808485ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 13791660ns max# 13718944ns}}}
2025-12-24T02:11:43.744084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241422514606115:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.744090Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241422514606115:2082] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.744695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254836:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.744717Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418765254836:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.741005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420325873827:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:43.743328Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241420411416055:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.743431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241420411416047:2092] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.749427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419188970353:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.749692Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255131:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
2025-12-24T02:11:43.744838Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241422514606115:2082] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.745752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241422514606112:2077] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.745765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.745773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241422514606112:2077] [node 6] ICP22 created new session: [5:7587241422514606160:2077]
2025-12-24T02:11:43.745780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241422514606160:2077] [node 6] ICS09 handshake done sender: [5:7587241422514606115:2082] self: [5:7587241422514606114:0] peer: [6:7587241420411416057:0] socket: 395 qp: -1
2025-12-24T02:11:43.745786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241422514606160:2077] [node 6] ICS10 traffic start
2025-12-24T02:11:43.745795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.753227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419188970290:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.753251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.753261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419188970290:2048] [node 2] ICP22 created new session: [1:7587241419188970378:2048]
2025-12-24T02:11:43.753268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419188970378:2048] [node 2] ICS09 handshake done sender: [1:7587241419188970353:2108] self: [1:7587241419188970352:0] peer: [2:7587241418765254839:0] socket: 384 qp: -1
2025-12-24T02:11:43.753274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419188970378:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:43.753282Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.752627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254846:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.752707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254847:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.752762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254848:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.759025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241420411416056:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.759133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241420411415979:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.759574Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254847:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.759613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254846:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.759638Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418765254824:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.760437Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241418765254849:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64865
2025-12-24T02:11:43.760158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416060:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.748950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873832:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.753491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873832:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.753510Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873822:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.753525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873823:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.761970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418765254836:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.762068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419188970375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16188
2025-12-24T02:11:43.762375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241420411416055:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.762985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970367:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.762999Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419188970367:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.763739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241420325873763:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.763788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241420325873762:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.763486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970375:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.764621Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241422514606112:2077] [node 6] ICP35 event processing took too much time PendingConnection:279(18769509ns) {HandleHandshakeStatus:329(18768552ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(428ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(9603731ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 9585673ns max# 9521842ns}}}
2025-12-24T02:11:43.767007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255131:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.767053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419923255052:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.767024Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241420411415981:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16073768ns) {Configure:104(16072102ns) {ProcessConfigured:135(15646290ns) {StartInitialHandshake:157(15645322ns) {DropHandshakes:470(684ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(198ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(906ns) {SwitchToState:179(580ns)}}}}}
2025-12-24T02:11:43.767096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241420411416047:2092] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.767111Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.767119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241420411416047:2092] [node 5] ICP22 created new session: [6:7587241420411416062:2092]
2025-12-24T02:11:43.767125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241420411416062:2092] [node 5] ICS09 handshake done sender: [6:7587241420411416055:2114] self: [6:7587241420411416057:0] peer: [5:7587241422514606114:0] socket: 402 qp: -1
2025-12-24T02:11:43.767134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241420411416062:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:43.767154Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.767885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416064:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.768591Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419188970377:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16188
2025-12-24T02:11:43.768682Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419188970367:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.767219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873835:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.767239Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241420325873835:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.767808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241420325873835:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.767885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241420325873827:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.767896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.767902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420325873827:2109] [node 4] ICP22 created new session: [3:7587241420325873848:2109]
2025-12-24T02:11:43.767907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241420325873848:2109] [node 4] ICS09 handshake done sender: [3:7587241420325873835:2114] self: [3:7587241420325873834:0] peer: [4:7587241419923255133:0] socket: 420 qp: -1
2025-12-24T02:11:43.767913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241420325873848:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:43.767920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.768138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873850:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.768147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873851:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.768821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873850:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.769706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241420325873827:2109] [node 4] ICP18 handshake (actor [3:7587241420325873850:2118]) is too old
2025-12-24T02:11:43.770741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419188970376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16188
2025-12-24T02:11:43.770800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241419188970374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16188
2025-12-24T02:11:43.771600Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241420411416056:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.771092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254825:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.771112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241418765254764:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.771127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.771135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418765254764:2048] [node 1] ICP22 created new session: [2:7587241418765254853:2048]
2025-12-24T02:11:43.771142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418765254853:2048] [node 1] ICS09 handshake done sender: [2:7587241418765254824:2107] self: [2:7587241418765254839:0] peer: [1:7587241419188970352:0] socket: 387 qp: -1
2025-12-24T02:11:43.771148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418765254853:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.771156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.771517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418765254834:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.771529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.771534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418765254834:2112] [node 5] ICP22 created new session: [2:7587241418765254855:2112]
2025-12-24T02:11:43.771539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418765254855:2112] [node 5] ICS09 handshake done sender: [2:7587241418765254836:2113] self: [2:7587241418765254835:0] peer: [5:7587241422514606158:0] socket: 409 qp: -1
2025-12-24T02:11:43.771543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418765254855:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:43.771548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.773488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416066:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.773507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241420411415979:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.773519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.773526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241420411415979:2048] [node 1] ICP22 created new session: [6:7587241420411416067:2048]
2025-12-24T02:11:43.773534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241420411416067:2048] [node 1] ICS09 handshake done sender: [6:7587241420411416056:2115] self: [6:7587241420411416059:0] peer: [1:7587241419188970366:0] socket: 414 qp: -1
2025-12-24T02:11:43.773540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241420411416067:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.773546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.773388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254849:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.775220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970377:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.775480Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970374:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.775756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419188970365:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.775808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.775814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419188970365:2112] [node 6] ICP22 created new session: [1:7587241419188970383:2112]
2025-12-24T02:11:43.775820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419188970383:2112] [node 6] ICS09 handshake done sender: [1:7587241419188970367:2113] self: [1:7587241419188970366:0] peer: [6:7587241420411416059:0] socket: 408 qp: -1
2025-12-24T02:11:43.775825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419188970383:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:43.775833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.776202Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241419188970290:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:43.776772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416069:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.778824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970351:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.778840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419188970351:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.781362Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419923255052:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241419923255131:2116] held: no
2025-12-24T02:11:43.780201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254827:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.780220Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418765254827:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.791110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241420411416066:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.791165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241420411415981:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.791316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873851:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.791407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241420411416069:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.791426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241420411415980:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.792719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419188970351:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.793666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419188970291:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.793692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.793708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419188970291:2049] [node 3] ICP22 created new session: [1:7587241419188970388:2049]
2025-12-24T02:11:43.793716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419188970388:2049] [node 3] ICS09 handshake done sender: [1:7587241419188970351:2107] self: [1:7587241419188970350:0] peer: [3:7587241420325873845:0] socket: 383 qp: -1
2025-12-24T02:11:43.793723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419188970388:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.793734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.791613Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255135:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
2025-12-24T02:11:43.793217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418765254827:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.793412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254841:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.793416Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418765254841:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.792628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241419923255131:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.793788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255136:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.793280Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873839:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.793294Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241420325873839:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.799279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418765254765:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.799311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.799318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418765254765:2049] [node 3] ICP22 created new session: [2:7587241418765254862:2049]
2025-12-24T02:11:43.799325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418765254862:2049] [node 3] ICS09 handshake done sender: [2:7587241418765254827:2109] self: [2:7587241418765254826:0] peer: [3:7587241420325873844:0] socket: 390 qp: -1
2025-12-24T02:11:43.799332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418765254862:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:43.799357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.802943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419923255052:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.802974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.802987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419923255052:2050] [node 3] ICP22 created new session: [4:7587241419923255138:2050]
2025-12-24T02:11:43.802996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241419923255138:2050] [node 3] ICS09 handshake done sender: [4:7587241419923255131:2116] self: [4:7587241419923255133:0] peer: [3:7587241420325873834:0] socket: 425 qp: -1
2025-12-24T02:11:43.803005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241419923255138:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.798805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241422514606074:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241422514606155:2116] held: no
2025-12-24T02:11:43.799046Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241422514606074:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21371907ns) {HandleHandshakeStatus:395(21371242ns) {LogHandshakeFail:472(7193ns) {UpdateErrorStateLog:545(3448ns)} DropOutgoingHandshake:457(1127ns) ProcessPendingSessionEvents:495(105574ns) {unordered DropSessionEvent:512 num# 100 sum# 98970ns max# 7698ns}}}
2025-12-24T02:11:43.799193Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.803033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.803728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255137:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
2025-12-24T02:11:43.805684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255136:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.805723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419923255050:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.805884Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418765254841:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.805959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241419188970372:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.805971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241419188970372:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.805257Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241422514606155:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.807134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606163:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.807180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241422514606074:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.807193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.807199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241422514606074:2049] [node 2] ICP22 created new session: [5:7587241422514606164:2049]
2025-12-24T02:11:43.807208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241422514606164:2049] [node 2] ICS09 handshake done sender: [5:7587241422514606155:2116] self: [5:7587241422514606158:0] peer: [2:7587241418765254835:0] socket: 410 qp: -1
2025-12-24T02:11:43.807215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241422514606164:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.807223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.809881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241422514606163:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.809905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241422514606098:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.807833Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241419188970372:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.809214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241419188970357:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.809229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.809235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241419188970357:2109] [node 4] ICP22 created new session: [1:7587241419188970391:2109]
2025-12-24T02:11:43.809239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241419188970391:2109] [node 4] ICS09 handshake done sender: [1:7587241419188970372:2114] self: [1:7587241419188970364:0] peer: [4:7587241419923255140:0] socket: 439 qp: -1
2025-12-24T02:11:43.809244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241419188970391:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:43.809250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.808651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418765254833:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.808675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.808684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418765254833:2110] [node 6] ICP22 created new session: [2:7587241418765254865:2110]
2025-12-24T02:11:43.808689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418765254865:2110] [node 6] ICS09 handshake done sender: [2:7587241418765254841:2115] self: [2:7587241418765254840:0] peer: [6:7587241420411416071:0] socket: 427 qp: -1
2025-12-24T02:11:43.808695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418765254865:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:43.808702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.810060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241418765254845:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.810066Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241418765254845:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.807104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255137:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.809705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419923255051:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.810831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255126:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.810840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241419923255126:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.811497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606167:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.811854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241422514606163:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.811939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241422514606098:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.811953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.811958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241422514606098:2068] [node 4] ICP22 created new session: [5:7587241422514606168:2068]
2025-12-24T02:11:43.811964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241422514606168:2068] [node 4] ICS09 handshake done sender: [5:7587241422514606163:2119] self: [5:7587241422514606166:0] peer: [4:7587241419923255095:0] socket: 411 qp: -1
2025-12-24T02:11:43.811969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241422514606168:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:43.811976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.811598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241418765254845:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.813057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241418765254832:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.813072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.813078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241418765254832:2111] [node 4] ICP22 created new session: [2:7587241418765254868:2111]
2025-12-24T02:11:43.813083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241418765254868:2111] [node 4] ICS09 handshake done sender: [2:7587241418765254845:2116] self: [2:7587241418765254844:0] peer: [4:7587241419923255141:0] socket: 441 qp: -1
2025-12-24T02:11:43.813089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241418765254868:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:43.813097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.813853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241420411415980:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241420411416069:2120] held: no
2025-12-24T02:11:43.811669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241419923255126:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.814863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241420325873839:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.814997Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241420325873763:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241420325873832:2112] held: no
2025-12-24T02:11:43.815109Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241420325873762:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241420325873823:2108] held: no
2025-12-24T02:11:43.815219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241419923255091:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.815248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.815256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419923255091:2086] [node 5] ICP22 created new session: [4:7587241419923255143:2086]
2025-12-24T02:11:43.815261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241419923255143:2086] [node 5] ICS09 handshake done sender: [4:7587241419923255126:2113] self: [4:7587241419923255095:0] peer: [5:7587241422514606166:0] socket: 399 qp: -1
2025-12-24T02:11:43.815265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241419923255143:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:43.815272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.816670Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873855:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.816754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241420325873829:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.816769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.816774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420325873829:2111] [node 6] ICP22 created new session: [3:7587241420325873856:2111]
2025-12-24T02:11:43.816779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241420325873856:2111] [node 6] ICS09 handshake done sender: [3:7587241420325873839:2116] self: [3:7587241420325873838:0] peer: [6:7587241420411416070:0] socket: 438 qp: -1
2025-12-24T02:11:43.816783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241420325873856:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:43.816788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.816959Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.817123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416073:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.818382Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241420325873823:2108] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.818473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241420325873858:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41590
2025-12-24T02:11:43.815635Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419923255050:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241419923255136:2119] held: no
2025-12-24T02:11:43.818609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241420325873762:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.818624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.818630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420325873762:2048] [node 1] ICP22 created new session: [3:7587241420325873860:2048]
2025-12-24T02:11:43.818637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241420325873860:2048] [node 1] ICS09 handshake done sender: [3:7587241420325873823:2108] self: [3:7587241420325873845:0] peer: [1:7587241419188970350:0] socket: 386 qp: -1
2025-12-24T02:11:43.818643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241420325873860:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.818650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:43.818955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241420325873832:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.819806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241420325873763:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.819822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.819829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420325873763:2049] [node 2] ICP22 created new session: [3:7587241420325873862:2049]
2025-12-24T02:11:43.819835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241420325873862:2049] [node 2] ICS09 handshake done sender: [3:7587241420325873832:2112] self: [3:7587241420325873844:0] peer: [2:7587241418765254826:0] socket: 393 qp: -1
2025-12-24T02:11:43.819841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241420325873862:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.819848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.820313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241420411415981:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30370) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241420411416066:2119] held: no
2025-12-24T02:11:43.820529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241420411416069:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.820610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241420411416066:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.820675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241420411415980:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.820689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.820695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241420411415980:2049] [node 2] ICP22 created new session: [6:7587241420411416074:2049]
2025-12-24T02:11:43.820701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241420411416074:2049] [node 2] ICS09 handshake done sender: [6:7587241420411416069:2120] self: [6:7587241420411416071:0] peer: [2:7587241418765254840:0] socket: 406 qp: -1
2025-12-24T02:11:43.820707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241420411416074:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.820712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.820865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241420411415981:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.820870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.820874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241420411415981:2050] [node 3] ICP22 created new session: [6:7587241420411416076:2050]
2025-12-24T02:11:43.820878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241420411416076:2050] [node 3] ICS09 handshake done sender: [6:7587241420411416066:2119] self: [6:7587241420411416070:0] peer: [3:7587241420325873838:0] socket: 440 qp: -1
2025-12-24T02:11:43.820882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241420411416076:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.820888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.827422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419923255051:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25085) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241419923255137:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.827891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255146:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.827940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241419923255147:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56350
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.828073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416078:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.828409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606170:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.829280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241419923255137:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:43.830408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419923255051:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.830436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:43.830445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419923255051:2049] [node 2] ICP22 created new session: [4:7587241419923255148:2049]
2025-12-24T02:11:43.830454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241419923255148:2049] [node 2] ICS09 handshake done sender: [4:7587241419923255137:2120] self: [4:7587241419923255141:0] peer: [2:7587241418765254844:0] socket: 422 qp: -1
2025-12-24T02:11:43.830471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241419923255148:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:43.830348Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241422514606170:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.830482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:43.830407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241422514606075:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.830514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241419923255136:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:43.830628Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241420325873843:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.830643Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241420325873843:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:43.830975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419923255050:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.830991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:43.830997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419923255050:2048] [node 1] ICP22 created new session: [4:7587241419923255150:2048]
2025-12-24T02:11:43.831001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241419923255150:2048] [node 1] ICS09 handshake done sender: [4:7587241419923255136:2119] self: [4:7587241419923255140:0] peer: [1:7587241419188970364:0] socket: 416 qp: -1
2025-12-24T02:11:43.831006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241419923255150:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:43.831014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.831352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241422514606172:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34360
2025-12-24T02:11:43.831589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241422514606170:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:43.831787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241422514606075:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.831805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:43.831812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241422514606075:2050] [node 3] ICP22 created new session: [5:7587241422514606173:2050]
2025-12-24T02:11:43.831818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241422514606173:2050] [node 3] ICS09 handshake done sender: [5:7587241422514606170:2121] self: [5:7587241422514606171:0] peer: [3:7587241420325873831:0] socket: 381 qp: -1
2025-12-24T02:11:43.831825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241422514606173:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:43.831833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:43.831462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241420325873843:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:43.831760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241420325873828:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.831801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:43.831811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420325873828:2110] [node 5] ICP22 created new session: [3:7587241420325873865:2110]
2025-12-24T02:11:43.831818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241420325873865:2110] [node 5] ICS09 handshake done sender: [3:7587241420325873843:2117] self: [3:7587241420325873831:0] peer: [5:7587241422514606171:0] socket: 379 qp: -1
2025-12-24T02:11:43.831824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241420325873865:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:43.831832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:43.832698Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241420411416078:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.832746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241420411416020:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:43.833206Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241419923255127:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:43.833221Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241419923255127:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:43.833816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241419923255127:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:43.833785Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241420411416080:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64376
2025-12-24T02:11:43.833912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241419923255094:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:43.833934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:43.833940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419923255094:2089] [node 6] ICP22 created new session: [4:7587241419923255153:2089]
2025-12-24T02:11:43.833948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241419923255153:2089] [node 6] ICS09 handshake done sender: [4:7587241419923255127:2114] self: [4:7587241419923255096:0] peer: [6:7587241420411416079:0] socket: 397 qp: -1
2025-12-24T02:11:43.833953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241419923255153:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:43.833963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:43.834170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241420411416078:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:43.834292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241420411416020:2087] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:43.834314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:43.834319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241420411416020:2087] [node 4] ICP22 created new session: [6:7587241420411416081:2087]
2025-12-24T02:11:43.834324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241420411416081:2087] [node 4] ICS09 handshake done sender: [6:7587241420411416078:2123] self: [6:7587241420411416079:0] peer: [4:7587241419923255096:0] socket: 400 qp: -1
2025-12-24T02:11:43.834329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241420411416081:2087] [node 4] ICS10 traffic start
2025-12-24T02:11:43.834336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:43.847302Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:43.847414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241418765254853:2048] [node 1] ICS01 socket: 387 reason# EndOfStream
2025-12-24T02:11:43.847427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241418765254764:2048] [node 1] ICP30 unregister session Session# [2:7587241418765254853:2048] VirtualId# [2:7587241418765254839:0]
2025-12-24T02:11:43.847435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241418765254853:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.847476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.847502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.850948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:43.851053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241420325873860:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:43.851062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241420325873762:2048] [node 1] ICP30 unregister session Session# [3:7587241420325873860:2048] VirtualId# [3:7587241420325873845:0]
2025-12-24T02:11:43.851070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241420325873860:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.851122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.851148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.854948Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:43.855045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241419923255150:2048] [node 1] ICS01 socket: 416 reason# EndOfStream
2025-12-24T02:11:43.855053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241419923255050:2048] [node 1] ICP30 unregister session Session# [4:7587241419923255150:2048] VirtualId# [4:7587241419923255140:0]
2025-12-24T02:11:43.855060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241419923255150:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.855104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.855129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.855284Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:43.855364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241420411416067:2048] [node 1] ICS01 socket: 414 reason# EndOfStream
2025-12-24T02:11:43.855370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241420411415979:2048] [node 1] ICP30 unregister session Session# [6:7587241420411416067:2048] VirtualId# [6:7587241420411416059:0]
2025-12-24T02:11:43.855376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241420411416067:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.855407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.855439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.855194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:43.855278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241422514606153:2048] [node 1] ICS07 socket disconnect 396 reason# EndOfStream
2025-12-24T02:11:43.855284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241422514606153:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.855339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:43.855364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241422514606153:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:43.855904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241422514606073:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15423) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:43.855915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241422514606153:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:43.855920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241422514606073:2048] [node 1] ICP30 unregister session Session# [5:7587241422514606153:2048] VirtualId# [5:7587241422514606150:0]
2025-12-24T02:11:43.855975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:43.855982Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241422514606073:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:43.855986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15423) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:43.863374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241422514606073:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:43.872863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:43.872974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241420325873862:2049] [node 2] ICS01 socket: 393 reason# EndOfStream
2025-12-24T02:11:43.872985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241420325873763:2049] [node 2] ICP30 unregister session Session# [3:7587241420325873862:2049] VirtualId# [3:7587241420325873844:0]
2025-12-24T02:11:43.872993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241420325873862:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.873035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.873067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.886841Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:43.886963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241422514606164:2049] [node 2] ICS01 socket: 410 reason# EndOfStream
2025-12-24T02:11:43.886974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241422514606074:2049] [node 2] ICP30 unregister session Session# [5:7587241422514606164:2049] VirtualId# [5:7587241422514606158:0]
2025-12-24T02:11:43.886984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241422514606164:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.887041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.887070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.887602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:43.887695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241420411416074:2049] [node 2] ICS01 socket: 406 reason# EndOfStream
2025-12-24T02:11:43.887704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241420411415980:2049] [node 2] ICP30 unregister session Session# [6:7587241420411416074:2049] VirtualId# [6:7587241420411416071:0]
2025-12-24T02:11:43.887712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241420411416074:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.887783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.887811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.887853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:43.887934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241419923255148:2049] [node 2] ICS01 socket: 422 reason# EndOfStream
2025-12-24T02:11:43.887939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241419923255051:2049] [node 2] ICP30 unregister session Session# [4:7587241419923255148:2049] VirtualId# [4:7587241419923255141:0]
2025-12-24T02:11:43.887944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241419923255148:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.887971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:43.887994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:43.902001Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:43.902133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241422514606173:2050] [node 3] ICS01 socket: 381 reason# EndOfStream
2025-12-24T02:11:43.902144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241422514606075:2050] [node 3] ICP30 unregister session Session# [5:7587241422514606173:2050] VirtualId# [5:7587241422514606171:0]
2025-12-24T02:11:43.902153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241422514606173:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.902213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.902243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.907060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:43.907171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241420411416076:2050] [node 3] ICS01 socket: 440 reason# EndOfStream
2025-12-24T02:11:43.907182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241420411415981:2050] [node 3] ICP30 unregister session Session# [6:7587241420411416076:2050] VirtualId# [6:7587241420411416070:0]
2025-12-24T02:11:43.907190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241420411416076:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.907253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:43.907281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:43.918342Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:43.918455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241420411416081:2087] [node 4] ICS01 socket: 400 reason# EndOfStream
2025-12-24T02:11:43.918467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241420411416020:2087] [node 4] ICP30 unregister session Session# [6:7587241420411416081:2087] VirtualId# [6:7587241420411416079:0]
2025-12-24T02:11:43.918477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241420411416081:2087] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:43.918528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:43.918555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:43.998871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420609403369:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:43.998891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241420609403370:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:43.998708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241420540547212:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:43.998718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241420540547213:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:44.003555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419290969645:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.003566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419290969646:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.003572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241419290969647:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.002978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420840661646:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.002988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241420840661647:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.005808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241425335121108:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.005819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241425335121109:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.005834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241425335121110:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.019036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426611540873:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.019208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426611540871:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.026859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426611540872:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.057216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241425135628982:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:44.057329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241425135628983:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:44.059205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241424835514569:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:44.058934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241425135628992:2089] [node 6] ICP01 ready to work
2025-12-24T02:11:44.073543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241423585936991:2094] [node 6] ICP01 ready to work
2025-12-24T02:11:44.075442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426611540924:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:44.076054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241425335121169:2080] [node 6] ICP01 ready to work
2025-12-24T02:11:44.078084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424904370728:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:44.078987Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241419290969645:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20325957ns) {Configure:104(20324840ns) {ProcessConfigured:135(19950785ns) {StartInitialHandshake:157(19950262ns) {DropHandshakes:470(540ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(523ns) PrepareNewSessionHandshake:477(1244ns) {SwitchToState:179(556ns)}}}}}
2025-12-24T02:11:44.082366Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241425135628992:2089] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23315899ns) {Configure:104(23315076ns) {ProcessConfigured:135(23047671ns) {StartInitialHandshake:157(23047323ns) {DropHandshakes:470(350ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(1317ns) {SwitchToState:179(712ns)}}}}}
2025-12-24T02:11:44.082733Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241420840661646:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23741372ns) {Configure:104(23740255ns) {ProcessConfigured:135(23222672ns) {StartInitialHandshake:157(23222548ns) {DropHandshakes:470(152ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(75ns) PrepareNewSessionHandshake:477(573ns) {SwitchToState:179(26ns)}}}}}
2025-12-24T02:11:44.082162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241424835514577:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:44.082882Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241424835514569:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23609794ns) {Configure:104(23609529ns) {ProcessConfigured:135(23226229ns) {StartInitialHandshake:157(23225905ns) {DropHandshakes:470(236ns) {DropIncomingHandshake:442(38ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(137ns) PrepareNewSessionHandshake:477(638ns) {SwitchToState:179(157ns)}}}}}
2025-12-24T02:11:44.080633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424904370730:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:44.083195Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426611540873:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21202975ns) {Configure:104(21201765ns) {ProcessConfigured:135(20737650ns) {StartInitialHandshake:157(20737551ns) {DropHandshakes:470(858ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(336ns) PrepareNewSessionHandshake:477(1018ns) {SwitchToState:179(72ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.091536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629021:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
2025-12-24T02:11:44.091567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629022:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.091122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241419290969646:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17479735ns) {Configure:104(17478747ns) {ProcessConfigured:135(17038158ns) {StartInitialHandshake:157(17037364ns) {DropHandshakes:470(1671ns) {DropIncomingHandshake:442(169ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(305ns) PrepareNewSessionHandshake:477(1906ns) {SwitchToState:179(922ns)}}}}}
2025-12-24T02:11:44.091615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629023:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
2025-12-24T02:11:44.091787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241423585936991:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(17604684ns) {Configure:104(17603777ns) {ProcessConfigured:135(17265870ns) {StartInitialHandshake:157(17265530ns) {DropHandshakes:470(477ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(796ns) {SwitchToState:179(425ns)}}}}}
2025-12-24T02:11:44.093305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629022:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.094992Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426611540872:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19414522ns) {Configure:104(19413824ns) {ProcessConfigured:135(19013118ns) {StartInitialHandshake:157(19012672ns) {DropHandshakes:470(727ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(1992ns) {SwitchToState:179(1117ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.095593Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121183:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.095657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121184:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.095694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121185:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.095347Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540942:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.098759Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629025:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
2025-12-24T02:11:44.098974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241420609403370:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20775047ns) {Configure:104(20774507ns) {ProcessConfigured:135(20377615ns) {StartInitialHandshake:157(20377315ns) {DropHandshakes:470(362ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(1395ns) {SwitchToState:179(1053ns)}}}}}
2025-12-24T02:11:44.098495Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241425335121169:2080] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20951030ns) {Configure:104(20949722ns) {ProcessConfigured:135(20543047ns) {StartInitialHandshake:157(20542419ns) {DropHandshakes:470(486ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(275ns) PrepareNewSessionHandshake:477(1709ns) {SwitchToState:179(1168ns)}}}}}
2025-12-24T02:11:44.098948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241425335121183:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.098986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241425335121110:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.102904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937016:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.102849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241425335121184:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.102901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241425335121109:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.103378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937016:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.103415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419290969646:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.106185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629025:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.107879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424904370727:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:44.110187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629004:2101] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.110205Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241425135629004:2101] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.110841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937015:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.110724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426611540923:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:44.110763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370732:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.110782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424904370732:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.111263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241424835514576:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:44.112202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241425335121168:2078] [node 4] ICP01 ready to work
2025-12-24T02:11:44.115009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937015:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.115067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419290969647:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.115241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629008:2103] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.115252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241425135629008:2103] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.115933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241425135629008:2103] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.116039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241425135628982:2084] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.116064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.116071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241425135628982:2084] [node 4] ICP22 created new session: [3:7587241425135629028:2084]
2025-12-24T02:11:44.116079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241425135629028:2084] [node 4] ICS09 handshake done sender: [3:7587241425135629008:2103] self: [3:7587241425135628990:0] peer: [4:7587241423585937018:0] socket: 391 qp: -1
2025-12-24T02:11:44.116087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241425135629028:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:44.116098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.117837Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426611540924:2067] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34553263ns) {Configure:104(34552680ns) {ProcessConfigured:135(34185810ns) {StartInitialHandshake:157(34185619ns) {DropHandshakes:470(606ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(1402ns) {SwitchToState:179(62ns)}}}}}
2025-12-24T02:11:44.116536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241424835514584:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:791
2025-12-24T02:11:44.117152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241424835514585:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:791
2025-12-24T02:11:44.117219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241424835514586:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:791
2025-12-24T02:11:44.117485Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241420540547212:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(30590577ns) {Configure:104(30589057ns) {ProcessConfigured:135(30201938ns) {StartInitialHandshake:157(30201515ns) {DropHandshakes:470(1156ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(472ns) PrepareNewSessionHandshake:477(841ns) {SwitchToState:179(365ns)}}}}}
2025-12-24T02:11:44.119326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370736:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.119345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424904370736:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.120206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241423585937020:2115] [node 5] ICP01 ready to work
2025-12-24T02:11:44.119942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514584:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.119969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514585:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.123680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426611540871:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61695570ns) {Configure:104(61694192ns) {ProcessConfigured:135(60972501ns) {StartInitialHandshake:157(60972303ns) {DropHandshakes:470(344ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(140ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(610ns) {SwitchToState:179(278ns)}}}}}
2025-12-24T02:11:44.123724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426611540942:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.123833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426611540871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.123976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514586:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.126973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370744:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.127055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370745:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.127095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370746:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.127132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370747:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.127969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121191:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.128388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241425335121183:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.129502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241425335121110:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.129525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.129533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241425335121110:2050] [node 3] ICP22 created new session: [5:7587241425335121192:2050]
2025-12-24T02:11:44.129539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241425335121192:2050] [node 3] ICS09 handshake done sender: [5:7587241425335121183:2113] self: [5:7587241425335121187:0] peer: [3:7587241425135628989:0] socket: 395 qp: -1
2025-12-24T02:11:44.129546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241425335121192:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.129552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.130847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370744:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.130882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370745:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.127889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629023:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.127976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241425135629004:2101] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.129181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241425135628983:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.129202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.129210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241425135628983:2086] [node 5] ICP22 created new session: [3:7587241425135629033:2086]
2025-12-24T02:11:44.129218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241425135629033:2086] [node 5] ICS09 handshake done sender: [3:7587241425135629004:2101] self: [3:7587241425135628989:0] peer: [5:7587241425335121187:0] socket: 388 qp: -1
2025-12-24T02:11:44.129227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241425135629033:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:44.129238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.131941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424904370732:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.132104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424904370730:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.132125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.132131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424904370730:2106] [node 5] ICP22 created new session: [2:7587241424904370751:2106]
2025-12-24T02:11:44.132137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241424904370751:2106] [node 5] ICS09 handshake done sender: [2:7587241424904370732:2109] self: [2:7587241424904370731:0] peer: [5:7587241425335121188:0] socket: 399 qp: -1
2025-12-24T02:11:44.132143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241424904370751:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:44.132150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.132903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241424835514593:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:791
2025-12-24T02:11:44.135605Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426611540923:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24698565ns) {Configure:104(24697266ns) {ProcessConfigured:135(24315280ns) {StartInitialHandshake:157(24314253ns) {DropHandshakes:470(1007ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(111ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(1115ns) {SwitchToState:179(289ns)}}}}}
2025-12-24T02:11:44.136138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514580:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.136151Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241424835514580:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.136444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370747:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.138021Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241425335121109:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241425335121184:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.143700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370746:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.143746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424904370736:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.143859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424904370728:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.143876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.143886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424904370728:2105] [node 4] ICP22 created new session: [2:7587241424904370756:2105]
2025-12-24T02:11:44.143892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241424904370756:2105] [node 4] ICS09 handshake done sender: [2:7587241424904370736:2111] self: [2:7587241424904370735:0] peer: [4:7587241423585937017:0] socket: 402 qp: -1
2025-12-24T02:11:44.143899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241424904370756:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:44.143909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.144159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241424904370728:2105] [node 4] ICP18 handshake (actor [2:7587241424904370746:2117]) is too old
2025-12-24T02:11:44.143134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937022:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.146373Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241425135628982:2084] [node 4] ICP35 event processing took too much time PendingConnection:279(30267854ns) {HandleHandshakeStatus:329(30267030ns) {DropIncomingHandshake:442(4815ns) DropOutgoingHandshake:457(257ns) SwitchToState:179(85ns) ProcessPendingSessionEvents:495(26857369ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 26845830ns max# 26808482ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.147685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.147698Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.147753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.146848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937023:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.147437Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241423585937020:2115] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27168115ns) {Configure:104(27167524ns) {ProcessConfigured:135(26744412ns) {StartInitialHandshake:157(26743476ns) {DropHandshakes:470(637ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(203ns)} GenerateSessionVirtualId:988(92ns) PrepareNewSessionHandshake:477(734ns) {SwitchToState:179(128ns)}}}}}
2025-12-24T02:11:44.147461Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419290969646:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241423585937016:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.148899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937024:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.149100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241423585937016:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.150402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419290969646:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.150421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.150429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419290969646:2049] [node 2] ICP22 created new session: [4:7587241423585937027:2049]
2025-12-24T02:11:44.150435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241423585937027:2049] [node 2] ICS09 handshake done sender: [4:7587241423585937016:2114] self: [4:7587241423585937017:0] peer: [2:7587241424904370735:0] socket: 418 qp: -1
2025-12-24T02:11:44.150443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241423585937027:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.150450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.150488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370743:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.150498Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424904370743:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.151014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424904370743:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.151084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629021:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.151125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241420840661646:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.151095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937023:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.151305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424904370727:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.151327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.151333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424904370727:2108] [node 6] ICP22 created new session: [2:7587241424904370760:2108]
2025-12-24T02:11:44.151339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241424904370760:2108] [node 6] ICS09 handshake done sender: [2:7587241424904370743:2114] self: [2:7587241424904370742:0] peer: [6:7587241426611540955:0] socket: 437 qp: -1
2025-12-24T02:11:44.151344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241424904370760:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:44.151351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.149859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.150228Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426611540953:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.150251Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426611540951:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.150278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426611540872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.150289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426611540873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.151147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.151398Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426611540954:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.151424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426611540924:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.151631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426611540952:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.151650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426611540923:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.152247Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426611540872:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426611540953:2118] held: no
2025-12-24T02:11:44.152400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426611540953:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.152505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241425335121185:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.152540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241425335121186:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.152543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241425335121186:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.153057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241425335121108:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.154304Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241425335121186:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.154818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937014:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.154830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241423585937014:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.155241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426611540872:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.155256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.155264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426611540872:2049] [node 2] ICP22 created new session: [6:7587241426611540960:2049]
2025-12-24T02:11:44.155274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426611540960:2049] [node 2] ICS09 handshake done sender: [6:7587241426611540953:2118] self: [6:7587241426611540955:0] peer: [2:7587241424904370742:0] socket: 439 qp: -1
2025-12-24T02:11:44.155280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426611540960:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.155297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.151712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514573:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.151723Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241424835514573:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.152457Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241424835514573:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.153214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514579:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.153218Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241424835514579:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.153653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241420540547213:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.153663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.153669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241420540547213:2049] [node 3] ICP22 created new session: [1:7587241424835514595:2049]
2025-12-24T02:11:44.153675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241424835514595:2049] [node 3] ICS09 handshake done sender: [1:7587241424835514573:2109] self: [1:7587241424835514572:0] peer: [3:7587241425135629036:0] socket: 390 qp: -1
2025-12-24T02:11:44.153680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241424835514595:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:44.153698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.154141Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241424835514579:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.154218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241424835514577:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.154232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.154238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241424835514577:2110] [node 5] ICP22 created new session: [1:7587241424835514599:2110]
2025-12-24T02:11:44.154243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241424835514599:2110] [node 5] ICS09 handshake done sender: [1:7587241424835514579:2111] self: [1:7587241424835514578:0] peer: [5:7587241425335121196:0] socket: 405 qp: -1
2025-12-24T02:11:44.154247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241424835514599:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:44.154252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.157066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419290969647:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241423585937015:2113] held: no
2025-12-24T02:11:44.157258Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241419290969647:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(36361821ns) {HandleHandshakeStatus:395(36361105ns) {LogHandshakeFail:472(3333ns) {UpdateErrorStateLog:545(996ns)} DropOutgoingHandshake:457(670ns) ProcessPendingSessionEvents:495(81097ns) {unordered DropSessionEvent:512 num# 90 sum# 75589ns max# 3708ns}}}
2025-12-24T02:11:44.157334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241423585937015:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.158717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629018:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.158738Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241425135629018:2112] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.162162Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241425335121108:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241425335121185:2115] held: no
2025-12-24T02:11:44.162370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241425335121169:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.162392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.162400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241425335121169:2080] [node 6] ICP22 created new session: [5:7587241425335121197:2080]
2025-12-24T02:11:44.162408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241425335121197:2080] [node 6] ICS09 handshake done sender: [5:7587241425335121186:2116] self: [5:7587241425335121176:0] peer: [6:7587241426611540959:0] socket: 430 qp: -1
2025-12-24T02:11:44.162416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241425335121197:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:44.162425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.162975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629037:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
2025-12-24T02:11:44.163024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629038:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.163769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540962:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.163141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121201:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.164178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121200:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.164462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241425335121185:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.164534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241425335121108:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.164552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.164558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241425335121108:2048] [node 1] ICP22 created new session: [5:7587241425335121202:2048]
2025-12-24T02:11:44.164565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241425335121202:2048] [node 1] ICS09 handshake done sender: [5:7587241425335121185:2115] self: [5:7587241425335121196:0] peer: [1:7587241424835514578:0] socket: 415 qp: -1
2025-12-24T02:11:44.164571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241425335121202:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.164580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.165028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241425335121184:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.165106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241425335121109:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.165117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.165122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241425335121109:2049] [node 2] ICP22 created new session: [5:7587241425335121204:2049]
2025-12-24T02:11:44.165128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241425335121204:2049] [node 2] ICS09 handshake done sender: [5:7587241425335121184:2114] self: [5:7587241425335121188:0] peer: [2:7587241424904370731:0] socket: 414 qp: -1
2025-12-24T02:11:44.165133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241425335121204:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.165139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.166956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426611540952:2117] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.169767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241425135629037:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.169812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241420840661647:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.169359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514593:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.169524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241420540547213:2049] [node 3] ICP18 handshake (actor [1:7587241424835514593:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.170043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370741:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.170056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424904370741:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.169977Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241425135629021:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.167389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419290969647:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.167411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.167420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419290969647:2050] [node 3] ICP22 created new session: [4:7587241423585937031:2050]
2025-12-24T02:11:44.167428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241423585937031:2050] [node 3] ICS09 handshake done sender: [4:7587241423585937015:2113] self: [4:7587241423585937018:0] peer: [3:7587241425135628990:0] socket: 392 qp: -1
2025-12-24T02:11:44.167435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241423585937031:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.167456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.169884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937034:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.169897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937035:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.173997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241423585937014:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.174076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937035:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.175173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241423585936991:2094] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.175202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.175212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241423585936991:2094] [node 6] ICP22 created new session: [4:7587241423585937037:2094]
2025-12-24T02:11:44.175218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241423585937037:2094] [node 6] ICS09 handshake done sender: [4:7587241423585937014:2112] self: [4:7587241423585937004:0] peer: [6:7587241426611540958:0] socket: 408 qp: -1
2025-12-24T02:11:44.175238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241423585937037:2094] [node 6] ICS10 traffic start
2025-12-24T02:11:44.175247Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.179427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241424835514580:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.179648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241424835514569:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.179661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.179670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241424835514569:2107] [node 6] ICP22 created new session: [1:7587241424835514603:2107]
2025-12-24T02:11:44.179677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241424835514603:2107] [node 6] ICS09 handshake done sender: [1:7587241424835514580:2112] self: [1:7587241424835514570:0] peer: [6:7587241426611540949:0] socket: 407 qp: -1
2025-12-24T02:11:44.179683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241424835514603:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:44.179691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.180183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937034:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.180226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241419290969645:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.179224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426611540923:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.179252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.179262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426611540923:2077] [node 5] ICP22 created new session: [6:7587241426611540963:2077]
2025-12-24T02:11:44.179269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426611540963:2077] [node 5] ICS09 handshake done sender: [6:7587241426611540952:2117] self: [6:7587241426611540959:0] peer: [5:7587241425335121176:0] socket: 438 qp: -1
2025-12-24T02:11:44.179277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426611540963:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:44.179303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.182955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121206:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.180874Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241420840661646:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241425135629021:2115] held: no
2025-12-24T02:11:44.180981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241420840661646:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.180992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.180998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420840661646:2048] [node 1] ICP22 created new session: [3:7587241425135629040:2048]
2025-12-24T02:11:44.181006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241425135629040:2048] [node 1] ICS09 handshake done sender: [3:7587241425135629021:2115] self: [3:7587241425135629036:0] peer: [1:7587241424835514572:0] socket: 393 qp: -1
2025-12-24T02:11:44.181011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241425135629040:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.181019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.181872Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241424904370727:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(30457214ns) {HandleHandshakeStatus:329(30456774ns) {DropIncomingHandshake:442(3758ns) DropOutgoingHandshake:457(125ns) SwitchToState:179(156ns) ProcessPendingSessionEvents:495(22855608ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22831876ns max# 22755129ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.186747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370762:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.188116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241425135629018:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.189270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241425135628992:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.189290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.189295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241425135628992:2089] [node 6] ICP22 created new session: [3:7587241425135629043:2089]
2025-12-24T02:11:44.189301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241425135629043:2089] [node 6] ICS09 handshake done sender: [3:7587241425135629018:2112] self: [3:7587241425135628996:0] peer: [6:7587241426611540956:0] socket: 423 qp: -1
2025-12-24T02:11:44.189306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241425135629043:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:44.189316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.189174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424904370762:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.189217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241420609403369:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.190288Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424904370764:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49512
2025-12-24T02:11:44.186961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514588:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.186977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241424835514588:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.189427Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241424835514588:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.189553Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241424835514589:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.189556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241424835514589:2117] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.189698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241424835514576:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.189713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.189719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241424835514576:2108] [node 4] ICP22 created new session: [1:7587241424835514606:2108]
2025-12-24T02:11:44.189725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241424835514606:2108] [node 4] ICS09 handshake done sender: [1:7587241424835514588:2116] self: [1:7587241424835514587:0] peer: [4:7587241423585937039:0] socket: 444 qp: -1
2025-12-24T02:11:44.189730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241424835514606:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:44.189736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.190193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241424835514589:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.190428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241420540547212:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.190455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.190463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241420540547212:2048] [node 2] ICP22 created new session: [1:7587241424835514609:2048]
2025-12-24T02:11:44.190469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241424835514609:2048] [node 2] ICS09 handshake done sender: [1:7587241424835514589:2117] self: [1:7587241424835514583:0] peer: [2:7587241424904370763:0] socket: 446 qp: -1
2025-12-24T02:11:44.190474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241424835514609:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:44.190481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.188398Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426611540924:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426611540954:2119] held: no
2025-12-24T02:11:44.188886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540965:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.192355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424904370741:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.192446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241420609403369:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241424904370762:2119] held: no
2025-12-24T02:11:44.192534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241420609403370:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.192549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.192554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420609403370:2049] [node 3] ICP22 created new session: [2:7587241424904370766:2049]
2025-12-24T02:11:44.192560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241424904370766:2049] [node 3] ICS09 handshake done sender: [2:7587241424904370741:2113] self: [2:7587241424904370729:0] peer: [3:7587241425135629039:0] socket: 425 qp: -1
2025-12-24T02:11:44.192566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241424904370766:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:44.192572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.193362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424904370762:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.193453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241420609403369:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.193466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.193472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241420609403369:2048] [node 1] ICP22 created new session: [2:7587241424904370768:2048]
2025-12-24T02:11:44.193477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241424904370768:2048] [node 1] ICS09 handshake done sender: [2:7587241424904370762:2119] self: [2:7587241424904370763:0] peer: [1:7587241424835514583:0] socket: 447 qp: -1
2025-12-24T02:11:44.193483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241424904370768:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.193489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.194641Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241419290969645:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5891) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241423585937034:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.195361Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426611540873:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29782) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426611540951:2116] held: no
2025-12-24T02:11:44.195520Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540966:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.195868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426611540967:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46641
2025-12-24T02:11:44.196657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426611540954:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.196777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426611540951:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.198003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426611540873:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.198029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.198037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426611540873:2050] [node 3] ICP22 created new session: [6:7587241426611540969:2050]
2025-12-24T02:11:44.198045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426611540969:2050] [node 3] ICS09 handshake done sender: [6:7587241426611540951:2116] self: [6:7587241426611540956:0] peer: [3:7587241425135628996:0] socket: 409 qp: -1
2025-12-24T02:11:44.198052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426611540969:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.198061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.198294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426611540942:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.198508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426611540924:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.198520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.198526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426611540924:2067] [node 4] ICP22 created new session: [6:7587241426611540971:2067]
2025-12-24T02:11:44.198531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426611540971:2067] [node 4] ICS09 handshake done sender: [6:7587241426611540954:2119] self: [6:7587241426611540958:0] peer: [4:7587241423585937004:0] socket: 440 qp: -1
2025-12-24T02:11:44.198535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426611540971:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:44.198540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.198655Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241420840661647:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26817) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241425135629037:2119] held: no
2025-12-24T02:11:44.199072Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241423585937040:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17473
2025-12-24T02:11:44.199120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241425135629045:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22132
2025-12-24T02:11:44.199059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241425335121206:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.199115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241425335121168:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.199328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241423585937034:2120] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.199488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241424835514611:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:791
2025-12-24T02:11:44.199626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241425135629037:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.199719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241419290969645:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.199735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.199741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241419290969645:2048] [node 1] ICP22 created new session: [4:7587241423585937041:2048]
2025-12-24T02:11:44.199748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241423585937041:2048] [node 1] ICS09 handshake done sender: [4:7587241423585937034:2120] self: [4:7587241423585937039:0] peer: [1:7587241424835514587:0] socket: 386 qp: -1
2025-12-24T02:11:44.199753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241423585937041:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.199762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.200257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241420840661647:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.199442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426611540871:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.199460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.199464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426611540871:2048] [node 1] ICP22 created new session: [6:7587241426611540973:2048]
2025-12-24T02:11:44.199469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426611540973:2048] [node 1] ICS09 handshake done sender: [6:7587241426611540942:2112] self: [6:7587241426611540949:0] peer: [1:7587241424835514570:0] socket: 411 qp: -1
2025-12-24T02:11:44.199474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426611540973:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.199480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.200285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.200291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241420840661647:2049] [node 2] ICP22 created new session: [3:7587241425135629046:2049]
2025-12-24T02:11:44.200297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241425135629046:2049] [node 2] ICS09 handshake done sender: [3:7587241425135629037:2119] self: [3:7587241425135629039:0] peer: [2:7587241424904370729:0] socket: 433 qp: -1
2025-12-24T02:11:44.200302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241425135629046:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.200308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.204129Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241424835514569:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:44.207640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241424835514576:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(17809173ns) {HandleHandshakeStatus:329(17808673ns) {DropIncomingHandshake:442(3273ns) DropOutgoingHandshake:457(94ns) SwitchToState:179(92ns) ProcessPendingSessionEvents:495(17742278ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17715842ns max# 17623568ns}}}
2025-12-24T02:11:44.208319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241423585937025:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.208340Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241423585937025:2119] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.209578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241425335121208:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62738
2025-12-24T02:11:44.212271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241423585937025:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.212405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241423585937020:2115] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.212432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.212441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241423585937020:2115] [node 5] ICP22 created new session: [4:7587241423585937044:2115]
2025-12-24T02:11:44.212448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241423585937044:2115] [node 5] ICS09 handshake done sender: [4:7587241423585937025:2119] self: [4:7587241423585937021:0] peer: [5:7587241425335121207:0] socket: 397 qp: -1
2025-12-24T02:11:44.212456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241423585937044:2115] [node 5] ICS10 traffic start
2025-12-24T02:11:44.212467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.213603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241425335121168:2078] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16708) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241425335121206:2121] held: no
2025-12-24T02:11:44.213714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241425335121206:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.213947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241425335121168:2078] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.213960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.213966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241425335121168:2078] [node 4] ICP22 created new session: [5:7587241425335121209:2078]
2025-12-24T02:11:44.213973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241425335121209:2078] [node 4] ICS09 handshake done sender: [5:7587241425335121206:2121] self: [5:7587241425335121207:0] peer: [4:7587241423585937021:0] socket: 452 qp: -1
2025-12-24T02:11:44.213979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241425335121209:2078] [node 4] ICS10 traffic start
2025-12-24T02:11:44.213991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.230799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:44.230914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241424904370768:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-12-24T02:11:44.230923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241420609403369:2048] [node 1] ICP30 unregister session Session# [2:7587241424904370768:2048] VirtualId# [2:7587241424904370763:0]
2025-12-24T02:11:44.230933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241424904370768:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.230966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.230998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.234356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:44.234458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241425135629040:2048] [node 1] ICS01 socket: 393 reason# EndOfStream
2025-12-24T02:11:44.234469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241420840661646:2048] [node 1] ICP30 unregister session Session# [3:7587241425135629040:2048] VirtualId# [3:7587241425135629036:0]
2025-12-24T02:11:44.234476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241425135629040:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.234530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.234573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.242876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:44.242972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241423585937041:2048] [node 1] ICS01 socket: 386 reason# EndOfStream
2025-12-24T02:11:44.242982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241419290969645:2048] [node 1] ICP30 unregister session Session# [4:7587241423585937041:2048] VirtualId# [4:7587241423585937039:0]
2025-12-24T02:11:44.242990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241423585937041:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.243033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.243069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.243108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:44.243170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241425335121202:2048] [node 1] ICS01 socket: 415 reason# EndOfStream
2025-12-24T02:11:44.243175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241425335121108:2048] [node 1] ICP30 unregister session Session# [5:7587241425335121202:2048] VirtualId# [5:7587241425335121196:0]
2025-12-24T02:11:44.243179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241425335121202:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.243203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.243223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.243654Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:44.243720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426611540973:2048] [node 1] ICS01 socket: 411 reason# EndOfStream
2025-12-24T02:11:44.243725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426611540871:2048] [node 1] ICP30 unregister session Session# [6:7587241426611540973:2048] VirtualId# [6:7587241426611540949:0]
2025-12-24T02:11:44.243730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426611540973:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.243766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.243796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.260674Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:44.260787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241425135629046:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:44.260798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241420840661647:2049] [node 2] ICP30 unregister session Session# [3:7587241425135629046:2049] VirtualId# [3:7587241425135629039:0]
2025-12-24T02:11:44.260808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241425135629046:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.260858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.260892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.262871Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:44.262961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241423585937027:2049] [node 2] ICS01 socket: 418 reason# EndOfStream
2025-12-24T02:11:44.262969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241419290969646:2049] [node 2] ICP30 unregister session Session# [4:7587241423585937027:2049] VirtualId# [4:7587241423585937017:0]
2025-12-24T02:11:44.262976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241423585937027:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.263023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.263047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.263085Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:44.263159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241425335121204:2049] [node 2] ICS01 socket: 414 reason# EndOfStream
2025-12-24T02:11:44.263165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241425335121109:2049] [node 2] ICP30 unregister session Session# [5:7587241425335121204:2049] VirtualId# [5:7587241425335121188:0]
2025-12-24T02:11:44.263170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241425335121204:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.263196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.263217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.263251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:44.263324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426611540960:2049] [node 2] ICS01 socket: 439 reason# EndOfStream
2025-12-24T02:11:44.263330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426611540872:2049] [node 2] ICP30 unregister session Session# [6:7587241426611540960:2049] VirtualId# [6:7587241426611540955:0]
2025-12-24T02:11:44.263335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426611540960:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.263364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.263383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.278849Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:44.278968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241423585937031:2050] [node 3] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:44.278980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241419290969647:2050] [node 3] ICP30 unregister session Session# [4:7587241423585937031:2050] VirtualId# [4:7587241423585937018:0]
2025-12-24T02:11:44.278988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241423585937031:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.279045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:44.279071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:44.279205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:44.279274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241425335121192:2050] [node 3] ICS07 socket disconnect 395 reason# EndOfStream
2025-12-24T02:11:44.279279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241425335121192:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.279307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:44.279328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241425335121192:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:44.280132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241425335121110:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29782) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:44.280142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241425335121192:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:44.280147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241425335121110:2050] [node 3] ICP30 unregister session Session# [5:7587241425335121192:2050] VirtualId# [5:7587241425335121187:0]
2025-12-24T02:11:44.280195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:44.280201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241425335121110:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29782) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.280204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29782) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.284199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:44.284286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426611540969:2050] [node 3] ICS01 socket: 409 reason# EndOfStream
2025-12-24T02:11:44.284292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426611540873:2050] [node 3] ICP30 unregister session Session# [6:7587241426611540969:2050] VirtualId# [6:7587241426611540956:0]
2025-12-24T02:11:44.284298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426611540969:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.284349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:44.284371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:44.294832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241425335121110:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:44.310961Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:44.311069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426611540971:2067] [node 4] ICS01 socket: 440 reason# EndOfStream
2025-12-24T02:11:44.311081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426611540924:2067] [node 4] ICP30 unregister session Session# [6:7587241426611540971:2067] VirtualId# [6:7587241426611540958:0]
2025-12-24T02:11:44.311090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426611540971:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.311138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:44.311162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:44.311204Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:44.311280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241425335121209:2078] [node 4] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:44.311287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241425335121168:2078] [node 4] ICP30 unregister session Session# [5:7587241425335121209:2078] VirtualId# [5:7587241425335121207:0]
2025-12-24T02:11:44.311293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241425335121209:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.311320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:44.311342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:44.386509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423802267899:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:44.386519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423802267900:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:44.401440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241423119311951:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.401448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241423119311952:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.399590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241423009311941:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.399617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241423009311942:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:44.414531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422760087336:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.414543Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422760087337:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.431377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241423435958815:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.438304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426345159758:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.443006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426345159756:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.443051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426345159757:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.414550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422760087338:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.443390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241423435958814:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.450937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241423435958813:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.451061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241423435958854:2085] [node 6] ICP01 ready to work
2025-12-24T02:11:44.451476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422760087382:2092] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.463369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312007:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.468030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426345159816:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:44.474974Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241423802267900:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24091979ns) {Configure:104(24090240ns) {ProcessConfigured:135(23662037ns) {StartInitialHandshake:157(23661362ns) {DropHandshakes:470(1117ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(2010ns) {SwitchToState:179(1127ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.475370Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241423119311952:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24420011ns) {Configure:104(24419017ns) {ProcessConfigured:135(24013872ns) {StartInitialHandshake:157(24013681ns) {DropHandshakes:470(618ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1029ns) {SwitchToState:179(593ns)}}}}}
2025-12-24T02:11:44.475826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312020:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.478803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241423435958814:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27883297ns) {Configure:104(27882328ns) {ProcessConfigured:135(26933433ns) {StartInitialHandshake:157(26932353ns) {DropHandshakes:470(468ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(1790ns) {SwitchToState:179(1160ns)}}}}}
2025-12-24T02:11:44.481263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312020:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.479033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422760087382:2092] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27465998ns) {Configure:104(27465352ns) {ProcessConfigured:135(27037765ns) {StartInitialHandshake:157(27037265ns) {DropHandshakes:470(428ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1902ns) {SwitchToState:179(1163ns)}}}}}
2025-12-24T02:11:44.479371Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241423435958854:2085] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28244444ns) {Configure:104(28243986ns) {ProcessConfigured:135(27355943ns) {StartInitialHandshake:157(27355170ns) {DropHandshakes:470(221ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:477(897ns) {SwitchToState:179(608ns)}}}}}
2025-12-24T02:11:44.482855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241423435958813:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27707967ns) {Configure:104(27707119ns) {ProcessConfigured:135(27336231ns) {StartInitialHandshake:157(27335003ns) {DropHandshakes:470(503ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(91ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(1152ns) {SwitchToState:179(375ns)}}}}}
2025-12-24T02:11:44.482172Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241423802267899:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27326429ns) {Configure:104(27325040ns) {ProcessConfigured:135(26950209ns) {StartInitialHandshake:157(26949472ns) {DropHandshakes:470(420ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(883ns) {SwitchToState:179(335ns)}}}}}
2025-12-24T02:11:44.483981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241423009311999:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:44.488207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241423009311941:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24158323ns) {Configure:104(24156381ns) {ProcessConfigured:135(23694338ns) {StartInitialHandshake:157(23692982ns) {DropHandshakes:470(1609ns) {DropIncomingHandshake:442(236ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(340ns) PrepareNewSessionHandshake:477(1416ns) {SwitchToState:179(798ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.490791Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426345159757:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25012183ns) {Configure:104(25010400ns) {ProcessConfigured:135(24560373ns) {StartInitialHandshake:157(24560087ns) {DropHandshakes:470(842ns) {DropIncomingHandshake:442(119ns) DropOutgoingHandshake:457(132ns)} GenerateSessionVirtualId:988(542ns) PrepareNewSessionHandshake:477(1033ns) {SwitchToState:179(655ns)}}}}}
2025-12-24T02:11:44.491101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159825:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.491119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159826:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.491462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426345159756:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24670495ns) {Configure:104(24667008ns) {ProcessConfigured:135(24334522ns) {StartInitialHandshake:157(24334171ns) {DropHandshakes:470(551ns) {DropIncomingHandshake:442(161ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(272ns) PrepareNewSessionHandshake:477(858ns) {SwitchToState:179(257ns)}}}}}
2025-12-24T02:11:44.495596Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241423119311951:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(28228522ns) {Configure:104(28227730ns) {ProcessConfigured:135(27833478ns) {StartInitialHandshake:157(27833001ns) {DropHandshakes:470(425ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(1562ns) {SwitchToState:179(1042ns)}}}}}
2025-12-24T02:11:44.495753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241423119311951:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.498401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423802267963:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:44.491834Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426345159758:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24552135ns) {Configure:104(24551775ns) {ProcessConfigured:135(24303385ns) {StartInitialHandshake:157(24303086ns) {DropHandshakes:470(264ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(529ns) {SwitchToState:179(291ns)}}}}}
2025-12-24T02:11:44.498426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241423119312012:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:44.502740Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422760087338:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31827159ns) {Configure:104(31826099ns) {ProcessConfigured:135(31450187ns) {StartInitialHandshake:157(31449368ns) {DropHandshakes:470(979ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(340ns)} GenerateSessionVirtualId:988(242ns) PrepareNewSessionHandshake:477(1341ns) {SwitchToState:179(776ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.503023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087408:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.503132Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241423435958815:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31754600ns) {Configure:104(31754031ns) {ProcessConfigured:135(31333657ns) {StartInitialHandshake:157(31332788ns) {DropHandshakes:470(481ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1363ns) {SwitchToState:179(941ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.508447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241423009312016:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59482
2025-12-24T02:11:44.508508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241423009312017:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59482
2025-12-24T02:11:44.508545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241423009312018:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59482
2025-12-24T02:11:44.508559Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241423009312019:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59482
2025-12-24T02:11:44.508910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312019:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.511148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087408:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.514882Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426345159826:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.514540Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422760087336:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27361911ns) {Configure:104(27360344ns) {ProcessConfigured:135(26861431ns) {StartInitialHandshake:157(26860734ns) {DropHandshakes:470(1298ns) {DropIncomingHandshake:442(214ns) DropOutgoingHandshake:457(212ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(1852ns) {SwitchToState:179(646ns)}}}}}
2025-12-24T02:11:44.514946Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241422760087337:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(27655368ns) {Configure:104(27654484ns) {ProcessConfigured:135(27279352ns) {StartInitialHandshake:157(27278514ns) {DropHandshakes:470(908ns) {DropIncomingHandshake:442(113ns) DropOutgoingHandshake:457(138ns)} GenerateSessionVirtualId:988(270ns) PrepareNewSessionHandshake:477(745ns) {SwitchToState:179(354ns)}}}}}
2025-12-24T02:11:44.514983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422760087337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.515306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312017:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.515437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241423009311998:2093] [node 6] ICP01 ready to work
2025-12-24T02:11:44.512605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423802267971:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45426
2025-12-24T02:11:44.512689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423802267972:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45426
2025-12-24T02:11:44.512702Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423802267973:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45426
2025-12-24T02:11:44.512757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423802267974:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45426
2025-12-24T02:11:44.516855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267967:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.516864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423802267967:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.517383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267971:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.517444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423802267962:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:44.516245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241423119312004:2098] [node 4] ICP01 ready to work
2025-12-24T02:11:44.519858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312016:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.518721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423802267961:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:44.518806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267973:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.518766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241423435958855:2087] [node 4] ICP01 ready to work
2025-12-24T02:11:44.527649Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241423119312012:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27430148ns) {Configure:104(27429129ns) {ProcessConfigured:135(26956887ns) {StartInitialHandshake:157(26956286ns) {DropHandshakes:470(1344ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(219ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:477(2132ns) {SwitchToState:179(1123ns)}}}}}
2025-12-24T02:11:44.527999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241423009312001:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:44.528297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312013:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.528311Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241423009312013:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.530689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426345159814:2059] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.530714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241422760087393:2097] [node 5] ICP01 ready to work
2025-12-24T02:11:44.530945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159833:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.530830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241423119312009:2099] [node 5] ICP01 ready to work
2025-12-24T02:11:44.531420Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312007:2101] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.531444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241423119311952:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.538809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426345159825:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.535845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087416:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.538831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009311985:2090] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.538848Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241423009311985:2090] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.540195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087402:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.540210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241422760087402:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.539812Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241426345159816:2064] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(71649303ns) {Configure:104(71648586ns) {ProcessConfigured:135(71376711ns) {StartInitialHandshake:157(71376212ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(577ns) {SwitchToState:179(249ns)}}}}}
2025-12-24T02:11:44.540515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267974:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.541173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267972:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.541845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423802267985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45426
2025-12-24T02:11:44.542014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423802267967:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.542109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423802267900:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.542128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.542136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423802267900:2049] [node 3] ICP22 created new session: [1:7587241423802267986:2049]
2025-12-24T02:11:44.542142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423802267986:2049] [node 3] ICS09 handshake done sender: [1:7587241423802267967:2110] self: [1:7587241423802267958:0] peer: [3:7587241423119312025:0] socket: 393 qp: -1
2025-12-24T02:11:44.542149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423802267986:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:44.542158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.540271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241423119312004:2098] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23796066ns) {Configure:104(23795494ns) {ProcessConfigured:135(23401332ns) {StartInitialHandshake:157(23400996ns) {DropHandshakes:470(313ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(668ns) {SwitchToState:179(308ns)}}}}}
2025-12-24T02:11:44.540055Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241423009311998:2093] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24445372ns) {Configure:104(24445048ns) {ProcessConfigured:135(24005628ns) {StartInitialHandshake:157(23992395ns) {DropHandshakes:470(771ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(367ns) PrepareNewSessionHandshake:477(687ns) {SwitchToState:179(29ns)}} IncomingHandshake:261(6214ns) {DropIncomingHandshake:442(30ns) PrepareNewSessionHandshake:477(213ns) {SwitchToState:179(175ns)} GenerateSessionVirtualId:988(582ns) IssueIncomingHandshakeReply:193(281ns)}}}}
2025-12-24T02:11:44.541086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241423009311985:2090] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.542811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312018:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.542819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241423009311942:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.542833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.542839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241423009311942:2049] [node 3] ICP22 created new session: [2:7587241423009312026:2049]
2025-12-24T02:11:44.542844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241423009312026:2049] [node 3] ICS09 handshake done sender: [2:7587241423009311985:2090] self: [2:7587241423009311984:0] peer: [3:7587241423119312032:0] socket: 390 qp: -1
2025-12-24T02:11:44.542849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241423009312026:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:44.542855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.539860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426345159816:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.543239Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241423435958855:2087] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24312603ns) {Configure:104(24311930ns) {ProcessConfigured:135(23887790ns) {StartInitialHandshake:157(23887254ns) {DropHandshakes:470(541ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(753ns)}}}}}
2025-12-24T02:11:44.543645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241423009311941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.543886Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241423119311951:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241423119312020:2110] held: no
2025-12-24T02:11:44.547041Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241423119311952:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241423119312007:2101] held: no
2025-12-24T02:11:44.546798Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426345159814:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.546852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267969:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.546864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423802267969:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.550794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241423009312013:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.550572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241423435958883:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.550585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241423435958883:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.551184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241423009312001:2101] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22984917ns) {Configure:104(22984301ns) {ProcessConfigured:135(22552899ns) {StartInitialHandshake:157(22537450ns) {DropHandshakes:470(1790ns) {DropIncomingHandshake:442(205ns) DropOutgoingHandshake:457(269ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(1053ns) {SwitchToState:179(437ns)}} IncomingHandshake:261(7475ns) {DropIncomingHandshake:442(28ns) PrepareNewSessionHandshake:477(411ns) {SwitchToState:179(371ns)} GenerateSessionVirtualId:988(533ns) IssueIncomingHandshakeReply:193(293ns)}}}}
2025-12-24T02:11:44.551204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241423009311999:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.551216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.551224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241423009311999:2096] [node 4] ICP22 created new session: [2:7587241423009312033:2096]
2025-12-24T02:11:44.551229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241423009312033:2096] [node 4] ICS09 handshake done sender: [2:7587241423009312013:2112] self: [2:7587241423009312012:0] peer: [4:7587241422760087413:0] socket: 408 qp: -1
2025-12-24T02:11:44.551236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241423009312033:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:44.551256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.554806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087416:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.554861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422760087336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.554951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087417:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.555426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422760087408:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.555419Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267985:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.555739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241423435958883:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.555577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422760087337:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.555604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.555613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422760087337:2049] [node 2] ICP22 created new session: [4:7587241422760087419:2049]
2025-12-24T02:11:44.555619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422760087419:2049] [node 2] ICS09 handshake done sender: [4:7587241422760087408:2112] self: [4:7587241422760087413:0] peer: [2:7587241423009312012:0] socket: 417 qp: -1
2025-12-24T02:11:44.555627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422760087419:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.555645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.555931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241423435958854:2085] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.555948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.555954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241423435958854:2085] [node 6] ICP22 created new session: [5:7587241423435958894:2085]
2025-12-24T02:11:44.555960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241423435958894:2085] [node 6] ICS09 handshake done sender: [5:7587241423435958883:2110] self: [5:7587241423435958857:0] peer: [6:7587241426345159838:0] socket: 401 qp: -1
2025-12-24T02:11:44.555966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241423435958894:2085] [node 6] ICS10 traffic start
2025-12-24T02:11:44.555974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.556327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312034:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.556379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312035:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.556398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312036:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.556426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312037:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.556511Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241423802267962:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37584251ns) {Configure:104(37583444ns) {ProcessConfigured:135(37204264ns) {StartInitialHandshake:157(37203707ns) {DropHandshakes:470(530ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(348ns) PrepareNewSessionHandshake:477(762ns) {SwitchToState:179(63ns)}}}}}
2025-12-24T02:11:44.557904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241423119312007:2101] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.557364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422760087402:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.557537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241422760087382:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.557547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.557555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422760087382:2092] [node 6] ICP22 created new session: [4:7587241422760087422:2092]
2025-12-24T02:11:44.557561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422760087422:2092] [node 6] ICS09 handshake done sender: [4:7587241422760087402:2109] self: [4:7587241422760087383:0] peer: [6:7587241426345159836:0] socket: 399 qp: -1
2025-12-24T02:11:44.557565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422760087422:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:44.557572Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.560848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958892:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.560982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958893:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.561285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241423435958892:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.561322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241423435958813:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.561364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241423435958893:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.561379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241423435958814:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.562923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426345159833:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.562971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426345159758:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.560390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241423119312038:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58718
2025-12-24T02:11:44.560538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241423119312020:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.562951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267976:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.562969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423802267976:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.563366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958899:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.563901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312031:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.563914Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241423009312031:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.564631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241423009312031:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.564933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958900:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.564789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241423009312001:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.564818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.564827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241423009312001:2101] [node 5] ICP22 created new session: [2:7587241423009312037:2101]
2025-12-24T02:11:44.564834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241423009312037:2101] [node 5] ICS09 handshake done sender: [2:7587241423009312031:2118] self: [2:7587241423009312022:0] peer: [5:7587241423435958898:0] socket: 437 qp: -1
2025-12-24T02:11:44.564842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241423009312037:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:44.564856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.565181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241423435958893:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.565361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241423435958814:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.565378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.565383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241423435958814:2049] [node 2] ICP22 created new session: [5:7587241423435958901:2049]
2025-12-24T02:11:44.565389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241423435958901:2049] [node 2] ICS09 handshake done sender: [5:7587241423435958893:2116] self: [5:7587241423435958898:0] peer: [2:7587241423009312022:0] socket: 440 qp: -1
2025-12-24T02:11:44.565395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241423435958901:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.565405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.566753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087425:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.570426Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159840:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.570488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159841:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.570952Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159842:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.571079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241423009312039:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59482
2025-12-24T02:11:44.571214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423802267969:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.571343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426345159840:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.571389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426345159757:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.571328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423802267899:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.571350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.571356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423802267899:2048] [node 2] ICP22 created new session: [1:7587241423802267993:2048]
2025-12-24T02:11:44.571365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423802267993:2048] [node 2] ICS09 handshake done sender: [1:7587241423802267969:2111] self: [1:7587241423802267964:0] peer: [2:7587241423009312029:0] socket: 402 qp: -1
2025-12-24T02:11:44.571371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423802267993:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:44.571389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.571776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426345159825:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.571898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159844:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.571992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241423009312018:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.572012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241423009312024:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.572015Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241423009312024:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.572117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426345159816:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.572131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.572136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426345159816:2064] [node 4] ICP22 created new session: [6:7587241426345159845:2064]
2025-12-24T02:11:44.572140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426345159845:2064] [node 4] ICS09 handshake done sender: [6:7587241426345159825:2110] self: [6:7587241426345159836:0] peer: [4:7587241422760087383:0] socket: 409 qp: -1
2025-12-24T02:11:44.572144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426345159845:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:44.572161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.572281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426345159826:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.574415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241423009312024:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.574497Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241423009311941:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241423009312018:2115] held: no
2025-12-24T02:11:44.574625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241423009311998:2093] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.574637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.574642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241423009311998:2093] [node 6] ICP22 created new session: [2:7587241423009312041:2093]
2025-12-24T02:11:44.574648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241423009312041:2093] [node 6] ICS09 handshake done sender: [2:7587241423009312024:2117] self: [2:7587241423009312020:0] peer: [6:7587241426345159843:0] socket: 435 qp: -1
2025-12-24T02:11:44.574654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241423009312041:2093] [node 6] ICS10 traffic start
2025-12-24T02:11:44.574661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.575923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267980:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.575944Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423802267980:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.578867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241423009311941:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.578882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.578889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241423009311941:2048] [node 1] ICP22 created new session: [2:7587241423009312043:2048]
2025-12-24T02:11:44.578895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241423009312043:2048] [node 1] ICS09 handshake done sender: [2:7587241423009312018:2115] self: [2:7587241423009312029:0] peer: [1:7587241423802267964:0] socket: 420 qp: -1
2025-12-24T02:11:44.578901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241423009312043:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.578914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.579821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958904:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.579053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312028:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.579068Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241423119312028:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.579723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241423119312028:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.579821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312038:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.579832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241423119311952:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.579843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.579850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241423119311952:2049] [node 2] ICP22 created new session: [3:7587241423119312041:2049]
2025-12-24T02:11:44.579857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241423119312041:2049] [node 2] ICS09 handshake done sender: [3:7587241423119312007:2101] self: [3:7587241423119312032:0] peer: [2:7587241423009311984:0] socket: 392 qp: -1
2025-12-24T02:11:44.579864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241423119312041:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.579871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.580074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312035:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.580098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312034:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.580106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241423119312012:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.580116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.580125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241423119312012:2103] [node 6] ICP22 created new session: [3:7587241423119312043:2103]
2025-12-24T02:11:44.580132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241423119312043:2103] [node 6] ICS09 handshake done sender: [3:7587241423119312028:2114] self: [3:7587241423119312026:0] peer: [6:7587241426345159839:0] socket: 429 qp: -1
2025-12-24T02:11:44.580137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241423119312043:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:44.580142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241423119311951:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.580150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.580154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241423119311951:2048] [node 1] ICP22 created new session: [3:7587241423119312044:2048]
2025-12-24T02:11:44.580159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241423119312044:2048] [node 1] ICS09 handshake done sender: [3:7587241423119312020:2110] self: [3:7587241423119312025:0] peer: [1:7587241423802267958:0] socket: 395 qp: -1
2025-12-24T02:11:44.580164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241423119312044:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.580168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.580170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.580423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241423119312012:2103] [node 6] ICP18 handshake (actor [3:7587241423119312034:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.585418Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241422760087337:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(29717983ns) {HandleHandshakeStatus:329(29717184ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(3171ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(24456527ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24427201ns max# 24338320ns}}}
2025-12-24T02:11:44.586087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423802267976:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.586690Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426345159757:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23272) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426345159840:2118] held: no
2025-12-24T02:11:44.586814Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426345159758:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426345159833:2115] held: no
2025-12-24T02:11:44.587048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426345159814:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.587063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.587069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426345159814:2059] [node 5] ICP22 created new session: [6:7587241426345159850:2059]
2025-12-24T02:11:44.587076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426345159850:2059] [node 5] ICS09 handshake done sender: [6:7587241426345159826:2111] self: [6:7587241426345159838:0] peer: [5:7587241423435958857:0] socket: 410 qp: -1
2025-12-24T02:11:44.587082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426345159850:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:44.587090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.587335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159847:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.587456Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159848:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.587794Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423802267980:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.587824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423802267963:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.587848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.587856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423802267963:2105] [node 4] ICP22 created new session: [1:7587241423802267998:2105]
2025-12-24T02:11:44.587862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423802267998:2105] [node 4] ICS09 handshake done sender: [1:7587241423802267976:2116] self: [1:7587241423802267975:0] peer: [4:7587241422760087418:0] socket: 425 qp: -1
2025-12-24T02:11:44.587867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423802267998:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:44.587885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.587844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426345159844:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.587877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426345159756:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.588308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087425:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.588367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423802267989:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.588377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423802267989:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.588862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423802267961:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.588875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.588882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423802267961:2106] [node 5] ICP22 created new session: [1:7587241423802268001:2106]
2025-12-24T02:11:44.588888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423802268001:2106] [node 5] ICS09 handshake done sender: [1:7587241423802267980:2117] self: [1:7587241423802267979:0] peer: [5:7587241423435958897:0] socket: 433 qp: -1
2025-12-24T02:11:44.588894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423802268001:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:44.588900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.589103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426345159853:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5454
2025-12-24T02:11:44.589254Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423802267989:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:44.589412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426345159844:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.590435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241422760087336:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241422760087416:2116] held: no
2025-12-24T02:11:44.590322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241423435958904:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.590569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241423435958899:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.590759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423802267962:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.590777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.590784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423802267962:2107] [node 6] ICP22 created new session: [1:7587241423802268003:2107]
2025-12-24T02:11:44.590790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423802268003:2107] [node 6] ICS09 handshake done sender: [1:7587241423802267989:2119] self: [1:7587241423802267978:0] peer: [6:7587241426345159852:0] socket: 455 qp: -1
2025-12-24T02:11:44.590795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423802268003:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:44.590802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.590662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241423435958815:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.590676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241423435958855:2087] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.590917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087427:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.590957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087426:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.591029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958905:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.591214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087426:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.591225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422760087416:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.591453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958908:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.591540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312031:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.591567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241423119312031:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.592636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241423119312031:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.592770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241423119312033:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.592777Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241423119312033:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.593213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241423119312009:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.593225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.593229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241423119312009:2099] [node 5] ICP22 created new session: [3:7587241423119312051:2099]
2025-12-24T02:11:44.593235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241423119312051:2099] [node 5] ICS09 handshake done sender: [3:7587241423119312031:2115] self: [3:7587241423119312030:0] peer: [5:7587241423435958906:0] socket: 407 qp: -1
2025-12-24T02:11:44.593241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241423119312051:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:44.593260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.593907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241423119312033:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.593980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241423119312004:2098] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.593996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.594003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241423119312004:2098] [node 4] ICP22 created new session: [3:7587241423119312054:2098]
2025-12-24T02:11:44.594008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241423119312054:2098] [node 4] ICS09 handshake done sender: [3:7587241423119312033:2116] self: [3:7587241423119312027:0] peer: [4:7587241422760087431:0] socket: 457 qp: -1
2025-12-24T02:11:44.594012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241423119312054:2098] [node 4] ICS10 traffic start
2025-12-24T02:11:44.594018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.592102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422760087336:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.592116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.592122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422760087336:2048] [node 1] ICP22 created new session: [4:7587241422760087429:2048]
2025-12-24T02:11:44.592128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422760087429:2048] [node 1] ICS09 handshake done sender: [4:7587241422760087416:2116] self: [4:7587241422760087418:0] peer: [1:7587241423802267975:0] socket: 427 qp: -1
2025-12-24T02:11:44.592134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422760087429:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.592141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.592377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241422760087338:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.598895Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241423435958813:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241423435958892:2115] held: no
2025-12-24T02:11:44.599342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426345159840:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:44.599490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426345159756:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.599506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.599516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426345159756:2048] [node 1] ICP22 created new session: [6:7587241426345159854:2048]
2025-12-24T02:11:44.599523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426345159854:2048] [node 1] ICS09 handshake done sender: [6:7587241426345159844:2121] self: [6:7587241426345159852:0] peer: [1:7587241423802267978:0] socket: 456 qp: -1
2025-12-24T02:11:44.599532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426345159854:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.599538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.599056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241423435958815:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241423435958904:2119] held: no
2025-12-24T02:11:44.599435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241423435958892:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.602763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241422760087415:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.602790Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241422760087415:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.607421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426345159833:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.606868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241423435958813:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.606896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.606907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241423435958813:2048] [node 1] ICP22 created new session: [5:7587241423435958909:2048]
2025-12-24T02:11:44.606915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241423435958909:2048] [node 1] ICS09 handshake done sender: [5:7587241423435958892:2115] self: [5:7587241423435958897:0] peer: [1:7587241423802267979:0] socket: 434 qp: -1
2025-12-24T02:11:44.606925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241423435958909:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.606934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.608090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241422760087338:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24293) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241422760087426:2119] held: no
2025-12-24T02:11:44.608467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426345159757:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.608484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:44.608491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426345159757:2049] [node 2] ICP22 created new session: [6:7587241426345159856:2049]
2025-12-24T02:11:44.608496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426345159856:2049] [node 2] ICS09 handshake done sender: [6:7587241426345159840:2118] self: [6:7587241426345159843:0] peer: [2:7587241423009312020:0] socket: 405 qp: -1
2025-12-24T02:11:44.608503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426345159856:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:44.608511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:44.607095Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241423802267962:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(16203687ns) {HandleHandshakeStatus:329(16202941ns) {DropIncomingHandshake:442(3685ns) DropOutgoingHandshake:457(64ns) SwitchToState:179(98ns) ProcessPendingSessionEvents:495(16123794ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16077208ns max# 15979942ns}}}
2025-12-24T02:11:44.608601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426345159758:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.608623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.608636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426345159758:2050] [node 3] ICP22 created new session: [6:7587241426345159858:2050]
2025-12-24T02:11:44.608643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426345159858:2050] [node 3] ICS09 handshake done sender: [6:7587241426345159833:2115] self: [6:7587241426345159839:0] peer: [3:7587241423119312026:0] socket: 430 qp: -1
2025-12-24T02:11:44.608651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426345159858:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.608660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.610995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958911:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.611603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241422760087432:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54606
2025-12-24T02:11:44.612098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241423435958904:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.612218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241423435958815:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.612236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.612244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241423435958815:2050] [node 3] ICP22 created new session: [5:7587241423435958912:2050]
2025-12-24T02:11:44.612250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241423435958912:2050] [node 3] ICS09 handshake done sender: [5:7587241423435958904:2119] self: [5:7587241423435958906:0] peer: [3:7587241423119312030:0] socket: 419 qp: -1
2025-12-24T02:11:44.612256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241423435958912:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.612280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:44.614986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422760087426:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.616243Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241422760087415:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.616344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241422760087393:2097] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.616363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.616373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422760087393:2097] [node 5] ICP22 created new session: [4:7587241422760087434:2097]
2025-12-24T02:11:44.616380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422760087434:2097] [node 5] ICS09 handshake done sender: [4:7587241422760087415:2115] self: [4:7587241422760087414:0] peer: [5:7587241423435958907:0] socket: 446 qp: -1
2025-12-24T02:11:44.616386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422760087434:2097] [node 5] ICS10 traffic start
2025-12-24T02:11:44.616394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.616858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241422760087338:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.616868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:44.616875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241422760087338:2050] [node 3] ICP22 created new session: [4:7587241422760087436:2050]
2025-12-24T02:11:44.616880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241422760087436:2050] [node 3] ICS09 handshake done sender: [4:7587241422760087426:2119] self: [4:7587241422760087431:0] peer: [3:7587241423119312027:0] socket: 421 qp: -1
2025-12-24T02:11:44.616886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241422760087436:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:44.616892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.618928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241423435958855:2087] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241423435958899:2117] held: no
2025-12-24T02:11:44.621753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241423435958914:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30550
2025-12-24T02:11:44.625208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241423435958899:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.625312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241423435958855:2087] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.625332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.625342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241423435958855:2087] [node 4] ICP22 created new session: [5:7587241423435958915:2087]
2025-12-24T02:11:44.625349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241423435958915:2087] [node 4] ICS09 handshake done sender: [5:7587241423435958899:2117] self: [5:7587241423435958907:0] peer: [4:7587241422760087414:0] socket: 447 qp: -1
2025-12-24T02:11:44.625358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241423435958915:2087] [node 4] ICS10 traffic start
2025-12-24T02:11:44.625370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.624673Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241426345159756:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(25060572ns) {HandleHandshakeStatus:329(25059788ns) {DropIncomingHandshake:442(776ns) DropOutgoingHandshake:457(1592ns) SwitchToState:179(229ns) ProcessPendingSessionEvents:495(13336660ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13306835ns max# 13214658ns}}}
2025-12-24T02:11:44.642897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:44.642956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:44.643073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241423009312043:2048] [node 1] ICS01 socket: 420 reason# EndOfStream
2025-12-24T02:11:44.643083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241423009311941:2048] [node 1] ICP30 unregister session Session# [2:7587241423009312043:2048] VirtualId# [2:7587241423009312029:0]
2025-12-24T02:11:44.643093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241423009312043:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.643151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.643183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.643320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:44.643397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241422760087429:2048] [node 1] ICS01 socket: 427 reason# EndOfStream
2025-12-24T02:11:44.643404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241422760087336:2048] [node 1] ICP30 unregister session Session# [4:7587241422760087429:2048] VirtualId# [4:7587241422760087418:0]
2025-12-24T02:11:44.643409Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241422760087429:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.643440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.643461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.643592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241423119312044:2048] [node 1] ICS01 socket: 395 reason# EndOfStream
2025-12-24T02:11:44.643606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241423119311951:2048] [node 1] ICP30 unregister session Session# [3:7587241423119312044:2048] VirtualId# [3:7587241423119312025:0]
2025-12-24T02:11:44.643612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241423119312044:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.643645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.643666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.646864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:44.646957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241423435958909:2048] [node 1] ICS01 socket: 434 reason# EndOfStream
2025-12-24T02:11:44.646965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241423435958813:2048] [node 1] ICP30 unregister session Session# [5:7587241423435958909:2048] VirtualId# [5:7587241423435958897:0]
2025-12-24T02:11:44.646972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241423435958909:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.647029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.647059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.647098Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:44.647158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241426345159854:2048] [node 1] ICS07 socket disconnect 456 reason# EndOfStream
2025-12-24T02:11:44.647163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426345159854:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.647190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:44.647218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241426345159854:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:44.647774Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426345159756:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:44.647793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426345159854:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:44.647797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426345159756:2048] [node 1] ICP30 unregister session Session# [6:7587241426345159854:2048] VirtualId# [6:7587241426345159852:0]
2025-12-24T02:11:44.647853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:44.647861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241426345159756:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.647863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29361) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.658804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241426345159756:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:44.671162Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:44.671232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241423435958901:2049] [node 2] ICS01 socket: 440 reason# EndOfStream
2025-12-24T02:11:44.671238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241423435958814:2049] [node 2] ICP30 unregister session Session# [5:7587241423435958901:2049] VirtualId# [5:7587241423435958898:0]
2025-12-24T02:11:44.671244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241423435958901:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.671273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.671295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.670861Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:44.670977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241423119312041:2049] [node 2] ICS01 socket: 392 reason# EndOfStream
2025-12-24T02:11:44.670989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241423119311952:2049] [node 2] ICP30 unregister session Session# [3:7587241423119312041:2049] VirtualId# [3:7587241423119312032:0]
2025-12-24T02:11:44.670999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241423119312041:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.671051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.671078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.672548Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:44.672641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241422760087419:2049] [node 2] ICS07 socket disconnect 417 reason# EndOfStream
2025-12-24T02:11:44.672648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241422760087419:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.672708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.672742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241422760087419:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:44.675007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:44.675086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426345159856:2049] [node 2] ICS01 socket: 405 reason# EndOfStream
2025-12-24T02:11:44.675095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426345159757:2049] [node 2] ICP30 unregister session Session# [6:7587241426345159856:2049] VirtualId# [6:7587241426345159843:0]
2025-12-24T02:11:44.675102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426345159856:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.675153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:44.675178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.677314Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241422760087337:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23272) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:44.677332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241422760087419:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:44.677338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241422760087337:2049] [node 2] ICP30 unregister session Session# [4:7587241422760087419:2049] VirtualId# [4:7587241422760087413:0]
2025-12-24T02:11:44.677393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:44.677400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241422760087337:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.677404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23272) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.689922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241422760087337:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:44.701806Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:44.701908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241423435958912:2050] [node 3] ICS01 socket: 419 reason# EndOfStream
2025-12-24T02:11:44.701920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241423435958815:2050] [node 3] ICP30 unregister session Session# [5:7587241423435958912:2050] VirtualId# [5:7587241423435958906:0]
2025-12-24T02:11:44.701929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241423435958912:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.701987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:44.702012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:44.702906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:44.702989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426345159858:2050] [node 3] ICS01 socket: 430 reason# EndOfStream
2025-12-24T02:11:44.702996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426345159758:2050] [node 3] ICP30 unregister session Session# [6:7587241426345159858:2050] VirtualId# [6:7587241426345159839:0]
2025-12-24T02:11:44.703002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426345159858:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.703046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:44.703082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:44.754951Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:44.755075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241426345159845:2064] [node 4] ICS07 socket disconnect 409 reason# EndOfStream
2025-12-24T02:11:44.755083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241426345159845:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:44.755155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:44.755182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241426345159845:2064] [node 4] ICS15 start handshake
2025-12-24T02:11:44.759266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426345159816:2064] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:44.759288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241426345159845:2064] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:44.759294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426345159816:2064] [node 4] ICP30 unregister session Session# [6:7587241426345159845:2064] VirtualId# [6:7587241426345159836:0]
2025-12-24T02:11:44.759358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:44.759366Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241426345159816:2064] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:44.759370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20181) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:44.814123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424559180258:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.814172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424559180259:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:44.830745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423945162123:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:44.830784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423945162124:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:44.833975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422961930776:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.834010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422961930777:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.834723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241426376659377:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.834751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241426376659378:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.834756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241426376659379:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.838715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241426648241689:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.838751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241426648241690:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.838757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241426648241691:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.830885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426029008336:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:44.830903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426029008337:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:44.830909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426029008338:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:44.885703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423945162184:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.886988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422961930834:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:44.887339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424559180317:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:44.886564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241426648241723:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:44.886944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241426648241728:2084] [node 4] ICP01 ready to work
2025-12-24T02:11:44.898866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426029008364:2061] [node 4] ICP01 ready to work
2025-12-24T02:11:44.899042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241426029008365:2069] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.901857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423945162190:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:44.901984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241423945162191:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:44.905054Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241426648241728:2084] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17600450ns) {Configure:104(17600042ns) {ProcessConfigured:135(17349397ns) {StartInitialHandshake:157(17349180ns) {DropHandshakes:470(309ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(200ns) PrepareNewSessionHandshake:477(731ns) {SwitchToState:179(369ns)}}}}}
2025-12-24T02:11:44.905325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241426648241742:2092] [node 6] ICP01 ready to work
2025-12-24T02:11:44.904740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659438:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:44.904769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659439:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:44.913015Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241424559180317:2105] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25603311ns) {Configure:104(25602427ns) {ProcessConfigured:135(25228478ns) {StartInitialHandshake:157(25227797ns) {DropHandshakes:470(519ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(90ns)} GenerateSessionVirtualId:988(62ns) PrepareNewSessionHandshake:477(1628ns) {SwitchToState:179(442ns)}}}}}
2025-12-24T02:11:44.913215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424559180321:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.927191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241426376659442:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:44.927285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659441:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:44.933954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241426648241723:2081] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.935060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241426376659455:2116] [node 5] ICP01 ready to work
2025-12-24T02:11:44.935126Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659441:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.936074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008414:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:44.936097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008415:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.941813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423945162202:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-12-24T02:11:44.941873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241423945162203:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-12-24T02:11:44.945193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162202:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.951116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422961930842:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:44.951235Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930834:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.951315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422961930843:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:44.951469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241422961930844:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:44.952471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422961930840:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:44.952550Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422961930841:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:44.952550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659438:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.958885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930840:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.963513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241426648241689:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.964074Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241424559180321:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(50670749ns) {Configure:104(50670313ns) {ProcessConfigured:135(50338160ns) {StartInitialHandshake:157(50337906ns) {DropHandshakes:470(646ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(99ns)} GenerateSessionVirtualId:988(169ns) PrepareNewSessionHandshake:477(991ns) {SwitchToState:179(337ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.967169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008416:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:44.967578Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659439:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.967643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:44.967655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241426376659377:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.970293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241422961930776:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.970384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930841:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.970872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426029008415:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.970905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426029008364:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.971075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426029008414:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.971094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426029008365:2069] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.973985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162186:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.974001Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423945162186:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.974426Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162195:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.974430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423945162195:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.976085Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423945162186:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.975927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659446:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.975938Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241426376659446:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.978807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241426648241762:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.978826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241426648241762:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.979377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241426648241769:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.986403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241426648241762:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.982663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241424559180332:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:44.983467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424559180333:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:44.983513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241424559180334:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:44.986571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241426648241742:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.986589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:44.986596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241426648241742:2092] [node 6] ICP22 created new session: [5:7587241426648241772:2092]
2025-12-24T02:11:44.986604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241426648241772:2092] [node 6] ICS09 handshake done sender: [5:7587241426648241762:2113] self: [5:7587241426648241743:0] peer: [6:7587241426029008418:0] socket: 414 qp: -1
2025-12-24T02:11:44.986610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241426648241772:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:44.986621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:44.986937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241426648241723:2081] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.987119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423945162195:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:44.986565Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:44.986590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241426376659463:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.992245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008420:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:44.992281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008419:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:44.993119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423945162184:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.993143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.993149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423945162184:2107] [node 5] ICP22 created new session: [1:7587241423945162207:2107]
2025-12-24T02:11:44.993156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423945162207:2107] [node 5] ICS09 handshake done sender: [1:7587241423945162186:2108] self: [1:7587241423945162185:0] peer: [5:7587241426648241766:0] socket: 397 qp: -1
2025-12-24T02:11:44.993162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423945162207:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:44.993170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.993556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162193:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.993572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423945162193:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:44.992452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241426648241689:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.992476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.992483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241426648241689:2048] [node 1] ICP22 created new session: [5:7587241426648241774:2048]
2025-12-24T02:11:44.992493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241426648241774:2048] [node 1] ICS09 handshake done sender: [5:7587241426648241723:2081] self: [5:7587241426648241766:0] peer: [1:7587241423945162185:0] socket: 399 qp: -1
2025-12-24T02:11:44.992501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241426648241774:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.993988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162203:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.994005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423945162190:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:44.994013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:44.994019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423945162190:2109] [node 4] ICP22 created new session: [1:7587241423945162209:2109]
2025-12-24T02:11:44.994024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241423945162209:2109] [node 4] ICS09 handshake done sender: [1:7587241423945162195:2112] self: [1:7587241423945162194:0] peer: [4:7587241426376659461:0] socket: 404 qp: -1
2025-12-24T02:11:44.992509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.994028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241423945162209:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:44.992781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241426648241771:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:44.991133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422961930861:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:44.994909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241422961930862:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:44.995029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426029008416:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.995092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426029008414:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:44.995405Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422961930840:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:44.995559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241422961930776:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.995574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:44.995581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422961930776:2048] [node 1] ICP22 created new session: [3:7587241422961930863:2048]
2025-12-24T02:11:44.995587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241422961930863:2048] [node 1] ICS09 handshake done sender: [3:7587241422961930840:2111] self: [3:7587241422961930859:0] peer: [1:7587241423945162192:0] socket: 412 qp: -1
2025-12-24T02:11:44.995593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241422961930863:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:44.995600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:44.996322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426029008365:2069] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:44.996340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:44.996347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426029008365:2069] [node 5] ICP22 created new session: [6:7587241426029008422:2069]
2025-12-24T02:11:44.996352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241426029008422:2069] [node 5] ICS09 handshake done sender: [6:7587241426029008414:2115] self: [6:7587241426029008418:0] peer: [5:7587241426648241743:0] socket: 419 qp: -1
2025-12-24T02:11:44.996359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241426029008422:2069] [node 5] ICS10 traffic start
2025-12-24T02:11:44.996366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:44.996718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426029008338:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:44.994034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:44.994519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241423945162190:2109] [node 4] ICP18 handshake (actor [1:7587241423945162203:2116]) is too old
2025-12-24T02:11:44.995192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423945162193:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:44.998452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659463:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:44.998500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659459:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.003158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241426376659439:2109] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.003975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241426376659377:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11971) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241426376659439:2109] held: no
2025-12-24T02:11:45.004102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241426376659377:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.004113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.004119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241426376659377:2048] [node 1] ICP22 created new session: [4:7587241430671626761:2048]
2025-12-24T02:11:45.004126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430671626761:2048] [node 1] ICS09 handshake done sender: [4:7587241426376659439:2109] self: [4:7587241426376659461:0] peer: [1:7587241423945162194:0] socket: 406 qp: -1
2025-12-24T02:11:45.004134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430671626761:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.004144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.004467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241426376659446:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.004929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241426376659378:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.004945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241426376659379:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.004954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241426376659442:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.004965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.004971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241426376659442:2110] [node 6] ICP22 created new session: [4:7587241430671626765:2110]
2025-12-24T02:11:45.004975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430671626765:2110] [node 6] ICS09 handshake done sender: [4:7587241426376659446:2113] self: [4:7587241426376659445:0] peer: [6:7587241426029008417:0] socket: 416 qp: -1
2025-12-24T02:11:45.004979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430671626765:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:45.004984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.003892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430943209074:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:45.008064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423945162124:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.008082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.008087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423945162124:2049] [node 3] ICP22 created new session: [1:7587241428240129510:2049]
2025-12-24T02:11:45.008094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428240129510:2049] [node 3] ICS09 handshake done sender: [1:7587241423945162193:2111] self: [1:7587241423945162192:0] peer: [3:7587241422961930859:0] socket: 403 qp: -1
2025-12-24T02:11:45.008099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428240129510:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.008106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.010957Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430943209075:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.011315Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930850:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.011329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422961930850:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.012008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930848:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.012012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422961930848:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.012716Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427256898162:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:45.013157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430943209074:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.013193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241426648241691:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.013431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180334:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.013535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180333:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.013567Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180320:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.013574Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424559180320:2107] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.014150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241428854147635:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:45.014189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241428854147636:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:45.014606Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424559180320:2107] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.015087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424559180317:2105] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.015103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.015110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424559180317:2105] [node 4] ICP22 created new session: [2:7587241428854147640:2105]
2025-12-24T02:11:45.015116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241428854147640:2105] [node 4] ICS09 handshake done sender: [2:7587241424559180320:2107] self: [2:7587241424559180318:0] peer: [4:7587241430671626763:0] socket: 435 qp: -1
2025-12-24T02:11:45.015122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241428854147640:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:45.015129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.015633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241426648241771:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.015656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241426648241690:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.015813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241426029008364:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241426029008415:2116] held: no
2025-12-24T02:11:45.015937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241426029008421:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:45.016008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426029008415:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.016318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426029008364:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.016329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.016334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426029008364:2061] [node 4] ICP22 created new session: [6:7587241430323975723:2061]
2025-12-24T02:11:45.016338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430323975723:2061] [node 4] ICS09 handshake done sender: [6:7587241426029008415:2116] self: [6:7587241426029008417:0] peer: [4:7587241426376659445:0] socket: 420 qp: -1
2025-12-24T02:11:45.016343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430323975723:2061] [node 4] ICS10 traffic start
2025-12-24T02:11:45.016348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.016630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430323975721:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.017977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430671626769:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:45.017998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430671626770:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40534
2025-12-24T02:11:45.018161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241426376659463:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.018214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241426376659378:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.018228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.018233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241426376659378:2049] [node 2] ICP22 created new session: [4:7587241430671626771:2049]
2025-12-24T02:11:45.018239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430671626771:2049] [node 2] ICS09 handshake done sender: [4:7587241426376659463:2120] self: [4:7587241430671626763:0] peer: [2:7587241424559180318:0] socket: 441 qp: -1
2025-12-24T02:11:45.018245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430671626771:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.018253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.019078Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241428854147635:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.019127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180326:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.019130Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424559180326:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.019734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241428854147636:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.019889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241428854147642:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:45.020120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241424559180258:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.020472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430943209075:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.020572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430943209078:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:45.020591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241426648241728:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.020586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422961930850:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.020729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424559180326:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.020805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426029008416:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.020871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426029008338:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.020885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.020890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426029008338:2050] [node 3] ICP22 created new session: [6:7587241430323975725:2050]
2025-12-24T02:11:45.020896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430323975725:2050] [node 3] ICS09 handshake done sender: [6:7587241426029008416:2117] self: [6:7587241426029008424:0] peer: [3:7587241422961930849:0] socket: 426 qp: -1
2025-12-24T02:11:45.020901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430323975725:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.020908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.020941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241426648241771:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.021130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241426648241690:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.021140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.021146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241426648241690:2049] [node 2] ICP22 created new session: [5:7587241430943209080:2049]
2025-12-24T02:11:45.021151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430943209080:2049] [node 2] ICS09 handshake done sender: [5:7587241426648241771:2117] self: [5:7587241430943209077:0] peer: [2:7587241424559180322:0] socket: 443 qp: -1
2025-12-24T02:11:45.021158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430943209080:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.021166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.022290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162199:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.022301Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423945162199:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.022919Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241428240129512:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49966
2025-12-24T02:11:45.024574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423945162199:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.024397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241426376659457:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.024407Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241426376659457:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.025136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241426376659457:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.025204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241426376659455:2116] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.025220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.025225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241426376659455:2116] [node 5] ICP22 created new session: [4:7587241430671626774:2116]
2025-12-24T02:11:45.025230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430671626774:2116] [node 5] ICS09 handshake done sender: [4:7587241426376659457:2117] self: [4:7587241426376659456:0] peer: [5:7587241430943209079:0] socket: 452 qp: -1
2025-12-24T02:11:45.025235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430671626774:2116] [node 5] ICS10 traffic start
2025-12-24T02:11:45.025241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.027277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424559180321:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.027297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.027303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424559180321:2106] [node 5] ICP22 created new session: [2:7587241428854147648:2106]
2025-12-24T02:11:45.027309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241428854147648:2106] [node 5] ICS09 handshake done sender: [2:7587241424559180326:2111] self: [2:7587241424559180322:0] peer: [5:7587241430943209077:0] socket: 436 qp: -1
2025-12-24T02:11:45.027314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241428854147648:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:45.027320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.028014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241428854147647:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28459
2025-12-24T02:11:45.028538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423945162123:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.028555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.028563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423945162123:2048] [node 2] ICP22 created new session: [1:7587241428240129514:2048]
2025-12-24T02:11:45.028568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428240129514:2048] [node 2] ICS09 handshake done sender: [1:7587241423945162199:2114] self: [1:7587241423945162198:0] peer: [2:7587241428854147645:0] socket: 449 qp: -1
2025-12-24T02:11:45.028574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428240129514:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:45.028582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.029341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241428854147636:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.031025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241424559180258:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.031040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.031046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424559180258:2048] [node 1] ICP22 created new session: [2:7587241428854147651:2048]
2025-12-24T02:11:45.031051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241428854147651:2048] [node 1] ICS09 handshake done sender: [2:7587241428854147636:2118] self: [2:7587241428854147645:0] peer: [1:7587241423945162198:0] socket: 458 qp: -1
2025-12-24T02:11:45.031056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241428854147651:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.031063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.032307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428240129512:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.032621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241423945162123:2048] [node 2] ICP18 handshake (actor [1:7587241428240129512:2117]) is too old
2025-12-24T02:11:45.032508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422961930848:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.033188Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241426648241728:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22174) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241430943209075:2119] held: no
2025-12-24T02:11:45.033385Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430943209082:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:45.033538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430943209075:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.033592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241426648241728:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.033604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.033609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241426648241728:2084] [node 4] ICP22 created new session: [5:7587241430943209083:2084]
2025-12-24T02:11:45.033614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430943209083:2084] [node 4] ICS09 handshake done sender: [5:7587241430943209075:2119] self: [5:7587241430943209079:0] peer: [4:7587241426376659456:0] socket: 453 qp: -1
2025-12-24T02:11:45.033619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430943209083:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:45.033624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.033152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241426029008421:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.034299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430323975727:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:45.034319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426029008336:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.034549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930855:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.034558Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241422961930855:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.035042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427256898162:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.035056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422961930843:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.035070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.035075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422961930843:2109] [node 6] ICP22 created new session: [3:7587241427256898165:2109]
2025-12-24T02:11:45.035080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427256898165:2109] [node 6] ICS09 handshake done sender: [3:7587241422961930850:2115] self: [3:7587241422961930849:0] peer: [6:7587241426029008424:0] socket: 425 qp: -1
2025-12-24T02:11:45.035085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427256898165:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:45.035093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.035449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422961930842:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.035455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.035459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422961930842:2108] [node 4] ICP22 created new session: [3:7587241427256898167:2108]
2025-12-24T02:11:45.035464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427256898167:2108] [node 4] ICS09 handshake done sender: [3:7587241422961930848:2114] self: [3:7587241422961930847:0] peer: [4:7587241430671626764:0] socket: 424 qp: -1
2025-12-24T02:11:45.035469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427256898167:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:45.035474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.037134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241422961930842:2108] [node 4] ICP18 handshake (actor [3:7587241427256898162:2120]) is too old
2025-12-24T02:11:45.037835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422961930855:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.037948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241422961930861:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.038000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241422961930844:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.038013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.038026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422961930844:2110] [node 5] ICP22 created new session: [3:7587241427256898170:2110]
2025-12-24T02:11:45.038030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427256898170:2110] [node 5] ICS09 handshake done sender: [3:7587241422961930855:2117] self: [3:7587241422961930854:0] peer: [5:7587241430943209076:0] socket: 418 qp: -1
2025-12-24T02:11:45.038035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427256898170:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:45.038040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.038437Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241423945162197:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.038450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241423945162197:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.042871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241428854147642:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.043072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430323975727:2122] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.043192Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430323975729:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:45.043208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241426029008337:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.043333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241422961930777:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.043412Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180336:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.043420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424559180336:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.043826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241424559180328:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.043829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241424559180328:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.044732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241423945162197:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.043154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430943209085:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12789
2025-12-24T02:11:45.046423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241423945162191:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.046445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.046452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241423945162191:2110] [node 6] ICP22 created new session: [1:7587241428240129517:2110]
2025-12-24T02:11:45.046458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428240129517:2110] [node 6] ICS09 handshake done sender: [1:7587241423945162197:2113] self: [1:7587241423945162196:0] peer: [6:7587241430323975728:0] socket: 448 qp: -1
2025-12-24T02:11:45.046464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428240129517:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:45.046474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.047016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424559180336:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.044636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430323975731:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10839
2025-12-24T02:11:45.044937Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241426029008421:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.044987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426029008336:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.045002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.045017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426029008336:2048] [node 1] ICP22 created new session: [6:7587241430323975732:2048]
2025-12-24T02:11:45.045023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430323975732:2048] [node 1] ICS09 handshake done sender: [6:7587241426029008421:2120] self: [6:7587241430323975728:0] peer: [1:7587241423945162196:0] socket: 451 qp: -1
2025-12-24T02:11:45.045030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430323975732:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.045038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.047531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241426376659379:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241426376659459:2118] held: no
2025-12-24T02:11:45.048029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241424559180328:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.051057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424559180332:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.051076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.051081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424559180332:2109] [node 6] ICP22 created new session: [2:7587241428854147656:2109]
2025-12-24T02:11:45.051087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241428854147656:2109] [node 6] ICS09 handshake done sender: [2:7587241424559180336:2116] self: [2:7587241424559180335:0] peer: [6:7587241430323975730:0] socket: 415 qp: -1
2025-12-24T02:11:45.051093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241428854147656:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:45.051099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.051356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241424559180259:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.051368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.051372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241424559180259:2049] [node 3] ICP22 created new session: [2:7587241428854147658:2049]
2025-12-24T02:11:45.051376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241428854147658:2049] [node 3] ICS09 handshake done sender: [2:7587241424559180328:2112] self: [2:7587241424559180327:0] peer: [3:7587241427256898172:0] socket: 437 qp: -1
2025-12-24T02:11:45.051380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241428854147658:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.051386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.048508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241426648241691:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4941) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241430943209074:2118] held: no
2025-12-24T02:11:45.048664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430943209074:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.048811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241426648241691:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.048820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.048825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241426648241691:2050] [node 3] ICP22 created new session: [5:7587241430943209086:2050]
2025-12-24T02:11:45.048831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430943209086:2050] [node 3] ICS09 handshake done sender: [5:7587241430943209074:2118] self: [5:7587241430943209076:0] peer: [3:7587241422961930854:0] socket: 417 qp: -1
2025-12-24T02:11:45.048838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430943209086:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.048846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.047730Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241426376659459:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.048094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241426376659379:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.048104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.048110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241426376659379:2050] [node 3] ICP22 created new session: [4:7587241430671626776:2050]
2025-12-24T02:11:45.048115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430671626776:2050] [node 3] ICS09 handshake done sender: [4:7587241426376659459:2118] self: [4:7587241430671626764:0] peer: [3:7587241422961930847:0] socket: 428 qp: -1
2025-12-24T02:11:45.048121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430671626776:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.048128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.052365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427256898173:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19731
2025-12-24T02:11:45.052706Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241422961930861:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.053278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241422961930777:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.053290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.053296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241422961930777:2049] [node 2] ICP22 created new session: [3:7587241427256898174:2049]
2025-12-24T02:11:45.053302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427256898174:2049] [node 2] ICS09 handshake done sender: [3:7587241422961930861:2118] self: [3:7587241427256898172:0] peer: [2:7587241424559180327:0] socket: 444 qp: -1
2025-12-24T02:11:45.053308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427256898174:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.053316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.053196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430323975727:2122] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.055947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241426029008337:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.055966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.055973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241426029008337:2049] [node 2] ICP22 created new session: [6:7587241430323975734:2049]
2025-12-24T02:11:45.055978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430323975734:2049] [node 2] ICS09 handshake done sender: [6:7587241430323975727:2122] self: [6:7587241430323975730:0] peer: [2:7587241424559180335:0] socket: 408 qp: -1
2025-12-24T02:11:45.055984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430323975734:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.055991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.056144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241426029008337:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:45.067172Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:45.067280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241428854147651:2048] [node 1] ICS01 socket: 458 reason# EndOfStream
2025-12-24T02:11:45.067290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241424559180258:2048] [node 1] ICP30 unregister session Session# [2:7587241428854147651:2048] VirtualId# [2:7587241428854147645:0]
2025-12-24T02:11:45.067298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241428854147651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.067330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.067353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.070797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:45.071401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:45.071477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241426648241774:2048] [node 1] ICS01 socket: 399 reason# EndOfStream
2025-12-24T02:11:45.071483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241426648241689:2048] [node 1] ICP30 unregister session Session# [5:7587241426648241774:2048] VirtualId# [5:7587241426648241766:0]
2025-12-24T02:11:45.071490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241426648241774:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.071519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.071541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.071822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:45.071904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430323975732:2048] [node 1] ICS01 socket: 451 reason# EndOfStream
2025-12-24T02:11:45.071909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426029008336:2048] [node 1] ICP30 unregister session Session# [6:7587241430323975732:2048] VirtualId# [6:7587241430323975728:0]
2025-12-24T02:11:45.071915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430323975732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.071948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.071969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.070912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241422961930863:2048] [node 1] ICS01 socket: 412 reason# EndOfStream
2025-12-24T02:11:45.070929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241422961930776:2048] [node 1] ICP30 unregister session Session# [3:7587241422961930863:2048] VirtualId# [3:7587241422961930859:0]
2025-12-24T02:11:45.070938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241422961930863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.071001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.071031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.074750Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:45.074862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241430671626761:2048] [node 1] ICS01 socket: 406 reason# EndOfStream
2025-12-24T02:11:45.074872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241426376659377:2048] [node 1] ICP30 unregister session Session# [4:7587241430671626761:2048] VirtualId# [4:7587241426376659461:0]
2025-12-24T02:11:45.074880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241430671626761:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.074937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.074965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.083583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:45.083691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241427256898174:2049] [node 2] ICS01 socket: 444 reason# EndOfStream
2025-12-24T02:11:45.083701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241422961930777:2049] [node 2] ICP30 unregister session Session# [3:7587241427256898174:2049] VirtualId# [3:7587241427256898172:0]
2025-12-24T02:11:45.083709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241427256898174:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.083747Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.083767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.087998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:45.088128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430323975734:2049] [node 2] ICS01 socket: 408 reason# EndOfStream
2025-12-24T02:11:45.088136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426029008337:2049] [node 2] ICP30 unregister session Session# [6:7587241430323975734:2049] VirtualId# [6:7587241430323975730:0]
2025-12-24T02:11:45.088144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430323975734:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.088193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.088214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.088297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:45.088361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430943209080:2049] [node 2] ICS01 socket: 443 reason# EndOfStream
2025-12-24T02:11:45.088366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241426648241690:2049] [node 2] ICP30 unregister session Session# [5:7587241430943209080:2049] VirtualId# [5:7587241430943209077:0]
2025-12-24T02:11:45.088370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430943209080:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.088394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.088413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.090748Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:45.090847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241430671626771:2049] [node 2] ICS01 socket: 441 reason# EndOfStream
2025-12-24T02:11:45.090855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241426376659378:2049] [node 2] ICP30 unregister session Session# [4:7587241430671626771:2049] VirtualId# [4:7587241430671626763:0]
2025-12-24T02:11:45.090862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241430671626771:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.090915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.090941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.096856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:45.097019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430323975725:2050] [node 3] ICS01 socket: 426 reason# EndOfStream
2025-12-24T02:11:45.097028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426029008338:2050] [node 3] ICP30 unregister session Session# [6:7587241430323975725:2050] VirtualId# [6:7587241426029008424:0]
2025-12-24T02:11:45.097036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430323975725:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.097079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.097104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.097138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:45.097200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430943209086:2050] [node 3] ICS01 socket: 417 reason# EndOfStream
2025-12-24T02:11:45.097206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241426648241691:2050] [node 3] ICP30 unregister session Session# [5:7587241430943209086:2050] VirtualId# [5:7587241430943209076:0]
2025-12-24T02:11:45.097210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430943209086:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.097237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.097255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.118848Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:45.118953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430323975723:2061] [node 4] ICS01 socket: 420 reason# EndOfStream
2025-12-24T02:11:45.118964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241426029008364:2061] [node 4] ICP30 unregister session Session# [6:7587241430323975723:2061] VirtualId# [6:7587241426029008417:0]
2025-12-24T02:11:45.118971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430323975723:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.119030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:45.119054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:45.167080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429134930243:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:45.167090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429134930244:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:45.178821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429532931992:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.178841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429532931993:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:45.194833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241427697165132:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.194845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241427697165133:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.194850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241427697165134:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.205360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241431012013093:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.205371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241431012013094:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.205376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241431012013095:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.198952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430454065378:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.198965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430454065379:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.198971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430454065380:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.194835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431180793611:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.194845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431180793612:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.261238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429134930307:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:45.261393Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241431012013145:2073] [node 4] ICP01 ready to work
2025-12-24T02:11:45.263988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429532932054:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:45.264087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429532932056:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:45.264162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429532932057:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.265652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431180793672:2104] [node 5] ICP01 ready to work
2025-12-24T02:11:45.265742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793671:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.267443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431180793686:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.267775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932065:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-12-24T02:11:45.278085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241431012013135:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:45.279057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430454065444:2105] [node 4] ICP01 ready to work
2025-12-24T02:11:45.283048Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241429134930244:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23476914ns) {Configure:104(23470105ns) {ProcessConfigured:135(23223385ns) {StartInitialHandshake:157(23222338ns) {DropHandshakes:470(250ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(39ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(2129ns) {SwitchToState:179(1477ns)}}}}}
2025-12-24T02:11:45.283261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429134930313:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:45.283367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429134930314:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.284299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241429134930315:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64786
2025-12-24T02:11:45.286819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241430454065378:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25798770ns) {Configure:104(25798216ns) {ProcessConfigured:135(25458443ns) {StartInitialHandshake:157(25458218ns) {DropHandshakes:470(422ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(1872ns) {SwitchToState:179(1447ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.287202Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241427697165133:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25983423ns) {Configure:104(25982379ns) {ProcessConfigured:135(25643304ns) {StartInitialHandshake:157(25642973ns) {DropHandshakes:470(334ns) {DropIncomingHandshake:442(38ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(1299ns) {SwitchToState:179(935ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.287076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932071:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-12-24T02:11:45.287371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241427697165197:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:45.288093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165198:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.287296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241429134930307:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25986099ns) {Configure:104(25985515ns) {ProcessConfigured:135(25699331ns) {StartInitialHandshake:157(25698931ns) {DropHandshakes:470(420ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1103ns) {SwitchToState:179(655ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.290076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241429134930323:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64786
2025-12-24T02:11:45.290104Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241429134930324:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64786
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.289814Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932072:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-12-24T02:11:45.293788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165198:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.290784Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241431012013095:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(29261952ns) {Configure:104(29261424ns) {ProcessConfigured:135(28906757ns) {StartInitialHandshake:157(28906316ns) {DropHandshakes:470(364ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(269ns) PrepareNewSessionHandshake:477(1168ns) {SwitchToState:179(436ns)}}}}}
2025-12-24T02:11:45.291984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013163:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.295403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431180793669:2102] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.298984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165206:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.299024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241427697165133:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.299212Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930323:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.302101Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241431012013145:2073] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23909181ns) {Configure:104(23907713ns) {ProcessConfigured:135(23464420ns) {StartInitialHandshake:157(23463244ns) {DropHandshakes:470(1300ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(721ns) PrepareNewSessionHandshake:477(1757ns) {SwitchToState:179(1189ns)}}}}}
2025-12-24T02:11:45.303202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241431012013163:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.303247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241431012013094:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.309498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065450:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.307322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241427697165209:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:45.310099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241427697165134:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(49620641ns) {Configure:104(49619888ns) {ProcessConfigured:135(49187657ns) {StartInitialHandshake:157(49187474ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(297ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(357ns)}}}}}
2025-12-24T02:11:45.314911Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165206:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.314815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930324:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.321320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430454065454:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.323231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241429532931992:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(59018515ns) {Configure:104(59017792ns) {ProcessConfigured:135(58563024ns) {StartInitialHandshake:157(58562771ns) {DropHandshakes:470(287ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(423ns) PrepareNewSessionHandshake:477(1006ns) {SwitchToState:179(496ns)}}}}}
2025-12-24T02:11:45.323365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932071:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.323386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932072:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.323423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932060:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.323427Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241429532932060:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.323861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932064:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.323864Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241429532932064:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.324500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241429532931992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.322984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793689:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.323037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793690:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.325750Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793690:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.325777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793671:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.325807Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793674:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.325810Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431180793674:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.326656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241429532932060:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.325017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065450:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.325149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430454065380:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.325178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065457:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.325214Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065458:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.325276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065459:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.325316Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065460:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.326554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065462:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.326906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241429532932064:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.326871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930306:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.326885Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241429134930306:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.326976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065460:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.327116Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065458:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.327139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430454065378:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.327290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065457:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.327308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430454065379:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.327282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241429532932054:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.327301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.327308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429532932054:2107] [node 4] ICP22 created new session: [2:7587241429532932078:2107]
2025-12-24T02:11:45.327315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241429532932078:2107] [node 4] ICS09 handshake done sender: [2:7587241429532932060:2110] self: [2:7587241429532932059:0] peer: [4:7587241427697165207:0] socket: 409 qp: -1
2025-12-24T02:11:45.327322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241429532932078:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:45.327333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.327748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431180793674:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.331096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793689:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.331120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431180793672:2104] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.331138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.331146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431180793672:2104] [node 5] ICP22 created new session: [3:7587241431180793697:2104]
2025-12-24T02:11:45.331158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241431180793697:2104] [node 5] ICS09 handshake done sender: [3:7587241431180793674:2107] self: [3:7587241431180793673:0] peer: [5:7587241430454065461:0] socket: 405 qp: -1
2025-12-24T02:11:45.331164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241431180793697:2104] [node 5] ICS10 traffic start
2025-12-24T02:11:45.331176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.331648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430454065450:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.331764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430454065380:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.331779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.331785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430454065380:2050] [node 3] ICP22 created new session: [5:7587241430454065466:2050]
2025-12-24T02:11:45.331803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430454065466:2050] [node 3] ICS09 handshake done sender: [5:7587241430454065450:2112] self: [5:7587241430454065461:0] peer: [3:7587241431180793673:0] socket: 414 qp: -1
2025-12-24T02:11:45.331809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430454065466:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.331815Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.331403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241429532932057:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.331421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.331429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429532932057:2109] [node 6] ICP22 created new session: [2:7587241429532932081:2109]
2025-12-24T02:11:45.331436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241429532932081:2109] [node 6] ICS09 handshake done sender: [2:7587241429532932064:2112] self: [2:7587241429532932063:0] peer: [6:7587241431012013172:0] socket: 412 qp: -1
2025-12-24T02:11:45.331442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241429532932081:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:45.331450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.331450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241431180793611:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.335042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932062:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.335054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241429532932062:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.335585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930315:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.335644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930311:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.335648Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241429134930311:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.336311Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241429134930306:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.336522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930317:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.336538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241429134930317:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.338466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.346830Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241427697165209:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39409850ns) {Configure:104(39408937ns) {ProcessConfigured:135(38940845ns) {StartInitialHandshake:157(38939973ns) {DropHandshakes:470(1098ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(307ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(1866ns) {SwitchToState:179(1072ns)}}}}}
2025-12-24T02:11:45.348466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241427697165133:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15578) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241427697165198:2110] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.348217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241429134930311:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.350823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065459:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.350884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430454065444:2105] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.350814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241431012013163:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.351022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241431012013094:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.351041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.351050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241431012013094:2049] [node 2] ICP22 created new session: [6:7587241431012013174:2049]
2025-12-24T02:11:45.351058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241431012013174:2049] [node 2] ICS09 handshake done sender: [6:7587241431012013163:2112] self: [6:7587241431012013172:0] peer: [2:7587241429532932063:0] socket: 431 qp: -1
2025-12-24T02:11:45.351072Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241431012013174:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.351085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.350903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793700:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.351971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793701:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.351090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429134930243:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.351106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.351114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429134930243:2048] [node 2] ICP22 created new session: [1:7587241429134930330:2048]
2025-12-24T02:11:45.351121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241429134930330:2048] [node 2] ICS09 handshake done sender: [1:7587241429134930306:2108] self: [1:7587241429134930303:0] peer: [2:7587241429532932076:0] socket: 416 qp: -1
2025-12-24T02:11:45.351128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241429134930330:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:45.351136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.352145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241429134930317:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.354888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065470:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.355096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430454065458:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.355182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430454065378:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.355203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.355209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430454065378:2048] [node 1] ICP22 created new session: [5:7587241430454065471:2048]
2025-12-24T02:11:45.355217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430454065471:2048] [node 1] ICS09 handshake done sender: [5:7587241430454065458:2117] self: [5:7587241430454065464:0] peer: [1:7587241429134930316:0] socket: 443 qp: -1
2025-12-24T02:11:45.355227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430454065471:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.355236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.355441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932080:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-12-24T02:11:45.355776Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932065:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.356098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932083:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
2025-12-24T02:11:45.356138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241429532932084:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55868
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.356546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065473:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.356928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241429532932054:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(29520941ns) {HandleHandshakeStatus:329(29520192ns) {DropIncomingHandshake:442(3071ns) DropOutgoingHandshake:457(358ns) SwitchToState:179(194ns) ProcessPendingSessionEvents:495(18039989ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18016089ns max# 17927667ns}}}
2025-12-24T02:11:45.357103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013176:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.356354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165216:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.356411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165215:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.358046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165202:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.358056Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241427697165202:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.358781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241427697165198:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.359434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241427697165202:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.362824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429134930313:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.362859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.362872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429134930313:2109] [node 5] ICP22 created new session: [1:7587241429134930334:2109]
2025-12-24T02:11:45.362881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241429134930334:2109] [node 5] ICS09 handshake done sender: [1:7587241429134930317:2113] self: [1:7587241429134930316:0] peer: [5:7587241430454065464:0] socket: 419 qp: -1
2025-12-24T02:11:45.362888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241429134930334:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:45.362903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.363411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429134930244:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.363423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.363427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429134930244:2049] [node 3] ICP22 created new session: [1:7587241429134930338:2049]
2025-12-24T02:11:45.363432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241429134930338:2049] [node 3] ICS09 handshake done sender: [1:7587241429134930311:2111] self: [1:7587241429134930304:0] peer: [3:7587241431180793699:0] socket: 417 qp: -1
2025-12-24T02:11:45.363437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241429134930338:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.363443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.368304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013178:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.369838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241429134930336:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64786
2025-12-24T02:11:45.369862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241429134930337:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64786
2025-12-24T02:11:45.369148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430454065475:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59730
2025-12-24T02:11:45.370410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430454065459:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.370472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430454065444:2105] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.370486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.370493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430454065444:2105] [node 4] ICP22 created new session: [5:7587241430454065476:2105]
2025-12-24T02:11:45.370499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430454065476:2105] [node 4] ICS09 handshake done sender: [5:7587241430454065459:2118] self: [5:7587241430454065469:0] peer: [4:7587241427697165201:0] socket: 444 qp: -1
2025-12-24T02:11:45.370506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430454065476:2105] [node 4] ICS10 traffic start
2025-12-24T02:11:45.370514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.371234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932083:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.374870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793701:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.374943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431180793689:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.375341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930337:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.375581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241427697165133:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.375607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.375617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241427697165133:2049] [node 2] ICP22 created new session: [4:7587241427697165218:2049]
2025-12-24T02:11:45.375626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241427697165218:2049] [node 2] ICS09 handshake done sender: [4:7587241427697165198:2110] self: [4:7587241427697165207:0] peer: [2:7587241429532932059:0] socket: 421 qp: -1
2025-12-24T02:11:45.377183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241431180793611:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.377202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.377209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431180793611:2048] [node 1] ICP22 created new session: [3:7587241431180793703:2048]
2025-12-24T02:11:45.377226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241431180793703:2048] [node 1] ICS09 handshake done sender: [3:7587241431180793689:2115] self: [3:7587241431180793699:0] peer: [1:7587241429134930304:0] socket: 439 qp: -1
2025-12-24T02:11:45.377232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241431180793703:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.375726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430454065457:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.375784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430454065379:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.375807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.375814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430454065379:2049] [node 2] ICP22 created new session: [5:7587241430454065478:2049]
2025-12-24T02:11:45.371322Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241429532932062:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.378899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241429532932056:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.378912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.378919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429532932056:2108] [node 5] ICP22 created new session: [2:7587241429532932089:2108]
2025-12-24T02:11:45.378925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241429532932089:2108] [node 5] ICS09 handshake done sender: [2:7587241429532932062:2111] self: [2:7587241429532932061:0] peer: [5:7587241430454065465:0] socket: 410 qp: -1
2025-12-24T02:11:45.378932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241429532932089:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:45.378941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.379780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241431012013178:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.379832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241431012013176:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.379866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241431012013135:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.379878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241431012013093:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.380989Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013181:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.382549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241431012013181:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.382581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241431012013145:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.375635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241427697165218:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.375646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.375925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241427697165197:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.375942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.375946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241427697165197:2108] [node 5] ICP22 created new session: [4:7587241427697165220:2108]
2025-12-24T02:11:45.375953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241427697165220:2108] [node 5] ICS09 handshake done sender: [4:7587241427697165202:2112] self: [4:7587241427697165201:0] peer: [5:7587241430454065469:0] socket: 424 qp: -1
2025-12-24T02:11:45.375957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241427697165220:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:45.375962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.380381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165216:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.380789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165222:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.380826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241427697165134:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.382726Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165213:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.382731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241427697165213:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.383096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241427697165222:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.383118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241427697165132:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.383855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241429532932071:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.377239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.383178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793688:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.383183Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431180793688:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.383607Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793702:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.386833Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241431180793611:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:45.387131Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431180793688:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.387344Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930322:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.387359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241429134930322:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.387161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013183:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.375820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430454065478:2049] [node 2] ICS09 handshake done sender: [5:7587241430454065457:2116] self: [5:7587241430454065465:0] peer: [2:7587241429532932061:0] socket: 438 qp: -1
2025-12-24T02:11:45.375826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430454065478:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.375835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.387439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430454065456:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.387447Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241430454065456:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.387596Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241429532932057:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:45.387786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241429532931992:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.387812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.387819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429532931992:2048] [node 1] ICP22 created new session: [2:7587241429532932091:2048]
2025-12-24T02:11:45.387825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241429532932091:2048] [node 1] ICS09 handshake done sender: [2:7587241429532932071:2115] self: [2:7587241429532932076:0] peer: [1:7587241429134930303:0] socket: 420 qp: -1
2025-12-24T02:11:45.387832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241429532932091:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.387840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.388010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241429532932056:2108] [node 5] ICP18 handshake (actor [2:7587241429532932083:2119]) is too old
2025-12-24T02:11:45.387542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930319:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.387551Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241429134930319:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.389115Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241427697165213:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.389233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241427697165209:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.389245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.389250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241427697165209:2114] [node 6] ICP22 created new session: [4:7587241427697165227:2114]
2025-12-24T02:11:45.389256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241427697165227:2114] [node 6] ICS09 handshake done sender: [4:7587241427697165213:2116] self: [4:7587241427697165210:0] peer: [6:7587241431012013182:0] socket: 406 qp: -1
2025-12-24T02:11:45.389262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241427697165227:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:45.389269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.389729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431180793686:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.389746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.389751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431180793686:2111] [node 4] ICP22 created new session: [3:7587241431180793708:2111]
2025-12-24T02:11:45.389756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241431180793708:2111] [node 4] ICS09 handshake done sender: [3:7587241431180793688:2114] self: [3:7587241431180793687:0] peer: [4:7587241427697165224:0] socket: 450 qp: -1
2025-12-24T02:11:45.389760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241431180793708:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:45.389765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.391962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241431012013183:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.391998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241431012013095:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.392339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241429134930319:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.392751Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241429134930336:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.392814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241429134930243:2048] [node 2] ICP18 handshake (actor [1:7587241429134930336:2118]) is too old
2025-12-24T02:11:45.393206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165229:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.394979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430454065456:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.395085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241430454065454:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.395102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.395108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430454065454:2114] [node 6] ICP22 created new session: [5:7587241430454065481:2114]
2025-12-24T02:11:45.395115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430454065481:2114] [node 6] ICS09 handshake done sender: [5:7587241430454065456:2115] self: [5:7587241430454065455:0] peer: [6:7587241431012013179:0] socket: 464 qp: -1
2025-12-24T02:11:45.395121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430454065481:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:45.395129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.395571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241429134930322:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.395638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013187:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.395646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013185:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.396441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241427697165230:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29236
2025-12-24T02:11:45.396541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429134930314:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.396559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.396565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429134930314:2110] [node 6] ICP22 created new session: [1:7587241429134930344:2110]
2025-12-24T02:11:45.396571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241429134930344:2110] [node 6] ICS09 handshake done sender: [1:7587241429134930319:2114] self: [1:7587241429134930318:0] peer: [6:7587241431012013180:0] socket: 455 qp: -1
2025-12-24T02:11:45.396576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241429134930344:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:45.396583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.396871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429134930307:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.396880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.396884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429134930307:2107] [node 4] ICP22 created new session: [1:7587241429134930346:2107]
2025-12-24T02:11:45.396888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241429134930346:2107] [node 4] ICS09 handshake done sender: [1:7587241429134930322:2115] self: [1:7587241429134930308:0] peer: [4:7587241427697165225:0] socket: 428 qp: -1
2025-12-24T02:11:45.396892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241429134930346:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:45.396897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.402932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793692:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.402949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431180793692:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.405048Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431180793702:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.405197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241431180793612:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.406840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431180793692:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.403895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013186:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.404473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241431012013135:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21225) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241431012013178:2118] held: no
2025-12-24T02:11:45.404610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241431012013181:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.405351Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241431012013145:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.405366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.405373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241431012013145:2073] [node 4] ICP22 created new session: [6:7587241431012013188:2073]
2025-12-24T02:11:45.405380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241431012013188:2073] [node 4] ICS09 handshake done sender: [6:7587241431012013181:2119] self: [6:7587241431012013182:0] peer: [4:7587241427697165210:0] socket: 436 qp: -1
2025-12-24T02:11:45.405388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241431012013188:2073] [node 4] ICS10 traffic start
2025-12-24T02:11:45.405397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.405904Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241431012013178:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.406202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241431012013176:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.406011Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241427697165134:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13837) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241427697165216:2118] held: no
2025-12-24T02:11:45.406160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241427697165216:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.406424Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241427697165132:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241427697165222:2119] held: no
2025-12-24T02:11:45.406522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241427697165134:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.406532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.406537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241427697165134:2050] [node 3] ICP22 created new session: [4:7587241427697165231:2050]
2025-12-24T02:11:45.406543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241427697165231:2050] [node 3] ICS09 handshake done sender: [4:7587241427697165216:2118] self: [4:7587241427697165224:0] peer: [3:7587241431180793687:0] socket: 462 qp: -1
2025-12-24T02:11:45.406549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241427697165231:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.406555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.406746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241427697165222:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.406920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431180793669:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.406941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.406948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431180793669:2102] [node 6] ICP22 created new session: [3:7587241431180793712:2102]
2025-12-24T02:11:45.406953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241431180793712:2102] [node 6] ICS09 handshake done sender: [3:7587241431180793692:2117] self: [3:7587241431180793691:0] peer: [6:7587241431012013184:0] socket: 469 qp: -1
2025-12-24T02:11:45.406958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241431180793712:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:45.406966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.408581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241427697165132:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.408595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.408603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241427697165132:2048] [node 1] ICP22 created new session: [4:7587241427697165233:2048]
2025-12-24T02:11:45.408608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241427697165233:2048] [node 1] ICS09 handshake done sender: [4:7587241427697165222:2119] self: [4:7587241427697165225:0] peer: [1:7587241429134930308:0] socket: 404 qp: -1
2025-12-24T02:11:45.408612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241427697165233:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.408619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.409169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241431012013093:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.409184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.409189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241431012013093:2048] [node 1] ICP22 created new session: [6:7587241431012013191:2048]
2025-12-24T02:11:45.409195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241431012013191:2048] [node 1] ICS09 handshake done sender: [6:7587241431012013176:2117] self: [6:7587241431012013180:0] peer: [1:7587241429134930318:0] socket: 457 qp: -1
2025-12-24T02:11:45.409199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241431012013191:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.409203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.409503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241431012013135:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.409509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.409513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241431012013135:2070] [node 5] ICP22 created new session: [6:7587241431012013193:2070]
2025-12-24T02:11:45.409519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241431012013193:2070] [node 5] ICS09 handshake done sender: [6:7587241431012013178:2118] self: [6:7587241431012013179:0] peer: [5:7587241430454065455:0] socket: 465 qp: -1
2025-12-24T02:11:45.409522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241431012013193:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:45.409527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.410778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241431012013093:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:45.410875Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241431012013190:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10342
2025-12-24T02:11:45.411190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241429532932070:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.411207Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241429532932070:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.411988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431180793714:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3382
2025-12-24T02:11:45.412200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241429532932070:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.412372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431180793702:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.412449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241431180793612:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.412464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.412469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431180793612:2049] [node 2] ICP22 created new session: [3:7587241431180793715:2049]
2025-12-24T02:11:45.412474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241431180793715:2049] [node 2] ICS09 handshake done sender: [3:7587241431180793702:2120] self: [3:7587241431180793711:0] peer: [2:7587241429532932069:0] socket: 425 qp: -1
2025-12-24T02:11:45.412479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241431180793715:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.412485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.413361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241429532931993:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.413380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.413385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429532931993:2049] [node 3] ICP22 created new session: [2:7587241429532932094:2049]
2025-12-24T02:11:45.413390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241429532932094:2049] [node 3] ICS09 handshake done sender: [2:7587241429532932070:2114] self: [2:7587241429532932069:0] peer: [3:7587241431180793711:0] socket: 460 qp: -1
2025-12-24T02:11:45.413395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241429532932094:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.413403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.414806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241431012013095:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13837) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241431012013183:2120] held: no
2025-12-24T02:11:45.415003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241431012013183:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.418768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241431012013095:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.418787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.418795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241431012013095:2050] [node 3] ICP22 created new session: [6:7587241431012013195:2050]
2025-12-24T02:11:45.418802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241431012013195:2050] [node 3] ICS09 handshake done sender: [6:7587241431012013183:2120] self: [6:7587241431012013184:0] peer: [3:7587241431180793691:0] socket: 470 qp: -1
2025-12-24T02:11:45.418810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241431012013195:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.418818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.442070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:45.442181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241429532932091:2048] [node 1] ICS07 socket disconnect 420 reason# EndOfStream
2025-12-24T02:11:45.442189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241429532932091:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.442228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.442247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241429532932091:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.442930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:45.443012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241431180793703:2048] [node 1] ICS07 socket disconnect 439 reason# EndOfStream
2025-12-24T02:11:45.443019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241431180793703:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.443061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.443082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241431180793703:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.445733Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241429532931992:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:45.445748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241429532932091:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.446032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241429532931992:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.446044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241429532932091:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.446050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241429532931992:2048] [node 1] ICP30 unregister session Session# [2:7587241429532932091:2048] VirtualId# [2:7587241429532932076:0]
2025-12-24T02:11:45.446110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.446117Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241429532931992:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.446120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.451273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241431180793611:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:45.451288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241431180793703:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.451578Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241431180793611:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.451591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241431180793703:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.451597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241431180793611:2048] [node 1] ICP30 unregister session Session# [3:7587241431180793703:2048] VirtualId# [3:7587241431180793699:0]
2025-12-24T02:11:45.451656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.451663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241431180793611:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.451666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4861) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.459103Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:45.459200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430454065471:2048] [node 1] ICS01 socket: 443 reason# EndOfStream
2025-12-24T02:11:45.459208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430454065378:2048] [node 1] ICP30 unregister session Session# [5:7587241430454065471:2048] VirtualId# [5:7587241430454065464:0]
2025-12-24T02:11:45.459216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430454065471:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.459255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.459279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.459044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:45.459132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241427697165233:2048] [node 1] ICS01 socket: 404 reason# EndOfStream
2025-12-24T02:11:45.459144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241427697165132:2048] [node 1] ICP30 unregister session Session# [4:7587241427697165233:2048] VirtualId# [4:7587241427697165225:0]
2025-12-24T02:11:45.459152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241427697165233:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.459191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.459220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.459258Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:45.459318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241431012013191:2048] [node 1] ICS01 socket: 457 reason# EndOfStream
2025-12-24T02:11:45.459322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241431012013093:2048] [node 1] ICP30 unregister session Session# [6:7587241431012013191:2048] VirtualId# [6:7587241431012013180:0]
2025-12-24T02:11:45.459327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241431012013191:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.459345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.459363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.463032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241431180793611:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:45.469958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:45.470066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241431180793715:2049] [node 2] ICS01 socket: 425 reason# EndOfStream
2025-12-24T02:11:45.470075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241431180793612:2049] [node 2] ICP30 unregister session Session# [3:7587241431180793715:2049] VirtualId# [3:7587241431180793711:0]
2025-12-24T02:11:45.470082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241431180793715:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.470127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.470148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.482493Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:45.482602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241427697165218:2049] [node 2] ICS01 socket: 421 reason# EndOfStream
2025-12-24T02:11:45.482613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241427697165133:2049] [node 2] ICP30 unregister session Session# [4:7587241427697165218:2049] VirtualId# [4:7587241427697165207:0]
2025-12-24T02:11:45.482621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241427697165218:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.482684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.482723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.482766Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:45.482829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241431012013174:2049] [node 2] ICS01 socket: 431 reason# EndOfStream
2025-12-24T02:11:45.482834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241431012013094:2049] [node 2] ICP30 unregister session Session# [6:7587241431012013174:2049] VirtualId# [6:7587241431012013172:0]
2025-12-24T02:11:45.482838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241431012013174:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.482866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.482886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.482940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:45.483056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430454065478:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:45.483064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430454065379:2049] [node 2] ICP30 unregister session Session# [5:7587241430454065478:2049] VirtualId# [5:7587241430454065465:0]
2025-12-24T02:11:45.483071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430454065478:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.483130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.483160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.498893Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:45.499002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241431012013195:2050] [node 3] ICS01 socket: 470 reason# EndOfStream
2025-12-24T02:11:45.499021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241431012013095:2050] [node 3] ICP30 unregister session Session# [6:7587241431012013195:2050] VirtualId# [6:7587241431012013184:0]
2025-12-24T02:11:45.499030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241431012013195:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.499080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.499104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.499142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:45.499212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241430454065466:2050] [node 3] ICS07 socket disconnect 414 reason# EndOfStream
2025-12-24T02:11:45.499218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430454065466:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.499242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.499264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241430454065466:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:45.500065Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430454065380:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13837) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.500075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430454065466:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.500080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430454065380:2050] [node 3] ICP30 unregister session Session# [5:7587241430454065466:2050] VirtualId# [5:7587241430454065461:0]
2025-12-24T02:11:45.500116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.500123Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241430454065380:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.500127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13837) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.511687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241430454065380:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:45.519144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:45.519264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241431012013188:2073] [node 4] ICS01 socket: 436 reason# EndOfStream
2025-12-24T02:11:45.519274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241431012013145:2073] [node 4] ICP30 unregister session Session# [6:7587241431012013188:2073] VirtualId# [6:7587241431012013182:0]
2025-12-24T02:11:45.519282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241431012013188:2073] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.519331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:45.519356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:45.555040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241428975154876:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:45.555056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241428975154877:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:45.563131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241430491724779:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.563147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241430491724780:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.563153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241430491724781:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.563437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430162786506:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.563454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430162786507:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.563460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430162786508:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.562960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241427761176150:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.562976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241427761176151:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:45.567137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241430296213800:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.567146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241430296213801:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:45.567153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241430296213802:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.559056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241427755782236:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.559067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241427755782237:2049] [node 2] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.611612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241430296213840:2086] [node 5] ICP01 ready to work
2025-12-24T02:11:45.612570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782296:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.611856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241428975154938:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:45.611960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241428975154939:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:45.615139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241428975154948:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.615438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786569:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.615818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430162786564:2104] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.615824Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241430162786564:2104] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.615056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782296:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.615203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241427755782298:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:45.615574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241427755782298:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.617611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241430296213844:2088] [node 4] ICP01 ready to work
2025-12-24T02:11:45.615849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724832:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.623181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430162786564:2104] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.625410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241430491724836:2102] [node 6] ICP01 ready to work
2025-12-24T02:11:45.627375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241427761176215:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.628268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241428975154954:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1041
2025-12-24T02:11:45.629858Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724841:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.630666Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782306:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.631001Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241427755782296:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.633899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724832:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.634800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782307:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
2025-12-24T02:11:45.639880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241430162786508:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.639907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.639917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430162786508:2050] [node 3] ICP22 created new session: [5:7587241430162786576:2050]
2025-12-24T02:11:45.639925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430162786576:2050] [node 3] ICS09 handshake done sender: [5:7587241430162786564:2104] self: [5:7587241430162786563:0] peer: [3:7587241427755782299:0] socket: 408 qp: -1
2025-12-24T02:11:45.639937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430162786576:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.639959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.643060Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241428975154956:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1041
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.643284Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430162786569:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.643369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430162786506:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.644398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241430491724836:2102] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18900309ns) {Configure:104(18899649ns) {ProcessConfigured:135(18453477ns) {StartInitialHandshake:157(18452799ns) {DropHandshakes:470(612ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(540ns) PrepareNewSessionHandshake:477(970ns) {SwitchToState:179(145ns)}}}}}
2025-12-24T02:11:45.644490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724841:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.643565Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154956:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.643595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154944:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.643600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241428975154944:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.645197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241428975154944:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.645383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241428975154939:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.645405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.645411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241428975154939:2108] [node 5] ICP22 created new session: [1:7587241428975154959:2108]
2025-12-24T02:11:45.645417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428975154959:2108] [node 5] ICS09 handshake done sender: [1:7587241428975154944:2110] self: [1:7587241428975154943:0] peer: [5:7587241430162786579:0] socket: 412 qp: -1
2025-12-24T02:11:45.645424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428975154959:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:45.645443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.643411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213874:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.646978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430296213874:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.647532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241427761176211:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:45.649482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430162786506:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241430162786569:2107] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.649239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241430296213800:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.651353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241427755782298:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.651378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.651386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241427755782298:2108] [node 5] ICP22 created new session: [3:7587241427755782310:2108]
2025-12-24T02:11:45.651404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427755782310:2108] [node 5] ICS09 handshake done sender: [3:7587241427755782296:2107] self: [3:7587241427755782299:0] peer: [5:7587241430162786563:0] socket: 409 qp: -1
2025-12-24T02:11:45.651412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427755782310:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:45.651450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.651879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241427755782312:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:45.651958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782307:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.652036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241427755782313:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:45.650973Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786580:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.651045Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786581:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.653558Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430162786569:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.653668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430162786506:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.653684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.653690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430162786506:2048] [node 1] ICP22 created new session: [5:7587241430162786582:2048]
2025-12-24T02:11:45.653697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430162786582:2048] [node 1] ICS09 handshake done sender: [5:7587241430162786569:2107] self: [5:7587241430162786579:0] peer: [1:7587241428975154943:0] socket: 413 qp: -1
2025-12-24T02:11:45.653703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430162786582:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.653720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.654425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241427761176150:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26991618ns) {Configure:104(26991176ns) {ProcessConfigured:135(26757920ns) {StartInitialHandshake:157(26757719ns) {DropHandshakes:470(187ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(107ns) PrepareNewSessionHandshake:477(992ns) {SwitchToState:179(137ns)}}}}}
2025-12-24T02:11:45.654564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241427761176220:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:45.656021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154950:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.656035Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241428975154950:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.656984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241428975154950:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.654178Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430162786580:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.654355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430162786585:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:45.654790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430162786585:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.657221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782320:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.657149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241428975154948:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.657164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.657171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241428975154948:2111] [node 6] ICP22 created new session: [1:7587241428975154964:2111]
2025-12-24T02:11:45.657177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428975154964:2111] [node 6] ICS09 handshake done sender: [1:7587241428975154950:2112] self: [1:7587241428975154949:0] peer: [6:7587241430296213875:0] socket: 414 qp: -1
2025-12-24T02:11:45.657183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428975154964:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:45.657190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.657367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241428975154963:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1041
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.658831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154954:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.657549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782321:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
2025-12-24T02:11:45.659340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782315:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.659344Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241427755782315:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.660292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241427755782315:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.660689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430162786588:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:45.660435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241427755782312:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.660460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.660464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241427755782312:2113] [node 6] ICP22 created new session: [3:7587241427755782323:2113]
2025-12-24T02:11:45.660470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427755782323:2113] [node 6] ICS09 handshake done sender: [3:7587241427755782315:2115] self: [3:7587241427755782314:0] peer: [6:7587241430296213880:0] socket: 441 qp: -1
2025-12-24T02:11:45.660475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427755782323:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:45.660482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.658954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213876:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.658975Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213877:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.659025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213878:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.659036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213879:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.659231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430296213877:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.659257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241430296213802:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.660023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.660188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430296213874:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.660338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241430296213800:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.660351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.660357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241430296213800:2048] [node 1] ICP22 created new session: [6:7587241430296213882:2048]
2025-12-24T02:11:45.660363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430296213882:2048] [node 1] ICS09 handshake done sender: [6:7587241430296213874:2113] self: [6:7587241430296213875:0] peer: [1:7587241428975154949:0] socket: 421 qp: -1
2025-12-24T02:11:45.660369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430296213882:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.660378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.660902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430296213877:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.660953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241430296213802:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.660964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.660969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241430296213802:2050] [node 3] ICP22 created new session: [6:7587241430296213884:2050]
2025-12-24T02:11:45.660974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430296213884:2050] [node 3] ICS09 handshake done sender: [6:7587241430296213877:2115] self: [6:7587241430296213880:0] peer: [3:7587241427755782314:0] socket: 451 qp: -1
2025-12-24T02:11:45.660978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430296213884:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.660984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.663433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154942:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.663450Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241428975154942:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.663483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241430491724779:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.663547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241430491724779:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(29536012ns) {Configure:104(29535595ns) {ProcessConfigured:135(28999853ns) {IncomingHandshake:261(28989494ns) {DropIncomingHandshake:442(258ns) PrepareNewSessionHandshake:477(641ns) {SwitchToState:179(82ns)} GenerateSessionVirtualId:988(436ns) IssueIncomingHandshakeReply:193(28983398ns) {SwitchToState:179(588ns)}}}}}
2025-12-24T02:11:45.664011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430296213851:2095] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.664030Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241430296213851:2095] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.662183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176228:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
2025-12-24T02:11:45.662287Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176229:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
2025-12-24T02:11:45.662311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176230:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
2025-12-24T02:11:45.662336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176231:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
2025-12-24T02:11:45.667773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782321:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.667897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241427755782236:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.671345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154955:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.671358Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241428975154955:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.666650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176228:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.666680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176231:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.671137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241430491724853:2113] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.680919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241428975154942:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.680470Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724855:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.680527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724856:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.680541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.681141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241428975154938:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.681159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.681166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241428975154938:2107] [node 4] ICP22 created new session: [1:7587241428975154969:2107]
2025-12-24T02:11:45.681173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428975154969:2107] [node 4] ICS09 handshake done sender: [1:7587241428975154942:2109] self: [1:7587241428975154941:0] peer: [4:7587241430491724842:0] socket: 411 qp: -1
2025-12-24T02:11:45.681181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428975154969:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:45.681192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.680717Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430296213876:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.680773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241430296213844:2088] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.682024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786593:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.682051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786594:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.682317Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724855:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.682579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724846:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.682588Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241430491724846:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.683011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241430491724832:2100] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.683591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241427761176150:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.686842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782320:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.686891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241427755782237:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.687192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430162786580:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.686733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176229:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.689547Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241430491724781:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(18326893ns) {Configure:104(18325845ns) {ProcessConfigured:135(17792656ns) {StartInitialHandshake:157(17791799ns) {DropHandshakes:470(1184ns) {DropIncomingHandshake:442(216ns) DropOutgoingHandshake:457(314ns)} GenerateSessionVirtualId:988(289ns) PrepareNewSessionHandshake:477(1228ns) {SwitchToState:179(1002ns)}}}}}
2025-12-24T02:11:45.689577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241430491724779:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.689587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.689594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241430491724779:2048] [node 1] ICP22 created new session: [4:7587241430491724863:2048]
2025-12-24T02:11:45.689600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430491724863:2048] [node 1] ICS09 handshake done sender: [4:7587241430491724832:2100] self: [4:7587241430491724842:0] peer: [1:7587241428975154941:0] socket: 415 qp: -1
2025-12-24T02:11:45.689606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430491724863:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.689623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.686652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430296213851:2095] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.687278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241430296213840:2086] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.687290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.687297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241430296213840:2086] [node 5] ICP22 created new session: [6:7587241430296213888:2086]
2025-12-24T02:11:45.687303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430296213888:2086] [node 5] ICS09 handshake done sender: [6:7587241430296213851:2095] self: [6:7587241430296213841:0] peer: [5:7587241430162786586:0] socket: 417 qp: -1
2025-12-24T02:11:45.687311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430296213888:2086] [node 5] ICS10 traffic start
2025-12-24T02:11:45.687319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.690592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241427755782317:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.690604Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241427755782317:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.693208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241427755782317:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.693770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241427755782313:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.693784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.693789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241427755782313:2114] [node 4] ICP22 created new session: [3:7587241427755782330:2114]
2025-12-24T02:11:45.693795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427755782330:2114] [node 4] ICS09 handshake done sender: [3:7587241427755782317:2116] self: [3:7587241427755782316:0] peer: [4:7587241430491724865:0] socket: 456 qp: -1
2025-12-24T02:11:45.693800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427755782330:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:45.693808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.693873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782331:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
2025-12-24T02:11:45.692670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241428975154955:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.694643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241428975154877:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.694660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.694666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241428975154877:2049] [node 3] ICP22 created new session: [1:7587241428975154972:2049]
2025-12-24T02:11:45.694672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428975154972:2049] [node 3] ICS09 handshake done sender: [1:7587241428975154955:2115] self: [1:7587241428975154952:0] peer: [3:7587241427755782326:0] socket: 448 qp: -1
2025-12-24T02:11:45.694676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428975154972:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.694684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.693247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430162786585:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.693270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.693303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430162786585:2113] [node 6] ICP22 created new session: [5:7587241430162786595:2113]
2025-12-24T02:11:45.693314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430162786595:2113] [node 6] ICS09 handshake done sender: [5:7587241430162786580:2111] self: [5:7587241430162786586:0] peer: [6:7587241430296213841:0] socket: 424 qp: -1
2025-12-24T02:11:45.693320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430162786595:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:45.693330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.689948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724856:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.689981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241430491724781:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.691971Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241430491724846:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.692181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241430491724836:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.692197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.692205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241430491724836:2102] [node 6] ICP22 created new session: [4:7587241430491724869:2102]
2025-12-24T02:11:45.692212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430491724869:2102] [node 6] ICS09 handshake done sender: [4:7587241430491724846:2111] self: [4:7587241430491724837:0] peer: [6:7587241430296213886:0] socket: 436 qp: -1
2025-12-24T02:11:45.692217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430491724869:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:45.692253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.694003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724868:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.694161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241430491724855:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.694202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241430491724781:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.694212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.694237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241430491724781:2050] [node 3] ICP22 created new session: [4:7587241430491724871:2050]
2025-12-24T02:11:45.694242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430491724871:2050] [node 3] ICS09 handshake done sender: [4:7587241430491724855:2114] self: [4:7587241430491724865:0] peer: [3:7587241427755782316:0] socket: 459 qp: -1
2025-12-24T02:11:45.694248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430491724871:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:45.694254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:45.700170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241427755782236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241427755782321:2118] held: no
2025-12-24T02:11:45.718609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782328:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
2025-12-24T02:11:45.716912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154953:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.716952Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241428975154953:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.718952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176214:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.718976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241427761176214:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.720014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241427761176214:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:45.720125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176230:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.720139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241427761176151:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.720159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:45.720167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241427761176151:2049] [node 3] ICP22 created new session: [2:7587241427761176235:2049]
2025-12-24T02:11:45.720173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241427761176235:2049] [node 3] ICS09 handshake done sender: [2:7587241427761176214:2109] self: [2:7587241427761176213:0] peer: [3:7587241427755782327:0] socket: 442 qp: -1
2025-12-24T02:11:45.720180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241427761176235:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:45.720188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.720885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724873:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.721499Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241427755782237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241427755782320:2117] held: no
2025-12-24T02:11:45.721635Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241427755782333:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63040
2025-12-24T02:11:45.721767Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241427755782321:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.721858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241427755782236:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.721874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.721880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241427755782236:2048] [node 1] ICP22 created new session: [3:7587241427755782334:2048]
2025-12-24T02:11:45.721886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427755782334:2048] [node 1] ICS09 handshake done sender: [3:7587241427755782321:2118] self: [3:7587241427755782326:0] peer: [1:7587241428975154952:0] socket: 449 qp: -1
2025-12-24T02:11:45.721892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427755782334:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.721900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.722462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241427755782320:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.722518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241427755782237:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.722530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.722535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241427755782237:2049] [node 2] ICP22 created new session: [3:7587241427755782336:2049]
2025-12-24T02:11:45.722540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241427755782336:2049] [node 2] ICS09 handshake done sender: [3:7587241427755782320:2117] self: [3:7587241427755782327:0] peer: [2:7587241427761176213:0] socket: 445 qp: -1
2025-12-24T02:11:45.722545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241427755782336:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.722550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.722650Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241427761176215:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(74989856ns) {Configure:104(74989271ns) {ProcessConfigured:135(74589502ns) {StartInitialHandshake:157(74588893ns) {DropHandshakes:470(501ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(86ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(1275ns) {SwitchToState:179(824ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.724278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241430296213878:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.724333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241430296213844:2088] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241430296213876:2114] held: no
2025-12-24T02:11:45.724542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213890:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
2025-12-24T02:11:45.724806Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241430296213840:2086] [node 5] ICP35 event processing took too much time PendingConnection:279(37418664ns) {HandleHandshakeStatus:329(37417980ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(321ns) SwitchToState:179(43ns) ProcessPendingSessionEvents:495(37352096ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 37332256ns max# 37256388ns}}}
2025-12-24T02:11:45.724824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241430296213801:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.725566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241428975154963:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.732946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724873:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.733004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241430491724780:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.733386Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430162786593:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.733455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430162786588:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.733124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176224:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.733185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241427761176224:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.733734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241430491724859:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.733747Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241430491724859:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.734462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241427761176224:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.734573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241427761176211:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.734594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.734600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241427761176211:2107] [node 4] ICP22 created new session: [2:7587241427761176242:2107]
2025-12-24T02:11:45.734605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241427761176242:2107] [node 4] ICS09 handshake done sender: [2:7587241427761176224:2113] self: [2:7587241427761176223:0] peer: [4:7587241430491724874:0] socket: 428 qp: -1
2025-12-24T02:11:45.734611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241427761176242:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:45.734619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.734785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430296213876:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.734870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241430296213844:2088] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.734881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.734887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241430296213844:2088] [node 4] ICP22 created new session: [6:7587241430296213895:2088]
2025-12-24T02:11:45.734894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430296213895:2088] [node 4] ICS09 handshake done sender: [6:7587241430296213876:2114] self: [6:7587241430296213886:0] peer: [4:7587241430491724837:0] socket: 439 qp: -1
2025-12-24T02:11:45.734899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430296213895:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:45.734907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.738783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176222:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.738795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241427761176222:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:45.738932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241430491724875:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60756
2025-12-24T02:11:45.739099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241430491724859:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.739900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241428975154953:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.740091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241428975154876:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.740113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.740121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241428975154876:2048] [node 2] ICP22 created new session: [1:7587241428975154976:2048]
2025-12-24T02:11:45.740127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241428975154976:2048] [node 2] ICS09 handshake done sender: [1:7587241428975154953:2113] self: [1:7587241428975154940:0] peer: [2:7587241427761176232:0] socket: 438 qp: -1
2025-12-24T02:11:45.740134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241428975154976:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:45.740142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.751501Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176246:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.752738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241430296213897:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16155
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.755502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241430296213878:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.754929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241427761176150:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241427761176231:2117] held: no
2025-12-24T02:11:45.755092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241427761176231:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:45.755195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241427761176245:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19040
2025-12-24T02:11:45.755310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241427761176222:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:45.755851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241430491724853:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.755876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.755883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241430491724853:2113] [node 5] ICP22 created new session: [4:7587241430491724877:2113]
2025-12-24T02:11:45.755889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430491724877:2113] [node 5] ICS09 handshake done sender: [4:7587241430491724859:2117] self: [4:7587241430491724858:0] peer: [5:7587241430162786597:0] socket: 462 qp: -1
2025-12-24T02:11:45.755896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430491724877:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:45.755904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.756151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241430491724780:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241430491724873:2120] held: no
2025-12-24T02:11:45.755602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241430296213801:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.755623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.755631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241430296213801:2049] [node 2] ICP22 created new session: [6:7587241430296213898:2049]
2025-12-24T02:11:45.755637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241430296213898:2049] [node 2] ICS09 handshake done sender: [6:7587241430296213878:2116] self: [6:7587241430296213893:0] peer: [2:7587241427761176221:0] socket: 452 qp: -1
2025-12-24T02:11:45.755644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241430296213898:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.755652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.756708Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176245:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.756723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241427761176150:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.756733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:45.756739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241427761176150:2048] [node 1] ICP22 created new session: [2:7587241427761176248:2048]
2025-12-24T02:11:45.756744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241427761176248:2048] [node 1] ICS09 handshake done sender: [2:7587241427761176231:2117] self: [2:7587241427761176232:0] peer: [1:7587241428975154940:0] socket: 458 qp: -1
2025-12-24T02:11:45.756749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241427761176248:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:45.756757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:45.757217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241427761176220:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.757231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:45.757236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241427761176220:2110] [node 6] ICP22 created new session: [2:7587241427761176251:2110]
2025-12-24T02:11:45.757241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241427761176251:2110] [node 6] ICS09 handshake done sender: [2:7587241427761176222:2112] self: [2:7587241427761176221:0] peer: [6:7587241430296213893:0] socket: 444 qp: -1
2025-12-24T02:11:45.757246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241427761176251:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:45.757265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:45.758456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241427755782313:2114] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:45.758229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241430491724873:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.761361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241430491724780:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.761378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.761386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241430491724780:2049] [node 2] ICP22 created new session: [4:7587241430491724879:2049]
2025-12-24T02:11:45.761393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241430491724879:2049] [node 2] ICS09 handshake done sender: [4:7587241430491724873:2120] self: [4:7587241430491724874:0] peer: [2:7587241427761176223:0] socket: 429 qp: -1
2025-12-24T02:11:45.761400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241430491724879:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.761409Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.767098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786598:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.767732Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430162786588:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21741) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241430162786593:2117] held: no
2025-12-24T02:11:45.767925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786599:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.768206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430162786593:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:45.768276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430162786588:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.768291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:45.768301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430162786588:2114] [node 4] ICP22 created new session: [5:7587241430162786600:2114]
2025-12-24T02:11:45.768307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430162786600:2114] [node 4] ICS09 handshake done sender: [5:7587241430162786593:2117] self: [5:7587241430162786597:0] peer: [4:7587241430491724858:0] socket: 464 qp: -1
2025-12-24T02:11:45.768315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430162786600:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:45.768324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:45.771388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241427761176239:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.771400Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241427761176239:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:45.772293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241427761176239:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:45.771146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241430162786599:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:45.771204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430162786507:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:45.772191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241430162786603:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36457
2025-12-24T02:11:45.772465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241430162786599:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:45.772528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430162786507:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:45.772541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:45.772549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430162786507:2049] [node 2] ICP22 created new session: [5:7587241430162786604:2049]
2025-12-24T02:11:45.772555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241430162786604:2049] [node 2] ICS09 handshake done sender: [5:7587241430162786599:2120] self: [5:7587241430162786602:0] peer: [2:7587241427761176217:0] socket: 427 qp: -1
2025-12-24T02:11:45.772561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241430162786604:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:45.772569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:45.774090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241427761176215:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:45.774112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:45.774119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241427761176215:2108] [node 5] ICP22 created new session: [2:7587241427761176254:2108]
2025-12-24T02:11:45.774124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241427761176254:2108] [node 5] ICS09 handshake done sender: [2:7587241427761176239:2118] self: [2:7587241427761176217:0] peer: [5:7587241430162786602:0] socket: 426 qp: -1
2025-12-24T02:11:45.774130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241427761176254:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:45.774138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:45.807032Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:45.807151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241427761176248:2048] [node 1] ICS01 socket: 458 reason# EndOfStream
2025-12-24T02:11:45.807162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241427761176150:2048] [node 1] ICP30 unregister session Session# [2:7587241427761176248:2048] VirtualId# [2:7587241427761176232:0]
2025-12-24T02:11:45.807172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241427761176248:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.807211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.807235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.810017Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:45.810105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241427755782334:2048] [node 1] ICS07 socket disconnect 449 reason# EndOfStream
2025-12-24T02:11:45.810111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241427755782334:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.810144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.810162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241427755782334:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.810909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241427755782236:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.810922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241427755782334:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.810928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241427755782236:2048] [node 1] ICP30 unregister session Session# [3:7587241427755782334:2048] VirtualId# [3:7587241427755782326:0]
2025-12-24T02:11:45.810946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.810953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241427755782236:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.810956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.819791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:45.819908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241430162786582:2048] [node 1] ICS07 socket disconnect 413 reason# EndOfStream
2025-12-24T02:11:45.819917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430162786582:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.819964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.819984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241430162786582:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.820625Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430162786506:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.820636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430162786582:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.820641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430162786506:2048] [node 1] ICP30 unregister session Session# [5:7587241430162786582:2048] VirtualId# [5:7587241430162786579:0]
2025-12-24T02:11:45.820660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.820666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241430162786506:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.820678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.822762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:45.822879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241430491724863:2048] [node 1] ICS07 socket disconnect 415 reason# EndOfStream
2025-12-24T02:11:45.822886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241430491724863:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.822947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.822973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241430491724863:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:45.823909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241427755782236:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:45.824094Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241430491724779:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.824105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241430491724863:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.824111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241430491724779:2048] [node 1] ICP30 unregister session Session# [4:7587241430491724863:2048] VirtualId# [4:7587241430491724842:0]
2025-12-24T02:11:45.824143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.824149Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241430491724779:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.824151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4356) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.820202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:45.820265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430296213882:2048] [node 1] ICS01 socket: 421 reason# EndOfStream
2025-12-24T02:11:45.820273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241430296213800:2048] [node 1] ICP30 unregister session Session# [6:7587241430296213882:2048] VirtualId# [6:7587241430296213875:0]
2025-12-24T02:11:45.820279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430296213882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.820305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:45.820329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:45.834021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:45.834128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241427755782336:2049] [node 2] ICS07 socket disconnect 445 reason# EndOfStream
2025-12-24T02:11:45.834136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241427755782336:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.834173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.834191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241427755782336:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:45.834637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241430162786506:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:45.836446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241427755782237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:45.836461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241427755782336:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:45.836827Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241427755782237:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.836838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241427755782336:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.836845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241427755782237:2049] [node 2] ICP30 unregister session Session# [3:7587241427755782336:2049] VirtualId# [3:7587241427755782327:0]
2025-12-24T02:11:45.836875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.836882Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241427755782237:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.836884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24650) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.838742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241430491724779:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:45.844610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241427755782237:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:45.846829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:45.846928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241430491724879:2049] [node 2] ICS01 socket: 429 reason# EndOfStream
2025-12-24T02:11:45.846935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241430491724780:2049] [node 2] ICP30 unregister session Session# [4:7587241430491724879:2049] VirtualId# [4:7587241430491724874:0]
2025-12-24T02:11:45.846943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241430491724879:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.847003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.847028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.847431Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:45.847501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430296213898:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:45.847510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241430296213801:2049] [node 2] ICP30 unregister session Session# [6:7587241430296213898:2049] VirtualId# [6:7587241430296213893:0]
2025-12-24T02:11:45.847516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430296213898:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.847543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.847569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.847256Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:45.847326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430162786604:2049] [node 2] ICS01 socket: 427 reason# EndOfStream
2025-12-24T02:11:45.847332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430162786507:2049] [node 2] ICP30 unregister session Session# [5:7587241430162786604:2049] VirtualId# [5:7587241430162786602:0]
2025-12-24T02:11:45.847339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430162786604:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.847367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:45.847389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:45.869856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:45.869967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241430296213884:2050] [node 3] ICS07 socket disconnect 451 reason# EndOfStream
2025-12-24T02:11:45.869974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430296213884:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.870036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.870058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241430296213884:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:45.870475Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:45.870550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241430162786576:2050] [node 3] ICS01 socket: 408 reason# EndOfStream
2025-12-24T02:11:45.870557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430162786508:2050] [node 3] ICP30 unregister session Session# [5:7587241430162786576:2050] VirtualId# [5:7587241430162786563:0]
2025-12-24T02:11:45.870567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241430162786576:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.870594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:45.870621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.871006Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241430296213802:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16630) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.871017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430296213884:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.871022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241430296213802:2050] [node 3] ICP30 unregister session Session# [6:7587241430296213884:2050] VirtualId# [6:7587241430296213880:0]
2025-12-24T02:11:45.871053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:45.871059Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241430296213802:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.871062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16630) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.882264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241430296213802:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:45.914618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:45.914739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430296213895:2088] [node 4] ICS01 socket: 439 reason# EndOfStream
2025-12-24T02:11:45.914750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241430296213844:2088] [node 4] ICP30 unregister session Session# [6:7587241430296213895:2088] VirtualId# [6:7587241430296213886:0]
2025-12-24T02:11:45.914760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430296213895:2088] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.914982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:45.915013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:45.934807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:45.934915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241430296213888:2086] [node 5] ICS07 socket disconnect 417 reason# EndOfStream
2025-12-24T02:11:45.934923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241430296213888:2086] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:45.934956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:45.934982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241430296213888:2086] [node 5] ICS15 start handshake
2025-12-24T02:11:45.935405Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241430296213840:2086] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27022) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:45.935418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241430296213888:2086] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:45.935423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241430296213840:2086] [node 5] ICP30 unregister session Session# [6:7587241430296213888:2086] VirtualId# [6:7587241430296213841:0]
2025-12-24T02:11:45.935477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:45.935483Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241430296213840:2086] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:45.935486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27022) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:45.982931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429556818621:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.982957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241429556818622:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:45.983383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429866705220:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:45.983392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241429866705221:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:46.003056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430468715755:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.003067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430468715756:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.003073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241430468715757:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.002843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241429297419757:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.002856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241429297419758:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.002862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241429297419759:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:45.986950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241428069969733:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:45.986960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241428069969734:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.020400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241433760680181:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.020412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241433760680182:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.020418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241433760680183:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.050127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241433760680243:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:46.050124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434161672577:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.051156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387114:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-12-24T02:11:46.050984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683112:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.053341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387114:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.053436Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241434763683112:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.053662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241434763683114:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:46.053724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241429297419757:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.054157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241434763683114:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.053927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434161672579:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.053940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434161672579:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.054331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241433760680245:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.054345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241433760680245:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.054738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434161672579:2108] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.054883Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387116:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-12-24T02:11:46.055013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241433760680245:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.054882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434161672577:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.054898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.054905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434161672577:2107] [node 4] ICP22 created new session: [1:7587241434161672582:2107]
2025-12-24T02:11:46.054911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434161672582:2107] [node 4] ICS09 handshake done sender: [1:7587241434161672579:2108] self: [1:7587241434161672578:0] peer: [4:7587241433592387115:0] socket: 415 qp: -1
2025-12-24T02:11:46.054919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434161672582:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:46.054932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.055157Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241433592387114:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.055166Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683116:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.055209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241433760680243:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.055224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.055231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241433760680243:2108] [node 5] ICP22 created new session: [6:7587241433760680248:2108]
2025-12-24T02:11:46.055236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241433760680248:2108] [node 5] ICS09 handshake done sender: [6:7587241433760680245:2109] self: [6:7587241433760680244:0] peer: [5:7587241434763683115:0] socket: 414 qp: -1
2025-12-24T02:11:46.055241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241433760680248:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:46.055261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.055250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241434763683112:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.055360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241429297419757:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.055381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.055388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241429297419757:2048] [node 1] ICP22 created new session: [4:7587241433592387117:2048]
2025-12-24T02:11:46.055408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241434763683114:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.055422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.055429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241434763683114:2109] [node 6] ICP22 created new session: [5:7587241434763683117:2109]
2025-12-24T02:11:46.055435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241434763683117:2109] [node 6] ICS09 handshake done sender: [5:7587241434763683112:2108] self: [5:7587241434763683115:0] peer: [6:7587241433760680244:0] socket: 416 qp: -1
2025-12-24T02:11:46.055441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241434763683117:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:46.055398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241433592387117:2048] [node 1] ICS09 handshake done sender: [4:7587241433592387114:2108] self: [4:7587241433592387115:0] peer: [1:7587241434161672578:0] socket: 419 qp: -1
2025-12-24T02:11:46.055405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241433592387117:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.055429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.055485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.056653Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241433851785977:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50804
2025-12-24T02:11:46.056889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434161672588:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.057572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683119:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.058464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241434763683119:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.058814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430468715755:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.058887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434161672590:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.058897Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434161672590:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.058476Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241433851785977:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.058988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241429556818621:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.059083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434161672585:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.059093Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434161672585:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.059421Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434161672590:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.059564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434161672585:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.059582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434161672588:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.059557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683121:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.059596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.059610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434161672588:2110] [node 5] ICP22 created new session: [1:7587241434161672597:2110]
2025-12-24T02:11:46.059616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434161672597:2110] [node 5] ICS09 handshake done sender: [1:7587241434161672590:2111] self: [1:7587241434161672589:0] peer: [5:7587241434763683120:0] socket: 426 qp: -1
2025-12-24T02:11:46.059627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434161672597:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:46.059634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.059676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429866705220:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.059691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.059698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429866705220:2048] [node 2] ICP22 created new session: [1:7587241434161672599:2048]
2025-12-24T02:11:46.059705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434161672599:2048] [node 2] ICS09 handshake done sender: [1:7587241434161672585:2109] self: [1:7587241434161672584:0] peer: [2:7587241433851785978:0] socket: 424 qp: -1
2025-12-24T02:11:46.059716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434161672599:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:46.059723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.059642Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241434763683119:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.059717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430468715755:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.059727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.059733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430468715755:2048] [node 1] ICP22 created new session: [5:7587241434763683122:2048]
2025-12-24T02:11:46.059744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241434763683122:2048] [node 1] ICS09 handshake done sender: [5:7587241434763683119:2111] self: [5:7587241434763683120:0] peer: [1:7587241434161672589:0] socket: 427 qp: -1
2025-12-24T02:11:46.059750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241434763683122:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.059758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.060027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241433851785979:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50804
2025-12-24T02:11:46.060113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241433851785977:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.060335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241429556818621:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.060359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.060367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429556818621:2048] [node 1] ICP22 created new session: [2:7587241433851785980:2048]
2025-12-24T02:11:46.060374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241433851785980:2048] [node 1] ICS09 handshake done sender: [2:7587241433851785977:2107] self: [2:7587241433851785978:0] peer: [1:7587241434161672584:0] socket: 425 qp: -1
2025-12-24T02:11:46.060389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241433851785980:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.060413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.061067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937089:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
2025-12-24T02:11:46.061219Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937089:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.061700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241428069969733:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.061799Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434161672595:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.061812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434161672595:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.062333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937091:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
2025-12-24T02:11:46.062091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434161672602:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:46.062479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434161672595:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.062524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241433851785983:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:46.062610Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432364937089:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.062717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241428069969733:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:46.062739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
::ffff:127.1.0.0
2025-12-24T02:11:46.062745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241428069969733:2048] [node 1] ICP22 created new session: [3:7587241432364937092:2048]
2025-12-24T02:11:46.062748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432364937092:2048] [node 1] ICS09 handshake done sender: [3:7587241432364937089:2107] self: [3:7587241432364937090:0] peer: [1:7587241434161672592:0] socket: 433 qp: -1
2025-12-24T02:11:46.062754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432364937092:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.062771Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.062799Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680250:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.063207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387119:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-12-24T02:11:46.063426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241429866705221:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.063447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.063456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241429866705221:2049] [node 3] ICP22 created new session: [1:7587241434161672607:2049]
2025-12-24T02:11:46.063464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434161672607:2049] [node 3] ICS09 handshake done sender: [1:7587241434161672595:2112] self: [1:7587241434161672592:0] peer: [3:7587241432364937090:0] socket: 431 qp: -1
2025-12-24T02:11:46.063471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434161672607:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:46.063479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.065281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241433760680250:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.065757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241433760680181:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.068724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387119:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.069188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241429297419758:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.069460Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241433851785985:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.069475Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241433851785985:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.070195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241433851785985:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.070362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387121:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.070620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241433851785983:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.070639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.071127Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937094:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
2025-12-24T02:11:46.070647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241433851785983:2109] [node 4] ICP22 created new session: [2:7587241433851785991:2109]
2025-12-24T02:11:46.070653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241433851785991:2109] [node 4] ICS09 handshake done sender: [2:7587241433851785985:2110] self: [2:7587241433851785984:0] peer: [4:7587241433592387120:0] socket: 438 qp: -1
2025-12-24T02:11:46.070661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241433851785991:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:46.070685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.071326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241433592387119:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.071403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241429297419758:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.071415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.071422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241429297419758:2049] [node 2] ICP22 created new session: [4:7587241433592387122:2049]
2025-12-24T02:11:46.071429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241433592387122:2049] [node 2] ICS09 handshake done sender: [4:7587241433592387119:2110] self: [4:7587241433592387120:0] peer: [2:7587241433851785984:0] socket: 439 qp: -1
2025-12-24T02:11:46.071434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241433592387122:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.071442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.071749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937094:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.072179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241428069969734:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.074734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434161672605:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.074758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434161672605:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.076828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683124:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.077156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680252:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.075501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241433851785988:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.075516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241433851785988:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.076066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241433851785994:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:46.077390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241433851785988:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.077495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241429556818622:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.077509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.077515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241429556818622:2049] [node 3] ICP22 created new session: [2:7587241433851785999:2049]
2025-12-24T02:11:46.077522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241433851785999:2049] [node 3] ICS09 handshake done sender: [2:7587241433851785988:2111] self: [2:7587241433851785987:0] peer: [3:7587241432364937095:0] socket: 441 qp: -1
2025-12-24T02:11:46.077530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241433851785999:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:46.077539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.079134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434161672605:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.079058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937096:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
2025-12-24T02:11:46.079543Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432364937094:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.079471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434161672602:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.079487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.079496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434161672602:2113] [node 6] ICP22 created new session: [1:7587241434161672610:2113]
2025-12-24T02:11:46.079511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434161672610:2113] [node 6] ICS09 handshake done sender: [1:7587241434161672605:2114] self: [1:7587241434161672604:0] peer: [6:7587241433760680251:0] socket: 436 qp: -1
2025-12-24T02:11:46.079519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434161672610:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:46.079545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.079677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241428069969734:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.079693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.079701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241428069969734:2049] [node 2] ICP22 created new session: [3:7587241432364937097:2049]
2025-12-24T02:11:46.079707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432364937097:2049] [node 2] ICS09 handshake done sender: [3:7587241432364937094:2109] self: [3:7587241432364937095:0] peer: [2:7587241433851785987:0] socket: 444 qp: -1
2025-12-24T02:11:46.079715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432364937097:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.079724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.079917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241434763683124:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.080456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430468715756:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.080582Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241433851785997:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.080590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241433851785997:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.081287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241433851785997:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.081445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241433760680250:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.081578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241433851785994:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.081592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.081598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241433851785994:2112] [node 5] ICP22 created new session: [2:7587241433851786002:2112]
2025-12-24T02:11:46.081604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241433851786002:2112] [node 5] ICS09 handshake done sender: [2:7587241433851785997:2113] self: [2:7587241433851785996:0] peer: [5:7587241434763683125:0] socket: 448 qp: -1
2025-12-24T02:11:46.081610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241433851786002:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:46.081616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.081881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241433760680181:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.081896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.081901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241433760680181:2048] [node 1] ICP22 created new session: [6:7587241433760680253:2048]
2025-12-24T02:11:46.081920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241433760680253:2048] [node 1] ICS09 handshake done sender: [6:7587241433760680250:2110] self: [6:7587241433760680251:0] peer: [1:7587241434161672604:0] socket: 437 qp: -1
2025-12-24T02:11:46.081926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241433760680253:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.081942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.082558Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683126:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.087068Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241434763683124:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.087415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430468715756:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.087437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.087447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430468715756:2049] [node 2] ICP22 created new session: [5:7587241434763683127:2049]
2025-12-24T02:11:46.087454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241434763683127:2049] [node 2] ICS09 handshake done sender: [5:7587241434763683124:2113] self: [5:7587241434763683125:0] peer: [2:7587241433851785996:0] socket: 449 qp: -1
2025-12-24T02:11:46.087461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241434763683127:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.087471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.091012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241433851786005:2114] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.091849Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680255:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.092605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432364937100:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:46.093848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241433760680255:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.094359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241433760680182:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.099365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387124:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.100257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680257:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.101071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241433760680255:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.101200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241433760680182:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.101213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.101220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241433760680182:2049] [node 2] ICP22 created new session: [6:7587241433760680258:2049]
2025-12-24T02:11:46.101226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241433760680258:2049] [node 2] ICS09 handshake done sender: [6:7587241433760680255:2112] self: [6:7587241433760680256:0] peer: [2:7587241433851786006:0] socket: 454 qp: -1
2025-12-24T02:11:46.101231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241433760680258:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.101240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.099509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241433851786007:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.099525Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241433851786007:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.100447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241433851786007:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.100582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241433851786005:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.100599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.100606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241433851786005:2114] [node 6] ICP22 created new session: [2:7587241433851786010:2114]
2025-12-24T02:11:46.100613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241433851786010:2114] [node 6] ICS09 handshake done sender: [2:7587241433851786007:2115] self: [2:7587241433851786006:0] peer: [6:7587241433760680256:0] socket: 453 qp: -1
2025-12-24T02:11:46.100620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241433851786010:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:46.100628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.103161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387124:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.103642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241429297419759:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.103851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937102:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.103865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432364937102:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.104510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432364937102:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.104612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432364937100:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.104626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.104633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432364937100:2111] [node 4] ICP22 created new session: [3:7587241432364937105:2111]
2025-12-24T02:11:46.104639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432364937105:2111] [node 4] ICS09 handshake done sender: [3:7587241432364937102:2112] self: [3:7587241432364937101:0] peer: [4:7587241433592387125:0] socket: 455 qp: -1
2025-12-24T02:11:46.104644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432364937105:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:46.104650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.105118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387126:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-12-24T02:11:46.105354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241433592387124:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.105524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241429297419759:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.105536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.105541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241429297419759:2050] [node 3] ICP22 created new session: [4:7587241433592387127:2050]
2025-12-24T02:11:46.105546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241433592387127:2050] [node 3] ICS09 handshake done sender: [4:7587241433592387124:2112] self: [4:7587241433592387125:0] peer: [3:7587241432364937101:0] socket: 456 qp: -1
2025-12-24T02:11:46.105551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241433592387127:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.105558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.110055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241433592387130:2114] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.114825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241433592387135:2116] [node 6] ICP01 ready to work
2025-12-24T02:11:46.114902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937107:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
2025-12-24T02:11:46.115224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241433760680261:2114] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.116008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432364937109:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:46.116656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432364937110:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35312
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.117409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680268:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.118207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241433760680268:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.118257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241433760680183:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.118482Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937110:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.118514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937112:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.118517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432364937112:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.118946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241434763683133:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:46.126989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432364937116:2117] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.132468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937107:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.131840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387137:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.135052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.135123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241433592387140:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49938
2025-12-24T02:11:46.135378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387137:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.139169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241433760680270:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.139224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241433760680261:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.140374Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432364937112:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.139526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387138:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.139538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241433592387138:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.140488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387140:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.140528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241433592387138:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.140625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241433592387135:2116] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.140643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.140650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241433592387135:2116] [node 6] ICP22 created new session: [4:7587241433592387144:2116]
2025-12-24T02:11:46.140657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241433592387144:2116] [node 6] ICS09 handshake done sender: [4:7587241433592387138:2118] self: [4:7587241433592387136:0] peer: [6:7587241433760680271:0] socket: 474 qp: -1
2025-12-24T02:11:46.140663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241433592387144:2116] [node 6] ICS10 traffic start
2025-12-24T02:11:46.140671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.141995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432364937109:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.142011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.142018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432364937109:2114] [node 6] ICP22 created new session: [3:7587241432364937122:2114]
2025-12-24T02:11:46.142024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432364937122:2114] [node 6] ICS09 handshake done sender: [3:7587241432364937112:2116] self: [3:7587241432364937111:0] peer: [6:7587241433760680269:0] socket: 471 qp: -1
2025-12-24T02:11:46.142029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432364937122:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:46.142035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.150976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683137:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.151759Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241433760680183:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241433760680268:2117] held: no
2025-12-24T02:11:46.152556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241433760680261:2114] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4803) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241433760680270:2118] held: no
2025-12-24T02:11:46.154422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241434763683137:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.154494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241434763683133:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.162671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.162855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680272:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.165547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241434763683139:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.165608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241430468715757:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.165228Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241433760680268:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.165302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241433760680183:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.165334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.165346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241433760680183:2050] [node 3] ICP22 created new session: [6:7587241433760680274:2050]
2025-12-24T02:11:46.165353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241433760680274:2050] [node 3] ICS09 handshake done sender: [6:7587241433760680268:2117] self: [6:7587241433760680269:0] peer: [3:7587241432364937111:0] socket: 472 qp: -1
2025-12-24T02:11:46.165360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241433760680274:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.165370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.166766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241433760680273:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10041
2025-12-24T02:11:46.170072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241433592387132:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.170099Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241433592387132:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.170900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241433760680270:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.170984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241433760680261:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.170998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.171008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241433760680261:2114] [node 4] ICP22 created new session: [6:7587241433760680276:2114]
2025-12-24T02:11:46.171016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241433760680276:2114] [node 4] ICS09 handshake done sender: [6:7587241433760680270:2118] self: [6:7587241433760680271:0] peer: [4:7587241433592387136:0] socket: 476 qp: -1
2025-12-24T02:11:46.171023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241433760680276:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:46.171035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.170989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241433592387132:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.171071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241433592387130:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.171089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.171116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241433592387130:2114] [node 5] ICP22 created new session: [4:7587241433592387147:2114]
2025-12-24T02:11:46.171123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241433592387147:2114] [node 5] ICS09 handshake done sender: [4:7587241433592387132:2115] self: [4:7587241433592387131:0] peer: [5:7587241434763683138:0] socket: 463 qp: -1
2025-12-24T02:11:46.171130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241433592387147:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:46.171136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.173226Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683141:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.178952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241434763683137:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.179183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241434763683133:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.179208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.179221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241434763683133:2116] [node 4] ICP22 created new session: [5:7587241434763683142:2116]
2025-12-24T02:11:46.179231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241434763683142:2116] [node 4] ICS09 handshake done sender: [5:7587241434763683137:2118] self: [5:7587241434763683138:0] peer: [4:7587241433592387131:0] socket: 465 qp: -1
2025-12-24T02:11:46.179241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241434763683142:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:46.179251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.178728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432364937118:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.178756Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432364937118:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.199128Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432364937118:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.199292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432364937116:2117] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.199324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.199334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432364937116:2117] [node 5] ICP22 created new session: [3:7587241432364937125:2117]
2025-12-24T02:11:46.199342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432364937125:2117] [node 5] ICS09 handshake done sender: [3:7587241432364937118:2118] self: [3:7587241432364937117:0] peer: [5:7587241434763683140:0] socket: 478 qp: -1
2025-12-24T02:11:46.199351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432364937125:2117] [node 5] ICS10 traffic start
2025-12-24T02:11:46.199364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.207678Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430468715757:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61577) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241434763683139:2119] held: no
2025-12-24T02:11:46.207832Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241434763683144:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40250
2025-12-24T02:11:46.210766Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241434763683139:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.211510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241430468715757:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.211546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.211555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241430468715757:2050] [node 3] ICP22 created new session: [5:7587241434763683145:2050]
2025-12-24T02:11:46.211564Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241434763683145:2050] [node 3] ICS09 handshake done sender: [5:7587241434763683139:2119] self: [5:7587241434763683140:0] peer: [3:7587241432364937117:0] socket: 481 qp: -1
2025-12-24T02:11:46.211574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241434763683145:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.211587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.227618Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:46.227722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241433851785980:2048] [node 1] ICS07 socket disconnect 425 reason# EndOfStream
2025-12-24T02:11:46.227729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241433851785980:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.227765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.227781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241433851785980:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.228425Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:46.228502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241432364937092:2048] [node 1] ICS07 socket disconnect 433 reason# EndOfStream
2025-12-24T02:11:46.228507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241432364937092:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.228532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.228548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241432364937092:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.230227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241428069969733:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:46.230237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241432364937092:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.230708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241429556818621:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:46.230724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241433851785980:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.243506Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241428069969733:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.243528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241432364937092:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.243535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241428069969733:2048] [node 1] ICP30 unregister session Session# [3:7587241432364937092:2048] VirtualId# [3:7587241432364937090:0]
2025-12-24T02:11:46.243571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.243580Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241428069969733:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.243583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.247111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:294: Proxy [2:7587241429556818621:2048] [node 1] ICP35 event processing took too much time StateWork:294(16395440ns) {HandleHandshakeStatus:395(16393952ns) {LogHandshakeFail:472(3010ns) {UpdateErrorStateLog:545(1207ns)} DropOutgoingHandshake:457(241ns) ProcessPendingSessionEvents:495(114ns) StartResumeHandshake:172(16349379ns) {DropOutgoingHandshake:457(49ns)}}}
2025-12-24T02:11:46.247481Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241429556818621:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.247493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241433851785980:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.247499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241429556818621:2048] [node 1] ICP30 unregister session Session# [2:7587241433851785980:2048] VirtualId# [2:7587241433851785978:0]
2025-12-24T02:11:46.247523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.247529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241429556818621:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.247531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.250655Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:46.250758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241433760680253:2048] [node 1] ICS07 socket disconnect 437 reason# EndOfStream
2025-12-24T02:11:46.250767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241433760680253:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.250829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.250850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241433760680253:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.250716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:46.250805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241433592387117:2048] [node 1] ICS07 socket disconnect 419 reason# EndOfStream
2025-12-24T02:11:46.250811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241433592387117:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.250857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.250878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241433592387117:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.251133Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:46.251211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241434763683122:2048] [node 1] ICS01 socket: 427 reason# EndOfStream
2025-12-24T02:11:46.251218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430468715755:2048] [node 1] ICP30 unregister session Session# [5:7587241434763683122:2048] VirtualId# [5:7587241434763683120:0]
2025-12-24T02:11:46.251223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241434763683122:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.251249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.251272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.251560Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241433760680181:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.251573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241433760680253:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.251577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241433760680181:2048] [node 1] ICP30 unregister session Session# [6:7587241433760680253:2048] VirtualId# [6:7587241433760680251:0]
2025-12-24T02:11:46.251601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.251607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241433760680181:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.251610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.255033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241428069969733:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:46.254869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241429297419757:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.254887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241433592387117:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.254892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241429297419757:2048] [node 1] ICP30 unregister session Session# [4:7587241433592387117:2048] VirtualId# [4:7587241433592387115:0]
2025-12-24T02:11:46.254949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.254955Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241429297419757:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.254957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14969) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.262619Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241433760680181:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:46.263222Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:46.263317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241432364937097:2049] [node 2] ICS01 socket: 444 reason# EndOfStream
2025-12-24T02:11:46.263326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241428069969734:2049] [node 2] ICP30 unregister session Session# [3:7587241432364937097:2049] VirtualId# [3:7587241432364937095:0]
2025-12-24T02:11:46.263333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241432364937097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.263363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.263383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.267784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241429297419757:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:46.274812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:46.274904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241433760680258:2049] [node 2] ICS07 socket disconnect 454 reason# EndOfStream
2025-12-24T02:11:46.274910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241433760680258:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.274962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.274985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241433760680258:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:46.274759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:46.274869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241434763683127:2049] [node 2] ICS07 socket disconnect 449 reason# EndOfStream
2025-12-24T02:11:46.274878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241434763683127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.274934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.274968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241434763683127:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:46.275526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:46.275596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241433592387122:2049] [node 2] ICS07 socket disconnect 439 reason# EndOfStream
2025-12-24T02:11:46.275601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241433592387122:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.275632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.275657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241433592387122:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:46.276241Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241429297419758:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.276256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241433592387122:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.276261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241429297419758:2049] [node 2] ICP30 unregister session Session# [4:7587241433592387122:2049] VirtualId# [4:7587241433592387120:0]
2025-12-24T02:11:46.276285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.276293Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241429297419758:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.276296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.276513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241430468715756:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.276524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241434763683127:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.276528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430468715756:2049] [node 2] ICP30 unregister session Session# [5:7587241434763683127:2049] VirtualId# [5:7587241434763683125:0]
2025-12-24T02:11:46.276546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.276549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241430468715756:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.276551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.283208Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241433760680182:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.283233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241433760680258:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.283243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241433760680182:2049] [node 2] ICP30 unregister session Session# [6:7587241433760680258:2049] VirtualId# [6:7587241433760680256:0]
2025-12-24T02:11:46.283277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.283285Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241433760680182:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.283288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29894) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.283448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241429297419758:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:46.285279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241430468715756:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:46.286144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:46.286248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241433592387127:2050] [node 3] ICS01 socket: 456 reason# EndOfStream
2025-12-24T02:11:46.286257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241429297419759:2050] [node 3] ICP30 unregister session Session# [4:7587241433592387127:2050] VirtualId# [4:7587241433592387125:0]
2025-12-24T02:11:46.286265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241433592387127:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.286332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:46.286359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:46.286502Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:46.286572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241433760680274:2050] [node 3] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:46.286578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241433760680183:2050] [node 3] ICP30 unregister session Session# [6:7587241433760680274:2050] VirtualId# [6:7587241433760680269:0]
2025-12-24T02:11:46.286583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241433760680274:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.286631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:46.286656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:46.288206Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:46.288289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241434763683145:2050] [node 3] ICS01 socket: 481 reason# EndOfStream
2025-12-24T02:11:46.288295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241430468715757:2050] [node 3] ICP30 unregister session Session# [5:7587241434763683145:2050] VirtualId# [5:7587241434763683140:0]
2025-12-24T02:11:46.288303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241434763683145:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.288352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:46.288377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:46.293255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241433760680182:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:46.338772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:46.338890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241433760680276:2114] [node 4] ICS07 socket disconnect 476 reason# EndOfStream
2025-12-24T02:11:46.338901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241433760680276:2114] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.338973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:46.338997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241433760680276:2114] [node 4] ICS15 start handshake
2025-12-24T02:11:46.346266Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241433760680261:2114] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4803) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.346287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241433760680276:2114] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.346293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241433760680261:2114] [node 4] ICP30 unregister session Session# [6:7587241433760680276:2114] VirtualId# [6:7587241433760680271:0]
2025-12-24T02:11:46.346315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:46.346322Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241433760680261:2114] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:4803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.346325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4803) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.354259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241433760680261:2114] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:46.439043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434851228341:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:46.439048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434851228340:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:46.442847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241432700092483:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.442864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241432700092484:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:46.478383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432695287798:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.478394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432695287799:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.483960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241432189420040:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.483985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241432189420041:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.483990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241432189420042:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.532352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241432104959641:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.532457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241432104959642:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.532463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241432104959643:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.547628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241432027265656:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.547919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241432027265654:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.551710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241432027265655:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.558076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241432700092545:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:46.558206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241432700092546:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.557808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241432104959703:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:46.559123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434851228404:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:46.559228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434851228406:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:46.559248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434851228407:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.560197Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959707:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.560162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265718:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.560730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265719:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.561597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241432700092548:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.563435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432695287861:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:46.563232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420101:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.567299Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265720:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.569308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241432027265720:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.569372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241432027265655:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.566889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241432700092556:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:46.569568Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092548:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.569603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092555:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.569606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241432700092555:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.570280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241432700092555:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.571357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432695287863:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.572524Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241432700092565:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.572856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420101:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.573591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241432700092546:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.573632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.573649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241432700092546:2108] [node 5] ICP22 created new session: [2:7587241432700092566:2108]
2025-12-24T02:11:46.573662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241432700092566:2108] [node 5] ICS09 handshake done sender: [2:7587241432700092555:2112] self: [2:7587241432700092551:0] peer: [5:7587241432027265721:0] socket: 434 qp: -1
2025-12-24T02:11:46.573677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241432700092566:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:46.573690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.572562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959708:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.572879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241432104959708:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.573398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241432104959642:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.575009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241432027265718:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.575238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241432027265724:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:46.575683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241432027265724:2113] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.576095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241434851228418:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.576083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241432104959705:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.576096Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241432104959705:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.576803Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241432104959705:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.576921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241432104959703:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.576937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.576946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241432104959703:2108] [node 5] ICP22 created new session: [6:7587241432104959711:2108]
2025-12-24T02:11:46.576953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241432104959711:2108] [node 5] ICS09 handshake done sender: [6:7587241432104959705:2109] self: [6:7587241432104959704:0] peer: [5:7587241432027265725:0] socket: 421 qp: -1
2025-12-24T02:11:46.576960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241432104959711:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:46.576977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.577355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265726:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.577618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241432027265718:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.577740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241432027265724:2113] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.577756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.577764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241432027265724:2113] [node 6] ICP22 created new session: [5:7587241432027265727:2113]
2025-12-24T02:11:46.577770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241432027265727:2113] [node 6] ICS09 handshake done sender: [5:7587241432027265718:2109] self: [5:7587241432027265725:0] peer: [6:7587241432104959704:0] socket: 423 qp: -1
2025-12-24T02:11:46.577777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241432027265727:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:46.577798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.578829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092549:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.578847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241432700092549:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.579046Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420103:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.579139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092559:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.579150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241432700092559:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.579618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420103:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.578682Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241434851228341:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19919435ns) {Configure:104(19917675ns) {ProcessConfigured:135(19666055ns) {StartInitialHandshake:157(19665315ns) {DropHandshakes:470(279ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1764ns) {SwitchToState:179(505ns)}}}}}
2025-12-24T02:11:46.578772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228418:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.579597Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241434851228419:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33851
2025-12-24T02:11:46.578954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265722:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.579206Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241432027265720:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.581933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228419:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.586626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241432700092484:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(24947408ns) {Configure:104(24946079ns) {ProcessConfigured:135(24621642ns) {StartInitialHandshake:157(24620736ns) {DropHandshakes:470(501ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(333ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(1193ns)}}}}}
2025-12-24T02:11:46.588487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241432189420041:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.589049Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092565:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.587645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241432027265719:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.588199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241432027265654:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.591141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241432104959707:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.591656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241432104959641:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.592230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241432695287874:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:46.591779Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241432700092546:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(21160709ns) {HandleHandshakeStatus:329(21160045ns) {DropIncomingHandshake:442(6967ns) DropOutgoingHandshake:457(249ns) SwitchToState:179(277ns) ProcessPendingSessionEvents:495(12739131ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 12722016ns max# 12653923ns}}}
2025-12-24T02:11:46.593392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241432700092549:2110] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.594047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241432027265655:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.594081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.594091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241432027265655:2049] [node 2] ICP22 created new session: [5:7587241432027265730:2049]
2025-12-24T02:11:46.594100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241432027265730:2049] [node 2] ICS09 handshake done sender: [5:7587241432027265720:2111] self: [5:7587241432027265721:0] peer: [2:7587241432700092551:0] socket: 435 qp: -1
2025-12-24T02:11:46.594109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241432027265730:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.594134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.594235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241432189420040:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.594439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241432700092559:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.594662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241432700092556:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.594673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.594678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241432700092556:2111] [node 6] ICP22 created new session: [2:7587241432700092572:2111]
2025-12-24T02:11:46.594685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241432700092572:2111] [node 6] ICS09 handshake done sender: [2:7587241432700092559:2113] self: [2:7587241432700092558:0] peer: [6:7587241432104959709:0] socket: 438 qp: -1
2025-12-24T02:11:46.594708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241432700092572:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:46.594716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.596065Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228413:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.596088Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434851228413:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.597749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420105:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.597225Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228415:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.597251Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434851228415:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.598053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420105:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.598472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241432189420042:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.593798Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959714:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.594579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241432104959708:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.598831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228410:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.598845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434851228410:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.598751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241432700092545:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.598772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.598780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241432700092545:2107] [node 4] ICP22 created new session: [2:7587241432700092575:2107]
2025-12-24T02:11:46.598788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241432700092575:2107] [node 4] ICS09 handshake done sender: [2:7587241432700092549:2110] self: [2:7587241432700092547:0] peer: [4:7587241432189420102:0] socket: 426 qp: -1
2025-12-24T02:11:46.598797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241432700092575:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:46.598806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.612068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420107:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.612314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265732:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.612590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241432189420101:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.612698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241432189420041:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.612717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.612724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241432189420041:2049] [node 2] ICP22 created new session: [4:7587241432189420108:2049]
2025-12-24T02:11:46.612732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241432189420108:2049] [node 2] ICS09 handshake done sender: [4:7587241432189420101:2108] self: [4:7587241432189420102:0] peer: [2:7587241432700092547:0] socket: 433 qp: -1
2025-12-24T02:11:46.612740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241432189420108:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.612764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.613147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241432027265719:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.613250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241432027265654:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.613264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.613270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241432027265654:2048] [node 1] ICP22 created new session: [5:7587241432027265733:2048]
2025-12-24T02:11:46.613276Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241432027265733:2048] [node 1] ICS09 handshake done sender: [5:7587241432027265719:2110] self: [5:7587241432027265729:0] peer: [1:7587241434851228412:0] socket: 431 qp: -1
2025-12-24T02:11:46.613281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241432027265733:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.613298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.613802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287869:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.613826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432695287869:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.613043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434851228413:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.614225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434851228415:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.614744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434851228407:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.614761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.614770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434851228407:2108] [node 5] ICP22 created new session: [1:7587241434851228425:2108]
2025-12-24T02:11:46.614777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434851228425:2108] [node 5] ICS09 handshake done sender: [1:7587241434851228413:2111] self: [1:7587241434851228412:0] peer: [5:7587241432027265729:0] socket: 429 qp: -1
2025-12-24T02:11:46.614786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434851228425:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:46.614795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.614840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434851228404:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.614847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.614852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434851228404:2107] [node 4] ICP22 created new session: [1:7587241434851228427:2107]
2025-12-24T02:11:46.614858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434851228427:2107] [node 4] ICS09 handshake done sender: [1:7587241434851228415:2112] self: [1:7587241434851228408:0] peer: [4:7587241432189420104:0] socket: 436 qp: -1
2025-12-24T02:11:46.614862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434851228427:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:46.614868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.617297Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241432027265655:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(37981435ns) {HandleHandshakeStatus:329(37981100ns) {DropIncomingHandshake:442(926ns) DropOutgoingHandshake:457(3879ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(19527515ns) {ForwardSessionEventToSession:612(19513398ns) ForwardSessionEventToSession:612(1232ns) ForwardSessionEventToSession:612(1856ns) ForwardSessionEventToSession:612(615ns) ForwardSessionEventToSession:612(618ns) ForwardSessionEventToSession:612(1311ns) ForwardSessionEventToSession:612(842ns) ForwardSessionEventToSession:612(612ns) ForwardSessionEventToSession:612(834ns) ForwardSessionEventToSession:612(1249ns)}}}
2025-12-24T02:11:46.617319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241432027265655:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:46.618242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092568:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.618255Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241432700092568:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.616800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287877:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
2025-12-24T02:11:46.618012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287877:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.618055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241432695287799:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.619210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241432189420111:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:46.620998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241432700092568:2116] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.622889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241432189420114:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:46.625809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434851228410:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.625128Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959715:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.625153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959716:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.625988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241432104959707:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.626830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241432104959642:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.626852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.626862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241432104959642:2049] [node 2] ICP22 created new session: [6:7587241432104959717:2049]
2025-12-24T02:11:46.626870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241432104959717:2049] [node 2] ICS09 handshake done sender: [6:7587241432104959708:2111] self: [6:7587241432104959709:0] peer: [2:7587241432700092558:0] socket: 440 qp: -1
2025-12-24T02:11:46.626879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241432104959717:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.626907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.633318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434851228406:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.633346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.633361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434851228406:2109] [node 6] ICP22 created new session: [1:7587241434851228432:2109]
2025-12-24T02:11:46.633373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434851228432:2109] [node 6] ICS09 handshake done sender: [1:7587241434851228410:2110] self: [1:7587241434851228409:0] peer: [6:7587241432104959713:0] socket: 428 qp: -1
2025-12-24T02:11:46.633381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434851228432:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:46.633394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.636357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420118:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.639020Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241434851228404:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(24074047ns) {HandleHandshakeStatus:329(24073586ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(101ns) SwitchToState:179(43ns) ProcessPendingSessionEvents:495(23998167ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23978687ns max# 23894232ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.640504Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287879:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.640575Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287880:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.640630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287881:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.642041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265738:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.639361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241432189420103:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.640188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241432189420111:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20885328ns) {Configure:104(20884604ns) {ProcessConfigured:135(20546510ns) {StartInitialHandshake:157(20545548ns) {DropHandshakes:470(691ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(200ns)} GenerateSessionVirtualId:988(267ns) PrepareNewSessionHandshake:477(845ns) {SwitchToState:179(634ns)}}}}}
2025-12-24T02:11:46.642705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241432189420040:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.642721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.642730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241432189420040:2048] [node 1] ICP22 created new session: [4:7587241432189420121:2048]
2025-12-24T02:11:46.642737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241432189420121:2048] [node 1] ICS09 handshake done sender: [4:7587241432189420103:2109] self: [4:7587241432189420104:0] peer: [1:7587241434851228408:0] socket: 439 qp: -1
2025-12-24T02:11:46.642744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241432189420121:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.642752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.639626Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241432700092545:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(40779013ns) {HandleHandshakeStatus:329(40778280ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(173ns) SwitchToState:179(234ns) ProcessPendingSessionEvents:495(24421312ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24401773ns max# 24313112ns}}}
2025-12-24T02:11:46.642481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287880:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.642545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241432695287798:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.642905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287881:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.642956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432695287877:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.643046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241432695287799:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.643069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.643075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432695287799:2049] [node 2] ICP22 created new session: [3:7587241432695287884:2049]
2025-12-24T02:11:46.643081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432695287884:2049] [node 2] ICS09 handshake done sender: [3:7587241432695287877:2115] self: [3:7587241432695287878:0] peer: [2:7587241432700092553:0] socket: 455 qp: -1
2025-12-24T02:11:46.643088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432695287884:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:46.643103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.646924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241432027265740:2118] [node 4] ICP01 ready to work
2025-12-24T02:11:46.647307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241432700092484:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.647354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.647366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241432700092484:2049] [node 3] ICP22 created new session: [2:7587241432700092578:2049]
2025-12-24T02:11:46.647374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241432700092578:2049] [node 3] ICS09 handshake done sender: [2:7587241432700092568:2116] self: [2:7587241432700092553:0] peer: [3:7587241432695287878:0] socket: 454 qp: -1
2025-12-24T02:11:46.647383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241432700092578:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:46.647410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.650053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241432104959642:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(51261630ns) {HandleHandshakeStatus:329(51260807ns) {DropIncomingHandshake:442(842ns) DropOutgoingHandshake:457(212ns) SwitchToState:179(257ns) ProcessPendingSessionEvents:495(592ns)}}
2025-12-24T02:11:46.650494Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228420:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.650512Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434851228420:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.651454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434851228420:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.651548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434851228341:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.651571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.651578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434851228341:2049] [node 3] ICP22 created new session: [1:7587241434851228435:2049]
2025-12-24T02:11:46.651586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434851228435:2049] [node 3] ICS09 handshake done sender: [1:7587241434851228420:2116] self: [1:7587241434851228402:0] peer: [3:7587241432695287882:0] socket: 465 qp: -1
2025-12-24T02:11:46.651592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434851228435:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:46.651600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.653756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287886:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
2025-12-24T02:11:46.654883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432695287880:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.655059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420123:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.654957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241432695287798:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.654972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.654978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432695287798:2048] [node 1] ICP22 created new session: [3:7587241432695287888:2048]
2025-12-24T02:11:46.654983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432695287888:2048] [node 1] ICS09 handshake done sender: [3:7587241432695287880:2117] self: [3:7587241432695287882:0] peer: [1:7587241434851228402:0] socket: 472 qp: -1
2025-12-24T02:11:46.654989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432695287888:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.654996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.656513Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432695287869:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.656961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241434851228417:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.656971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241434851228417:2113] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.657816Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241434851228417:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:46.655866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241432104959723:2116] [node 4] ICP01 ready to work
2025-12-24T02:11:46.658050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434851228340:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.658067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:46.658072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434851228340:2048] [node 2] ICP22 created new session: [1:7587241434851228438:2048]
2025-12-24T02:11:46.658078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241434851228438:2048] [node 2] ICS09 handshake done sender: [1:7587241434851228417:2113] self: [1:7587241434851228400:0] peer: [2:7587241432700092581:0] socket: 464 qp: -1
2025-12-24T02:11:46.658083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241434851228438:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:46.658090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:46.656334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241432700092580:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36139
2025-12-24T02:11:46.656843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241432700092580:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.656870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241432700092483:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.657744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241432700092582:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36139
2025-12-24T02:11:46.657919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241432700092580:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:46.658140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241432700092483:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.658150Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.658157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241432700092483:2048] [node 1] ICP22 created new session: [2:7587241432700092583:2048]
2025-12-24T02:11:46.658162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241432700092583:2048] [node 1] ICS09 handshake done sender: [2:7587241432700092580:2117] self: [2:7587241432700092581:0] peer: [1:7587241434851228400:0] socket: 469 qp: -1
2025-12-24T02:11:46.658169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241432700092583:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.658175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.658810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241432189420105:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.660431Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241432189420040:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(17627541ns) {HandleHandshakeStatus:329(17626825ns) {DropIncomingHandshake:442(360ns) DropOutgoingHandshake:457(67ns) SwitchToState:179(59ns) ProcessPendingSessionEvents:495(17524425ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17503704ns max# 17421681ns}}}
2025-12-24T02:11:46.660678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241432189420042:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.660687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.660693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241432189420042:2050] [node 3] ICP22 created new session: [4:7587241432189420124:2050]
2025-12-24T02:11:46.660698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241432189420124:2050] [node 3] ICS09 handshake done sender: [4:7587241432189420105:2110] self: [4:7587241432189420106:0] peer: [3:7587241432695287864:0] socket: 452 qp: -1
2025-12-24T02:11:46.660704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241432189420124:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.660711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.659469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241434851228407:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(44582261ns) {HandleHandshakeStatus:329(44581102ns) {DropIncomingHandshake:442(89ns) DropOutgoingHandshake:457(560ns) SwitchToState:179(476ns) ProcessPendingSessionEvents:495(32505797ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 32483673ns max# 32406511ns}}}
2025-12-24T02:11:46.656615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265742:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.661056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241432695287798:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:46.661143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432695287861:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.661151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.661157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432695287861:2107] [node 4] ICP22 created new session: [3:7587241432695287892:2107]
2025-12-24T02:11:46.661162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432695287892:2107] [node 4] ICS09 handshake done sender: [3:7587241432695287869:2111] self: [3:7587241432695287864:0] peer: [4:7587241432189420106:0] socket: 448 qp: -1
2025-12-24T02:11:46.661167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432695287892:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:46.661173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.667074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241432104959641:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.667100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:46.667110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241432104959641:2048] [node 1] ICP22 created new session: [6:7587241432104959727:2048]
2025-12-24T02:11:46.667117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241432104959727:2048] [node 1] ICS09 handshake done sender: [6:7587241432104959707:2110] self: [6:7587241432104959713:0] peer: [1:7587241434851228409:0] socket: 430 qp: -1
2025-12-24T02:11:46.667123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241432104959727:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:46.667142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:46.667203Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241432104959641:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(40527816ns) {HandleHandshakeStatus:329(40527077ns) {DropIncomingHandshake:442(839ns) DropOutgoingHandshake:457(66ns) SwitchToState:179(192ns) ProcessPendingSessionEvents:495(96ns)}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.671072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241432695287799:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(27895927ns) {HandleHandshakeStatus:329(27895403ns) {DropIncomingHandshake:442(193ns) DropOutgoingHandshake:457(4351ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(23402941ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23378716ns max# 23291476ns}}}
2025-12-24T02:11:46.671194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241432695287890:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57134
2025-12-24T02:11:46.671719Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241432027265740:2118] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24459997ns) {Configure:104(24458694ns) {ProcessConfigured:135(24053769ns) {StartInitialHandshake:157(24052921ns) {DropHandshakes:470(400ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(508ns) PrepareNewSessionHandshake:477(1033ns) {SwitchToState:179(443ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.675099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420126:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.675168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241432189420127:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4343
2025-12-24T02:11:46.675455Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420126:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.675376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287890:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.679378Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241432700092484:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(58162837ns) {HandleHandshakeStatus:329(58161538ns) {DropIncomingHandshake:442(14333ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(136ns) ProcessPendingSessionEvents:495(16848130ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16823388ns max# 16736096ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.686077Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959729:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.687702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287876:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.687716Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432695287876:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.679142Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420127:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.687925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241432027265738:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.687983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241432027265740:2118] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.686294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241432104959729:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.686349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241432104959643:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.688576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959731:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
2025-12-24T02:11:46.688678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241432104959716:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.688709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241432104959723:2116] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.688874Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432695287876:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.689020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432695287874:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.689049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.689057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432695287874:2113] [node 6] ICP22 created new session: [3:7587241432695287897:2113]
2025-12-24T02:11:46.689070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432695287897:2113] [node 6] ICS09 handshake done sender: [3:7587241432695287876:2114] self: [3:7587241432695287875:0] peer: [6:7587241432104959730:0] socket: 445 qp: -1
2025-12-24T02:11:46.689077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432695287897:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:46.689123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.689316Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241432104959729:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.689412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241432104959733:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.689399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241432027265742:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.689498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241432027265656:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:46.689564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265746:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.688215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420119:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.688222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241432189420119:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.688701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241432189420119:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.688818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241432189420116:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.688821Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241432189420116:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.689947Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241432189420116:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:46.690048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241432189420114:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.690067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:46.690073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241432189420114:2113] [node 6] ICP22 created new session: [4:7587241432189420132:2113]
2025-12-24T02:11:46.690079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241432189420132:2113] [node 6] ICS09 handshake done sender: [4:7587241432189420116:2114] self: [4:7587241432189420115:0] peer: [6:7587241432104959732:0] socket: 466 qp: -1
2025-12-24T02:11:46.690084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241432189420132:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:46.690091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:46.690656Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241432027265738:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.690100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241432695287872:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:46.690108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241432695287872:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:46.691072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241432104959716:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:46.691193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241432695287872:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:46.691255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241432104959643:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.691268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.691278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241432104959643:2050] [node 3] ICP22 created new session: [6:7587241432104959734:2050]
2025-12-24T02:11:46.691287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241432104959734:2050] [node 3] ICS09 handshake done sender: [6:7587241432104959729:2118] self: [6:7587241432104959730:0] peer: [3:7587241432695287875:0] socket: 449 qp: -1
2025-12-24T02:11:46.691292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241432104959734:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.691299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.690811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241432027265740:2118] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.690826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.690832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241432027265740:2118] [node 4] ICP22 created new session: [5:7587241432027265748:2118]
2025-12-24T02:11:46.690837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241432027265748:2118] [node 4] ICS09 handshake done sender: [5:7587241432027265738:2117] self: [5:7587241432027265745:0] peer: [4:7587241432189420112:0] socket: 474 qp: -1
2025-12-24T02:11:46.690843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241432027265748:2118] [node 4] ICS10 traffic start
2025-12-24T02:11:46.690853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.691523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241432695287863:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.691539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.691544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241432695287863:2108] [node 5] ICP22 created new session: [3:7587241432695287900:2108]
2025-12-24T02:11:46.691550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241432695287900:2108] [node 5] ICS09 handshake done sender: [3:7587241432695287872:2112] self: [3:7587241432695287871:0] peer: [5:7587241432027265747:0] socket: 481 qp: -1
2025-12-24T02:11:46.691555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241432695287900:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:46.691560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.691820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241432027265750:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22350
2025-12-24T02:11:46.691924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241432027265742:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:46.691971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241432027265656:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.691987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:46.691991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241432027265656:2050] [node 3] ICP22 created new session: [5:7587241432027265751:2050]
2025-12-24T02:11:46.691997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241432027265751:2050] [node 3] ICS09 handshake done sender: [5:7587241432027265742:2119] self: [5:7587241432027265747:0] peer: [3:7587241432695287871:0] socket: 482 qp: -1
2025-12-24T02:11:46.692002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241432027265751:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:46.692008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:46.692529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241432104959723:2116] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:46.692543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:46.692548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241432104959723:2116] [node 4] ICP22 created new session: [6:7587241432104959736:2116]
2025-12-24T02:11:46.692553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241432104959736:2116] [node 4] ICS09 handshake done sender: [6:7587241432104959716:2114] self: [6:7587241432104959732:0] peer: [4:7587241432189420115:0] socket: 468 qp: -1
2025-12-24T02:11:46.692557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241432104959736:2116] [node 4] ICS10 traffic start
2025-12-24T02:11:46.692563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:46.698572Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241432189420111:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:46.698605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:46.698615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241432189420111:2112] [node 5] ICP22 created new session: [4:7587241432189420134:2112]
2025-12-24T02:11:46.698621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241432189420134:2112] [node 5] ICS09 handshake done sender: [4:7587241432189420119:2116] self: [4:7587241432189420112:0] peer: [5:7587241432027265745:0] socket: 471 qp: -1
2025-12-24T02:11:46.698627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241432189420134:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:46.698655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:46.719785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:46.719915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241432700092583:2048] [node 1] ICS07 socket disconnect 469 reason# EndOfStream
2025-12-24T02:11:46.719923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241432700092583:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.719960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.719977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241432700092583:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.721975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:46.722056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241432695287888:2048] [node 1] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:46.722067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241432695287798:2048] [node 1] ICP30 unregister session Session# [3:7587241432695287888:2048] VirtualId# [3:7587241432695287882:0]
2025-12-24T02:11:46.722076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241432695287888:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.722102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.722124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.722908Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241432700092483:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:46.722922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241432700092583:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.723275Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241432700092483:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.723285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241432700092583:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.723289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241432700092483:2048] [node 1] ICP30 unregister session Session# [2:7587241432700092583:2048] VirtualId# [2:7587241432700092581:0]
2025-12-24T02:11:46.723342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.723348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241432700092483:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.723351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.730853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:46.730947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241432027265733:2048] [node 1] ICS07 socket disconnect 431 reason# EndOfStream
2025-12-24T02:11:46.730954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241432027265733:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.731014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.731040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241432027265733:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.731278Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:46.731367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241432104959727:2048] [node 1] ICS01 socket: 430 reason# EndOfStream
2025-12-24T02:11:46.731374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241432104959641:2048] [node 1] ICP30 unregister session Session# [6:7587241432104959727:2048] VirtualId# [6:7587241432104959713:0]
2025-12-24T02:11:46.731381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241432104959727:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.731419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.731446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.732054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241432027265654:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.732067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241432027265733:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.732073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241432027265654:2048] [node 1] ICP30 unregister session Session# [5:7587241432027265733:2048] VirtualId# [5:7587241432027265729:0]
2025-12-24T02:11:46.732125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.732132Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241432027265654:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.732135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.733902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:46.733987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241432189420121:2048] [node 1] ICS07 socket disconnect 439 reason# EndOfStream
2025-12-24T02:11:46.733993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241432189420121:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.734045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:46.734067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241432189420121:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:46.737477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241432189420040:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.737494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241432189420121:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.737500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241432189420040:2048] [node 1] ICP30 unregister session Session# [4:7587241432189420121:2048] VirtualId# [4:7587241432189420104:0]
2025-12-24T02:11:46.737549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:46.737555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241432189420040:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.737558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15236) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.742300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241432027265654:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:46.745828Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:46.745926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241432695287884:2049] [node 2] ICS01 socket: 455 reason# EndOfStream
2025-12-24T02:11:46.745936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241432695287799:2049] [node 2] ICP30 unregister session Session# [3:7587241432695287884:2049] VirtualId# [3:7587241432695287878:0]
2025-12-24T02:11:46.745942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241432695287884:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.745973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.745992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.749953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241432189420040:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:46.758268Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:46.758379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241432027265730:2049] [node 2] ICS07 socket disconnect 435 reason# EndOfStream
2025-12-24T02:11:46.758389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241432027265730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.758438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.758460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241432027265730:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:46.758877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:46.758955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241432189420108:2049] [node 2] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:46.758961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241432189420041:2049] [node 2] ICP30 unregister session Session# [4:7587241432189420108:2049] VirtualId# [4:7587241432189420102:0]
2025-12-24T02:11:46.758967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241432189420108:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.758998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.759031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.762076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:46.762157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241432104959717:2049] [node 2] ICS07 socket disconnect 440 reason# EndOfStream
2025-12-24T02:11:46.762164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241432104959717:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.762215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:46.762236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241432104959717:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:46.763010Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241432027265655:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.763030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241432027265730:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.763035Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241432027265655:2049] [node 2] ICP30 unregister session Session# [5:7587241432027265730:2049] VirtualId# [5:7587241432027265721:0]
2025-12-24T02:11:46.763098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.763106Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241432027265655:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.763109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.763385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241432104959642:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:46.763396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241432104959717:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:46.763401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241432104959642:2049] [node 2] ICP30 unregister session Session# [6:7587241432104959717:2049] VirtualId# [6:7587241432104959709:0]
2025-12-24T02:11:46.763457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:46.763463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241432104959642:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.763465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11149) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:46.775313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241432027265655:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:46.777866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241432104959642:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:46.777904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:46.778006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241432104959734:2050] [node 3] ICS01 socket: 449 reason# EndOfStream
2025-12-24T02:11:46.778015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241432104959643:2050] [node 3] ICP30 unregister session Session# [6:7587241432104959734:2050] VirtualId# [6:7587241432104959730:0]
2025-12-24T02:11:46.778023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241432104959734:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.778072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:46.778098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:46.783316Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:46.783436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241432027265751:2050] [node 3] ICS01 socket: 482 reason# EndOfStream
2025-12-24T02:11:46.783447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241432027265656:2050] [node 3] ICP30 unregister session Session# [5:7587241432027265751:2050] VirtualId# [5:7587241432027265747:0]
2025-12-24T02:11:46.783455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241432027265751:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.783510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:46.783538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:46.807171Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:46.807290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241432104959736:2116] [node 4] ICS01 socket: 468 reason# EndOfStream
2025-12-24T02:11:46.807301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241432104959723:2116] [node 4] ICP30 unregister session Session# [6:7587241432104959736:2116] VirtualId# [6:7587241432104959732:0]
2025-12-24T02:11:46.807310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241432104959736:2116] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:46.807375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:46.807402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:46.868803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434723991006:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:46.868813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434723991007:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:46.895043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241431432362309:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.895062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241431432362310:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.895068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241431432362311:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.926732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431976484195:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.926758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431976484196:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.927043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241435469730012:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.927078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241435469730013:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:46.926186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241433009310070:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.926203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241433009310071:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.926210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241433009310072:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.942978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241434471687157:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:46.943018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241434471687158:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:46.943025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241434471687159:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:46.954039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241433009310083:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:46.957070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434723991068:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:46.957174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241434723991069:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:46.960482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241434471687185:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:46.960591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241434471687186:2062] [node 5] ICP01 ready to work
2025-12-24T02:11:46.961824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431976484247:2098] [node 5] ICP01 ready to work
2025-12-24T02:11:46.955002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241431432362323:2059] [node 6] ICP01 ready to work
2025-12-24T02:11:46.955158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241433009310090:2063] [node 6] ICP01 ready to work
2025-12-24T02:11:46.972930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241435469730073:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.981096Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241434471687159:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20413323ns) {Configure:104(20412794ns) {ProcessConfigured:135(20020883ns) {StartInitialHandshake:157(20019972ns) {DropHandshakes:470(939ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(270ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(1603ns) {SwitchToState:179(972ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.984615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241433009310136:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:46.991707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241431432362323:2059] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36534580ns) {Configure:104(36533808ns) {ProcessConfigured:135(36104188ns) {StartInitialHandshake:157(36103557ns) {DropHandshakes:470(442ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(326ns) PrepareNewSessionHandshake:477(1239ns) {SwitchToState:179(844ns)}}}}}
2025-12-24T02:11:46.991837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241431432362324:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:46.992156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241431432362369:2065] [node 5] ICP01 ready to work
2025-12-24T02:11:46.993639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241431432362370:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:46.990486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241434471687227:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:46.991624Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241431976484247:2098] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(29694423ns) {Configure:104(29693642ns) {ProcessConfigured:135(29270893ns) {StartInitialHandshake:157(29270051ns) {DropHandshakes:470(637ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(230ns) PrepareNewSessionHandshake:477(1627ns) {SwitchToState:179(355ns)}}}}}
2025-12-24T02:11:46.991901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431976484260:2100] [node 6] ICP01 ready to work
2025-12-24T02:11:46.992153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241431976484261:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:46.993481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241431976484262:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:46.996435Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241431432362324:2061] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.002817Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241434723991068:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(45561048ns) {Configure:104(45560286ns) {ProcessConfigured:135(45154145ns) {StartInitialHandshake:157(45153208ns) {DropHandshakes:470(683ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(105ns) PrepareNewSessionHandshake:477(2587ns) {SwitchToState:179(1118ns)}}}}}
2025-12-24T02:11:47.003784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439018958371:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:47.004758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439018958372:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15929
2025-12-24T02:11:47.004831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439018958373:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15929
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.013719Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277445:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.013766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277446:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.018865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241434471687227:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.022680Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241435469730012:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(49661828ns) {Configure:104(49660200ns) {ProcessConfigured:135(49193789ns) {StartInitialHandshake:157(49192747ns) {DropHandshakes:470(1655ns) {DropIncomingHandshake:442(337ns) DropOutgoingHandshake:457(284ns)} GenerateSessionVirtualId:988(323ns) PrepareNewSessionHandshake:477(2254ns) {SwitchToState:179(1520ns)}}}}}
2025-12-24T02:11:47.023265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958372:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.027212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241439764697376:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:47.027366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241439764697377:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:47.028391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697378:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.037739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277447:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.042892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329686:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.064148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436271451571:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:47.064206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436271451572:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:47.064249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436271451573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:47.067038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436271451571:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.067236Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437304277445:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.067278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241433009310072:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.067439Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431976484259:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.067449Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431976484259:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.068308Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277449:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.068410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431976484259:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.068631Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437304277445:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.068704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241433009310072:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.068726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.068733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241433009310072:2050] [node 3] ICP22 created new session: [5:7587241437304277450:2050]
2025-12-24T02:11:47.068740Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437304277450:2050] [node 3] ICS09 handshake done sender: [5:7587241437304277445:2116] self: [5:7587241437304277448:0] peer: [3:7587241431976484248:0] socket: 445 qp: -1
2025-12-24T02:11:47.068748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437304277450:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.068758Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.069344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431976484247:2098] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.069359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.069365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431976484247:2098] [node 5] ICP22 created new session: [3:7587241436271451576:2098]
2025-12-24T02:11:47.069371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436271451576:2098] [node 5] ICS09 handshake done sender: [3:7587241431976484259:2109] self: [3:7587241431976484248:0] peer: [5:7587241437304277448:0] socket: 440 qp: -1
2025-12-24T02:11:47.069377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436271451576:2098] [node 5] ICS10 traffic start
2025-12-24T02:11:47.069383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.087631Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697388:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
2025-12-24T02:11:47.093550Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697378:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.097074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436271451573:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.097134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241431976484196:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.097329Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697380:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.097341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241439764697380:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.098056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241439764697380:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.098534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436271451580:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:47.098738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436271451573:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.098155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241435469730013:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.098181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.098189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241435469730013:2049] [node 3] ICP22 created new session: [2:7587241439764697391:2049]
2025-12-24T02:11:47.098196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241439764697391:2049] [node 3] ICS09 handshake done sender: [2:7587241439764697380:2113] self: [2:7587241439764697379:0] peer: [3:7587241436271451579:0] socket: 463 qp: -1
2025-12-24T02:11:47.098204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241439764697391:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:47.098214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.098851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437304277447:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.098947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241433009310071:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.099208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329687:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.099245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654532:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.099286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654533:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.099320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241434471687186:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.099388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241431432362370:2105] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.098806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241431976484196:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.098821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.098827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431976484196:2049] [node 2] ICP22 created new session: [3:7587241436271451581:2049]
2025-12-24T02:11:47.098833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436271451581:2049] [node 2] ICS09 handshake done sender: [3:7587241436271451573:2118] self: [3:7587241436271451579:0] peer: [2:7587241439764697379:0] socket: 466 qp: -1
2025-12-24T02:11:47.098838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436271451581:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.098846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.099679Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697388:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.099716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697375:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.099718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241439764697375:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.099695Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241433009310136:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.099731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241433009310094:2066] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.099734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241433009310094:2066] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.101069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241433009310094:2066] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.101159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241433009310090:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.101178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.101184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241433009310090:2063] [node 6] ICP22 created new session: [5:7587241437304277456:2063]
2025-12-24T02:11:47.101191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437304277456:2063] [node 6] ICS09 handshake done sender: [5:7587241433009310094:2066] self: [5:7587241433009310092:0] peer: [6:7587241438766654534:0] socket: 429 qp: -1
2025-12-24T02:11:47.101196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437304277456:2063] [node 6] ICS10 traffic start
2025-12-24T02:11:47.101205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.102875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241438766654532:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.103544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241434471687186:2062] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11024) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241434471687227:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.108041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439018958384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15929
2025-12-24T02:11:47.108091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439018958383:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15929
2025-12-24T02:11:47.108265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241435727329686:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.108299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241431432362309:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.108414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437304277446:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.108441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241433009310070:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.108559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958368:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.108565Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439018958368:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.108869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958373:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.108890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958370:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.108892Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439018958370:2111] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.109590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329690:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:47.109677Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277459:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.114861Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439018958368:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.115175Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241435727329686:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.115282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241431432362309:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.115308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.115316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241431432362309:2048] [node 1] ICP22 created new session: [4:7587241435727329691:2048]
2025-12-24T02:11:47.115324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241435727329691:2048] [node 1] ICS09 handshake done sender: [4:7587241435727329686:2117] self: [4:7587241435727329689:0] peer: [1:7587241434723991070:0] socket: 455 qp: -1
2025-12-24T02:11:47.115333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241435727329691:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.115345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.116655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277460:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.118061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439018958370:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.118284Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437304277446:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.118997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241434471687185:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.119143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654535:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.118361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241433009310070:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.118381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.118388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241433009310070:2048] [node 1] ICP22 created new session: [5:7587241437304277461:2048]
2025-12-24T02:11:47.118396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437304277461:2048] [node 1] ICS09 handshake done sender: [5:7587241437304277446:2117] self: [5:7587241437304277458:0] peer: [1:7587241439018958369:0] socket: 454 qp: -1
2025-12-24T02:11:47.118403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437304277461:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.118415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.119564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277463:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.120900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431976484262:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.121940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241439764697375:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.122263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437304277447:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.122346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241435469730073:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.122364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.122371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241435469730073:2107] [node 5] ICP22 created new session: [2:7587241439764697396:2107]
2025-12-24T02:11:47.122377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241439764697396:2107] [node 5] ICS09 handshake done sender: [2:7587241439764697375:2111] self: [2:7587241439764697374:0] peer: [5:7587241437304277453:0] socket: 462 qp: -1
2025-12-24T02:11:47.121080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241434471687227:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.122382Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241439764697396:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:47.122389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.121842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241438766654533:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.121854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241434471687186:2062] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.121868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.121875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241434471687186:2062] [node 5] ICP22 created new session: [6:7587241438766654538:2062]
2025-12-24T02:11:47.121881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241438766654538:2062] [node 5] ICS09 handshake done sender: [6:7587241434471687227:2111] self: [6:7587241438766654534:0] peer: [5:7587241433009310092:0] socket: 431 qp: -1
2025-12-24T02:11:47.121890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241438766654538:2062] [node 5] ICS10 traffic start
2025-12-24T02:11:47.121901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.122254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241434471687159:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.122797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241433009310071:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.122817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.122830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241433009310071:2049] [node 2] ICP22 created new session: [5:7587241437304277464:2049]
2025-12-24T02:11:47.122836Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437304277464:2049] [node 2] ICS09 handshake done sender: [5:7587241437304277447:2118] self: [5:7587241437304277453:0] peer: [2:7587241439764697374:0] socket: 465 qp: -1
2025-12-24T02:11:47.122843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437304277464:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.122850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.122878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434723991068:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.122899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.122905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434723991068:2107] [node 4] ICP22 created new session: [1:7587241439018958389:2107]
2025-12-24T02:11:47.122911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439018958389:2107] [node 4] ICS09 handshake done sender: [1:7587241439018958368:2110] self: [1:7587241434723991070:0] peer: [4:7587241435727329689:0] socket: 450 qp: -1
2025-12-24T02:11:47.122915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439018958389:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:47.122927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.123054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431976484264:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.123064Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431976484264:2111] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.124185Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654541:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.125555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958383:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.125629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958384:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.125636Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434723991069:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.125648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.125656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434723991069:2108] [node 5] ICP22 created new session: [1:7587241439018958391:2108]
2025-12-24T02:11:47.125661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439018958391:2108] [node 5] ICS09 handshake done sender: [1:7587241439018958370:2111] self: [1:7587241439018958369:0] peer: [5:7587241437304277458:0] socket: 451 qp: -1
2025-12-24T02:11:47.125666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439018958391:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:47.125673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.125160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697398:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
2025-12-24T02:11:47.125836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436271451572:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.125865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241431976484195:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.125940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431976484264:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.128114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654542:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.128130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241438766654533:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.128207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431976484260:2100] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.128229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.128235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431976484260:2100] [node 6] ICP22 created new session: [3:7587241436271451587:2100]
2025-12-24T02:11:47.128242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436271451587:2100] [node 6] ICS09 handshake done sender: [3:7587241431976484264:2111] self: [3:7587241431976484263:0] peer: [6:7587241438766654540:0] socket: 441 qp: -1
2025-12-24T02:11:47.128247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436271451587:2100] [node 6] ICS10 traffic start
2025-12-24T02:11:47.128256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.128683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241434471687159:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.128702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.128708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241434471687159:2050] [node 3] ICP22 created new session: [6:7587241438766654543:2050]
2025-12-24T02:11:47.128715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241438766654543:2050] [node 3] ICS09 handshake done sender: [6:7587241438766654533:2117] self: [6:7587241438766654540:0] peer: [3:7587241431976484263:0] socket: 459 qp: -1
2025-12-24T02:11:47.128722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241438766654543:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.128730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.129084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241431432362366:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.129098Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241431432362366:2102] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.129740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329694:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:47.130468Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437304277460:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.130505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241433009310083:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.130805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697398:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.134929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241431432362366:2102] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.135320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241431432362323:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.135342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.135349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241431432362323:2059] [node 6] ICP22 created new session: [4:7587241435727329696:2059]
2025-12-24T02:11:47.135357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241435727329696:2059] [node 6] ICS09 handshake done sender: [4:7587241431432362366:2102] self: [4:7587241431432362325:0] peer: [6:7587241438766654536:0] socket: 438 qp: -1
2025-12-24T02:11:47.135363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241435727329696:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:47.135374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.136020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241431432362376:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.136029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241431432362376:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.136554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241435727329687:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.136580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241435727329694:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.136673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241431432362310:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.136684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241431432362311:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.136967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241431976484270:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.136987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241431976484270:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.137726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241431976484270:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.138927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241431432362311:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21520) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241435727329694:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.137828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241431976484261:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.137849Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.137855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431976484261:2108] [node 4] ICP22 created new session: [3:7587241436271451590:2108]
2025-12-24T02:11:47.137862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436271451590:2108] [node 4] ICS09 handshake done sender: [3:7587241431976484270:2114] self: [3:7587241431976484269:0] peer: [4:7587241435727329701:0] socket: 482 qp: -1
2025-12-24T02:11:47.137869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436271451590:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:47.137878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.141154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697382:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.141172Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241439764697382:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.141830Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697401:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
2025-12-24T02:11:47.142383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241439764697382:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.142500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241439764697376:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.142516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.142524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241439764697376:2108] [node 4] ICP22 created new session: [2:7587241439764697403:2108]
2025-12-24T02:11:47.142532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241439764697403:2108] [node 4] ICS09 handshake done sender: [2:7587241439764697382:2114] self: [2:7587241439764697381:0] peer: [4:7587241435727329700:0] socket: 460 qp: -1
2025-12-24T02:11:47.142540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241439764697403:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:47.142552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.176390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241431432362376:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.176964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437304277467:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4139
2025-12-24T02:11:47.177933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958375:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.177949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439018958375:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.178923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437304277460:2121] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.178950Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241434471687185:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12508) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241438766654532:2116] held: no
2025-12-24T02:11:47.179064Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697405:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.179478Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654545:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.179400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241431432362369:2065] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.179426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.179432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241431432362369:2065] [node 5] ICP22 created new session: [4:7587241435727329704:2065]
2025-12-24T02:11:47.179439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241435727329704:2065] [node 5] ICS09 handshake done sender: [4:7587241431432362376:2108] self: [4:7587241431432362375:0] peer: [5:7587241437304277466:0] socket: 442 qp: -1
2025-12-24T02:11:47.179444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241435727329704:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:47.179451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.179027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241433009310083:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.179048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.179057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241433009310083:2059] [node 4] ICP22 created new session: [5:7587241437304277468:2059]
2025-12-24T02:11:47.179063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437304277468:2059] [node 4] ICS09 handshake done sender: [5:7587241437304277460:2121] self: [5:7587241437304277466:0] peer: [4:7587241431432362375:0] socket: 480 qp: -1
2025-12-24T02:11:47.179071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437304277468:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:47.179092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.182922Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654546:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.183058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241438766654542:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.183094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241438766654532:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.185200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436271451592:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4180
2025-12-24T02:11:47.185114Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439018958398:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15929
2025-12-24T02:11:47.185956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241434471687185:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.185979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.185988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241434471687185:2060] [node 4] ICP22 created new session: [6:7587241438766654548:2060]
2025-12-24T02:11:47.185996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241438766654548:2060] [node 4] ICS09 handshake done sender: [6:7587241438766654532:2116] self: [6:7587241438766654536:0] peer: [4:7587241431432362325:0] socket: 458 qp: -1
2025-12-24T02:11:47.186003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241438766654548:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:47.186016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.186248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241434471687158:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.189046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697384:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.189063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241439764697384:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.189993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241439764697384:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.190317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439018958375:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.190542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436271451572:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.190094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241439764697377:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.190119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.190128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241439764697377:2109] [node 6] ICP22 created new session: [2:7587241439764697407:2109]
2025-12-24T02:11:47.190136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241439764697407:2109] [node 6] ICS09 handshake done sender: [2:7587241439764697384:2115] self: [2:7587241439764697383:0] peer: [6:7587241438766654550:0] socket: 486 qp: -1
2025-12-24T02:11:47.190144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241439764697407:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:47.190153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.190488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434723991007:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.190508Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.190520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434723991007:2049] [node 3] ICP22 created new session: [1:7587241439018958399:2049]
2025-12-24T02:11:47.190528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439018958399:2049] [node 3] ICS09 handshake done sender: [1:7587241439018958375:2114] self: [1:7587241439018958374:0] peer: [3:7587241436271451586:0] socket: 452 qp: -1
2025-12-24T02:11:47.190536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439018958399:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:47.190545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.191092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329703:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:47.190631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241431976484195:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.190646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.190652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241431976484195:2048] [node 1] ICP22 created new session: [3:7587241436271451593:2048]
2025-12-24T02:11:47.190658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436271451593:2048] [node 1] ICS09 handshake done sender: [3:7587241436271451572:2117] self: [3:7587241436271451586:0] peer: [1:7587241439018958374:0] socket: 456 qp: -1
2025-12-24T02:11:47.190663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436271451593:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.190671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.191333Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241435727329702:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56368
2025-12-24T02:11:47.192044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241435727329687:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.193241Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697401:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.193276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241439764697376:2108] [node 4] ICP18 handshake (actor [2:7587241439764697401:2118]) is too old
2025-12-24T02:11:47.193432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241435727329694:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.193534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241431432362310:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.193550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.193561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241431432362310:2049] [node 2] ICP22 created new session: [4:7587241435727329706:2049]
2025-12-24T02:11:47.193568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241435727329706:2049] [node 2] ICS09 handshake done sender: [4:7587241435727329687:2118] self: [4:7587241435727329700:0] peer: [2:7587241439764697381:0] socket: 474 qp: -1
2025-12-24T02:11:47.193574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241435727329706:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.193581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.193945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241431432362311:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.193952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.193956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241431432362311:2050] [node 3] ICP22 created new session: [4:7587241435727329708:2050]
2025-12-24T02:11:47.193961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241435727329708:2050] [node 3] ICS09 handshake done sender: [4:7587241435727329694:2120] self: [4:7587241435727329701:0] peer: [3:7587241431976484269:0] socket: 487 qp: -1
2025-12-24T02:11:47.193966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241435727329708:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.193972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.195708Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241434471687158:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9945) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241438766654542:2120] held: no
2025-12-24T02:11:47.196013Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654551:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.196188Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241438766654542:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.196355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241434471687158:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.196378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.196386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241434471687158:2049] [node 2] ICP22 created new session: [6:7587241438766654552:2049]
2025-12-24T02:11:47.196393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241438766654552:2049] [node 2] ICS09 handshake done sender: [6:7587241438766654542:2120] self: [6:7587241438766654550:0] peer: [2:7587241439764697383:0] socket: 488 qp: -1
2025-12-24T02:11:47.196402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241438766654552:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.196429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.200308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958398:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.207026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241439764697405:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.207081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241435469730012:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.207316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958377:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.207329Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439018958377:2115] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.207768Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439018958379:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.207771Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439018958379:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.208667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241439764697410:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55590
2025-12-24T02:11:47.207167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241438766654546:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.207198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241434471687157:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.208587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241438766654555:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39745
2025-12-24T02:11:47.231005Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439018958377:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.231300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241439764697405:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.231440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241435469730012:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.231466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.231474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241435469730012:2048] [node 1] ICP22 created new session: [2:7587241439764697411:2048]
2025-12-24T02:11:47.231481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241439764697411:2048] [node 1] ICS09 handshake done sender: [2:7587241439764697405:2119] self: [2:7587241439764697409:0] peer: [1:7587241439018958376:0] socket: 433 qp: -1
2025-12-24T02:11:47.231489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241439764697411:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.231503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.232726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439018958379:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.233999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241434723991006:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.234021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.234027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241434723991006:2048] [node 2] ICP22 created new session: [1:7587241439018958404:2048]
2025-12-24T02:11:47.234032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439018958404:2048] [node 2] ICS09 handshake done sender: [1:7587241439018958377:2115] self: [1:7587241439018958376:0] peer: [2:7587241439764697409:0] socket: 448 qp: -1
2025-12-24T02:11:47.234037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439018958404:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:47.234043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.234524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439018958371:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.234532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.234536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439018958371:2109] [node 6] ICP22 created new session: [1:7587241439018958406:2109]
2025-12-24T02:11:47.234539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439018958406:2109] [node 6] ICS09 handshake done sender: [1:7587241439018958379:2116] self: [1:7587241439018958378:0] peer: [6:7587241438766654554:0] socket: 469 qp: -1
2025-12-24T02:11:47.234542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439018958406:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:47.234548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.232902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241438766654546:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.232957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241434471687157:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.232969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.232974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241434471687157:2048] [node 1] ICP22 created new session: [6:7587241438766654556:2048]
2025-12-24T02:11:47.232980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241438766654556:2048] [node 1] ICS09 handshake done sender: [6:7587241438766654546:2122] self: [6:7587241438766654554:0] peer: [1:7587241439018958378:0] socket: 491 qp: -1
2025-12-24T02:11:47.232985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241438766654556:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.232994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.250093Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:47.250203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241439764697411:2048] [node 1] ICS01 socket: 433 reason# EndOfStream
2025-12-24T02:11:47.250215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241435469730012:2048] [node 1] ICP30 unregister session Session# [2:7587241439764697411:2048] VirtualId# [2:7587241439764697409:0]
2025-12-24T02:11:47.250225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241439764697411:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.250276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.250301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.250811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:47.250897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241436271451593:2048] [node 1] ICS07 socket disconnect 456 reason# EndOfStream
2025-12-24T02:11:47.250903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241436271451593:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.250962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.250987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241436271451593:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:47.251375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241431976484195:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14654) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:47.251387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241436271451593:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:47.251392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241431976484195:2048] [node 1] ICP30 unregister session Session# [3:7587241436271451593:2048] VirtualId# [3:7587241436271451586:0]
2025-12-24T02:11:47.251447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.251453Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241431976484195:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14654) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.251456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14654) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.259179Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:47.259297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241438766654556:2048] [node 1] ICS01 socket: 491 reason# EndOfStream
2025-12-24T02:11:47.259310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241434471687157:2048] [node 1] ICP30 unregister session Session# [6:7587241438766654556:2048] VirtualId# [6:7587241438766654554:0]
2025-12-24T02:11:47.259321Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241438766654556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.259378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.259405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.266458Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:47.266541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241435727329691:2048] [node 1] ICS01 socket: 455 reason# EndOfStream
2025-12-24T02:11:47.266549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241431432362309:2048] [node 1] ICP30 unregister session Session# [4:7587241435727329691:2048] VirtualId# [4:7587241435727329689:0]
2025-12-24T02:11:47.266555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241435727329691:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.266588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.266611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.265787Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:47.265900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437304277461:2048] [node 1] ICS01 socket: 454 reason# EndOfStream
2025-12-24T02:11:47.265911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241433009310070:2048] [node 1] ICP30 unregister session Session# [5:7587241437304277461:2048] VirtualId# [5:7587241437304277458:0]
2025-12-24T02:11:47.265920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437304277461:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.266015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.266047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.266739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241431976484195:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:47.270839Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:47.270942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241436271451581:2049] [node 2] ICS07 socket disconnect 466 reason# EndOfStream
2025-12-24T02:11:47.270952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241436271451581:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.271017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.271040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241436271451581:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:47.271514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241431976484196:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9945) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:47.271527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241436271451581:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:47.271532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241431976484196:2049] [node 2] ICP30 unregister session Session# [3:7587241436271451581:2049] VirtualId# [3:7587241436271451579:0]
2025-12-24T02:11:47.271591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.271598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241431976484196:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.271601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9945) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.278891Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:47.279023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437304277464:2049] [node 2] ICS01 socket: 465 reason# EndOfStream
2025-12-24T02:11:47.279036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241433009310071:2049] [node 2] ICP30 unregister session Session# [5:7587241437304277464:2049] VirtualId# [5:7587241437304277453:0]
2025-12-24T02:11:47.279047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437304277464:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.279116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.278944Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:47.279041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241438766654552:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:47.279049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241434471687158:2049] [node 2] ICP30 unregister session Session# [6:7587241438766654552:2049] VirtualId# [6:7587241438766654550:0]
2025-12-24T02:11:47.279057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241438766654552:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.279124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.279170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.279202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:47.279271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241435727329706:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-12-24T02:11:47.279277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241431432362310:2049] [node 2] ICP30 unregister session Session# [4:7587241435727329706:2049] VirtualId# [4:7587241435727329700:0]
2025-12-24T02:11:47.279283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241435727329706:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.279314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.279339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.279149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.298456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:47.298594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437304277450:2050] [node 3] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:47.298610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241433009310072:2050] [node 3] ICP30 unregister session Session# [5:7587241437304277450:2050] VirtualId# [5:7587241437304277448:0]
2025-12-24T02:11:47.298620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437304277450:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.298704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:47.298736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:47.307365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:47.307499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241438766654543:2050] [node 3] ICS01 socket: 459 reason# EndOfStream
2025-12-24T02:11:47.307512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241434471687159:2050] [node 3] ICP30 unregister session Session# [6:7587241438766654543:2050] VirtualId# [6:7587241438766654540:0]
2025-12-24T02:11:47.307522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241438766654543:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.307602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:47.307635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:47.319370Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:47.319502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241438766654548:2060] [node 4] ICS01 socket: 458 reason# EndOfStream
2025-12-24T02:11:47.319515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241434471687185:2060] [node 4] ICP30 unregister session Session# [6:7587241438766654548:2060] VirtualId# [6:7587241438766654536:0]
2025-12-24T02:11:47.319524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241438766654548:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.319593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:47.319635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:47.358986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439807825298:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:47.358994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439807825299:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:47.371009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241436504764665:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.371032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241436504764666:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:47.378811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437735967654:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.378828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437735967655:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.378835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437735967656:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.371248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436329184089:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.371259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436329184090:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.387244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241439627705131:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.387263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241439627705132:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.387269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241439627705133:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.393199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241437178660438:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.393218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241437178660439:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.393224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241437178660440:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.491744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241437178660466:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:47.491857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241437178660468:2071] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.493055Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241436504764725:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47428
2025-12-24T02:11:47.495720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439807825361:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22587
2025-12-24T02:11:47.495989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764725:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.496579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705192:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.496741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967715:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.497218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825359:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.497228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439807825359:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.497311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241436504764665:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.497871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241436504764727:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47428
2025-12-24T02:11:47.497990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439807825359:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.498195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241436504764725:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.498276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241436504764665:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.498295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.498302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241436504764665:2048] [node 1] ICP22 created new session: [2:7587241436504764728:2048]
2025-12-24T02:11:47.498309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241436504764728:2048] [node 1] ICS09 handshake done sender: [2:7587241436504764725:2107] self: [2:7587241436504764726:0] peer: [1:7587241439807825358:0] socket: 436 qp: -1
2025-12-24T02:11:47.498317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241436504764728:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.498340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.498552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439807825298:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.498564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.498569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439807825298:2048] [node 2] ICP22 created new session: [1:7587241439807825363:2048]
2025-12-24T02:11:47.498574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439807825363:2048] [node 2] ICS09 handshake done sender: [1:7587241439807825359:2107] self: [1:7587241439807825358:0] peer: [2:7587241436504764726:0] socket: 432 qp: -1
2025-12-24T02:11:47.498579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439807825363:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:47.498585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.503743Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439807825368:2109] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.504625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184149:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.504807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705193:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
2025-12-24T02:11:47.507549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437735967715:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.507870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437735967717:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:47.508690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437735967717:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.509025Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705192:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.509182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241439627705195:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:47.509582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241439627705195:2110] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.509926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439807825374:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:47.510400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436329184151:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:47.510952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184149:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.511368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241436329184089:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.511615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705197:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
2025-12-24T02:11:47.511667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241436504764731:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.512374Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967719:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.514977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705197:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.515364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241439627705133:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.515512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184153:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.515524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436329184153:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.516218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436329184153:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.516301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241436504764739:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:47.516400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241436504764740:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:47.516448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436329184158:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:47.516466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436329184151:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.517555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241437178660474:2078] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.517573Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241437178660474:2078] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.517615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241437178660472:2077] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.517622Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241437178660472:2077] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.516478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.516485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436329184151:2108] [node 4] ICP22 created new session: [3:7587241436329184159:2108]
2025-12-24T02:11:47.516493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436329184159:2108] [node 4] ICS09 handshake done sender: [3:7587241436329184153:2109] self: [3:7587241436329184152:0] peer: [4:7587241439627705198:0] socket: 448 qp: -1
2025-12-24T02:11:47.516500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436329184159:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:47.516534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.517734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436329184165:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:47.517483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437735967719:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.517873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437735967655:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.523238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764733:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.523252Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241436504764733:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.524870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241436504764733:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.524978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241436504764731:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.524996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.525006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241436504764731:2109] [node 5] ICP22 created new session: [2:7587241436504764748:2109]
2025-12-24T02:11:47.525013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241436504764748:2109] [node 5] ICS09 handshake done sender: [2:7587241436504764733:2110] self: [2:7587241436504764732:0] peer: [5:7587241437735967720:0] socket: 451 qp: -1
2025-12-24T02:11:47.525020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241436504764748:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:47.525029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.531468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439807825377:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:47.531595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825367:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.531600Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439807825367:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.534544Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241437178660474:2078] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.534713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241437178660472:2077] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.534780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241437178660468:2071] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.534798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.534805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241437178660468:2071] [node 5] ICP22 created new session: [6:7587241437178660518:2071]
2025-12-24T02:11:47.534813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241437178660518:2071] [node 5] ICS09 handshake done sender: [6:7587241437178660474:2078] self: [6:7587241437178660473:0] peer: [5:7587241437735967718:0] socket: 435 qp: -1
2025-12-24T02:11:47.534820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241437178660518:2071] [node 5] ICS10 traffic start
2025-12-24T02:11:47.534830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.534925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825361:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.535002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439807825367:2110] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.539153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967723:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.542099Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241436504764731:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(17012897ns) {HandleHandshakeStatus:329(17011949ns) {DropIncomingHandshake:442(191ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(292ns) ProcessPendingSessionEvents:495(16919075ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 16900502ns max# 16822369ns}}}
2025-12-24T02:11:47.542160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241436504764750:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47428
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.543411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967724:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.543481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967725:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.543702Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437735967719:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.551137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241437178660466:2068] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.551162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.551176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241437178660466:2068] [node 4] ICP22 created new session: [6:7587241437178660523:2068]
2025-12-24T02:11:47.551184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241437178660523:2068] [node 4] ICS09 handshake done sender: [6:7587241437178660472:2077] self: [6:7587241437178660471:0] peer: [4:7587241439627705196:0] socket: 434 qp: -1
2025-12-24T02:11:47.551193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241437178660523:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:47.551204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.551383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439807825299:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.551411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.551416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439807825299:2049] [node 3] ICP22 created new session: [1:7587241439807825383:2049]
2025-12-24T02:11:47.551423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439807825383:2049] [node 3] ICS09 handshake done sender: [1:7587241439807825367:2110] self: [1:7587241439807825366:0] peer: [3:7587241436329184155:0] socket: 444 qp: -1
2025-12-24T02:11:47.551427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439807825383:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:47.551432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.553007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241439807825374:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42933510ns) {Configure:104(42932970ns) {ProcessConfigured:135(42569148ns) {StartInitialHandshake:157(42567824ns) {DropHandshakes:470(371ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(741ns) {SwitchToState:179(88ns)}}}}}
2025-12-24T02:11:47.555153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437735967722:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:47.555201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437735967715:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.559694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437735967655:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.559717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.559723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437735967655:2049] [node 2] ICP22 created new session: [5:7587241437735967730:2049]
2025-12-24T02:11:47.559731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437735967730:2049] [node 2] ICS09 handshake done sender: [5:7587241437735967719:2110] self: [5:7587241437735967720:0] peer: [2:7587241436504764732:0] socket: 452 qp: -1
2025-12-24T02:11:47.559739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437735967730:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.559761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.560203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437735967717:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.560212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.560217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437735967717:2109] [node 6] ICP22 created new session: [5:7587241437735967732:2109]
2025-12-24T02:11:47.560222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437735967732:2109] [node 6] ICS09 handshake done sender: [5:7587241437735967715:2108] self: [5:7587241437735967718:0] peer: [6:7587241437178660473:0] socket: 440 qp: -1
2025-12-24T02:11:47.560226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437735967732:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:47.560233Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.561923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439807825387:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22587
2025-12-24T02:11:47.561065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967734:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.559221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241439627705205:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:47.559562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705202:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.562150Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241439627705131:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18958667ns) {Configure:104(18958175ns) {ProcessConfigured:135(18582225ns) {StartInitialHandshake:157(18581593ns) {DropHandshakes:470(495ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(407ns) PrepareNewSessionHandshake:477(1325ns) {SwitchToState:179(135ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.567538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764750:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.568054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764738:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.568061Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241436504764738:2113] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.569727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241436504764738:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.569828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241436504764666:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.569859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.569865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241436504764666:2049] [node 3] ICP22 created new session: [2:7587241436504764754:2049]
2025-12-24T02:11:47.569872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241436504764754:2049] [node 3] ICS09 handshake done sender: [2:7587241436504764738:2113] self: [2:7587241436504764737:0] peer: [3:7587241436329184175:0] socket: 456 qp: -1
2025-12-24T02:11:47.569877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241436504764754:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:47.569885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.567349Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184172:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.567412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184173:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.567879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184172:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.567926Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184174:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.567941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241436329184090:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.567997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436329184149:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.568906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184176:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.568983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241436329184089:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.568999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.569007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436329184089:2048] [node 1] ICP22 created new session: [3:7587241436329184177:2048]
2025-12-24T02:11:47.569014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436329184177:2048] [node 1] ICS09 handshake done sender: [3:7587241436329184149:2107] self: [3:7587241436329184155:0] peer: [1:7587241439807825366:0] socket: 446 qp: -1
2025-12-24T02:11:47.569023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436329184177:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.569032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.570519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436329184172:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.570636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241437178660466:2068] [node 4] ICP35 event processing took too much time PendingConnection:279(19376720ns) {HandleHandshakeStatus:329(19375657ns) {DropIncomingHandshake:442(187ns) DropOutgoingHandshake:457(245ns) SwitchToState:179(206ns) ProcessPendingSessionEvents:495(15129901ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15109273ns max# 15021771ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.573994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437735967724:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:47.574062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437735967656:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
2025-12-24T02:11:47.574245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660520:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.574425Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241437178660468:2071] [node 5] ICP35 event processing took too much time PendingConnection:279(39512234ns) {HandleHandshakeStatus:329(39511225ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(240ns) SwitchToState:179(224ns) ProcessPendingSessionEvents:495(26556507ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26528211ns max# 26437282ns}}}
2025-12-24T02:11:47.574826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437735967734:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.574850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437735967654:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.574156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705203:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
2025-12-24T02:11:47.578095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660522:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.575939Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241436329184090:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17593) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241436329184172:2115] held: no
2025-12-24T02:11:47.576045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241436329184090:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.576061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.576070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436329184090:2049] [node 2] ICP22 created new session: [3:7587241436329184179:2049]
2025-12-24T02:11:47.576077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436329184179:2049] [node 2] ICS09 handshake done sender: [3:7587241436329184172:2115] self: [3:7587241436329184175:0] peer: [2:7587241436504764737:0] socket: 460 qp: -1
2025-12-24T02:11:47.576088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436329184179:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.576098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.576504Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184163:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.576508Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436329184163:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.577561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436329184163:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.577633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436329184158:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.577643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.577650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436329184158:2110] [node 5] ICP22 created new session: [3:7587241436329184182:2110]
2025-12-24T02:11:47.577656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436329184182:2110] [node 5] ICS09 handshake done sender: [3:7587241436329184163:2112] self: [3:7587241436329184162:0] peer: [5:7587241437735967740:0] socket: 461 qp: -1
2025-12-24T02:11:47.577662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436329184182:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:47.577669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.578035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705204:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.580220Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825385:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.580241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439807825385:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.581219Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967742:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.579007Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660521:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.579081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241437178660520:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.581636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967743:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.581658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437735967724:2113] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.582945Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439807825385:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.582870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241437178660440:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.583087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439807825374:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.583103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.583109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439807825374:2112] [node 5] ICP22 created new session: [1:7587241439807825389:2112]
2025-12-24T02:11:47.583115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439807825389:2112] [node 5] ICS09 handshake done sender: [1:7587241439807825385:2115] self: [1:7587241439807825375:0] peer: [5:7587241437735967741:0] socket: 475 qp: -1
2025-12-24T02:11:47.583121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439807825389:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:47.583130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.583575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437735967656:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.583595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.583601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437735967656:2050] [node 3] ICP22 created new session: [5:7587241437735967744:2050]
2025-12-24T02:11:47.583608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437735967744:2050] [node 3] ICS09 handshake done sender: [5:7587241437735967724:2113] self: [5:7587241437735967740:0] peer: [3:7587241436329184162:0] socket: 469 qp: -1
2025-12-24T02:11:47.583615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437735967744:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.583622Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.583711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705193:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.583745Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705203:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.583786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241439627705197:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.584124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241437178660521:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.584160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241437178660522:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.584187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241437178660438:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.584199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241437178660439:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.584386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241439627705192:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.585618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660530:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.583003Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241436504764756:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47428
2025-12-24T02:11:47.585851Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241437178660521:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.585735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184168:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.585748Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436329184168:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.586259Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436329184184:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13684
2025-12-24T02:11:47.586395Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764756:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.584359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825380:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.584376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439807825380:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.585077Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439807825380:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.585155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439807825377:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.585168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.585174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439807825377:2113] [node 6] ICP22 created new session: [1:7587241439807825392:2113]
2025-12-24T02:11:47.585180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439807825392:2113] [node 6] ICS09 handshake done sender: [1:7587241439807825380:2114] self: [1:7587241439807825379:0] peer: [6:7587241437178660528:0] socket: 465 qp: -1
2025-12-24T02:11:47.585186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439807825392:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:47.585193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.587057Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436329184168:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.587051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241437178660438:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.587069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.587078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241437178660438:2048] [node 1] ICP22 created new session: [6:7587241437178660531:2048]
2025-12-24T02:11:47.587084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241437178660531:2048] [node 1] ICS09 handshake done sender: [6:7587241437178660521:2116] self: [6:7587241437178660528:0] peer: [1:7587241439807825379:0] socket: 473 qp: -1
2025-12-24T02:11:47.587089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241437178660531:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.587096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.590743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764744:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.590763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241436504764744:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.592088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437735967734:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.592577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660533:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.592697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241437178660520:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.592745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241437178660440:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.592757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.592762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241437178660440:2050] [node 3] ICP22 created new session: [6:7587241437178660534:2050]
2025-12-24T02:11:47.592767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241437178660534:2050] [node 3] ICS09 handshake done sender: [6:7587241437178660520:2115] self: [6:7587241437178660527:0] peer: [3:7587241436329184167:0] socket: 466 qp: -1
2025-12-24T02:11:47.592772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241437178660534:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.592789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.594402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241439627705195:2110] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.594417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.594423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241439627705195:2110] [node 6] ICP22 created new session: [4:7587241439627705212:2110]
2025-12-24T02:11:47.594428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241439627705212:2110] [node 6] ICS09 handshake done sender: [4:7587241439627705192:2108] self: [4:7587241439627705196:0] peer: [6:7587241437178660471:0] socket: 439 qp: -1
2025-12-24T02:11:47.594434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241439627705212:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:47.594455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.593920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436329184165:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.593939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.593946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436329184165:2111] [node 6] ICP22 created new session: [3:7587241436329184186:2111]
2025-12-24T02:11:47.593952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436329184186:2111] [node 6] ICS09 handshake done sender: [3:7587241436329184168:2114] self: [3:7587241436329184167:0] peer: [6:7587241437178660527:0] socket: 463 qp: -1
2025-12-24T02:11:47.593958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436329184186:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:47.593966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.592304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437735967654:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.592324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.592332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437735967654:2048] [node 1] ICP22 created new session: [5:7587241437735967746:2048]
2025-12-24T02:11:47.592338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437735967746:2048] [node 1] ICS09 handshake done sender: [5:7587241437735967734:2117] self: [5:7587241437735967741:0] peer: [1:7587241439807825375:0] socket: 478 qp: -1
2025-12-24T02:11:47.592344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437735967746:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.592351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.595361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436329184184:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.595394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241436329184165:2111] [node 6] ICP18 handshake (actor [3:7587241436329184184:2119]) is too old
2025-12-24T02:11:47.595856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241439627705133:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.595874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.595879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241439627705133:2050] [node 3] ICP22 created new session: [4:7587241439627705214:2050]
2025-12-24T02:11:47.595886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241439627705214:2050] [node 3] ICS09 handshake done sender: [4:7587241439627705197:2111] self: [4:7587241439627705198:0] peer: [3:7587241436329184152:0] socket: 449 qp: -1
2025-12-24T02:11:47.595891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241439627705214:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.595896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.596319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241439627705131:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.596338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241439627705132:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.598046Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764742:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.598060Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241436504764742:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.598811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241436504764758:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47428
2025-12-24T02:11:47.596731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439807825370:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.596746Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439807825370:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.602309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241436504764744:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.602891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241436504764742:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.602732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705210:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.602748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439807825370:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.604249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439807825368:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.604264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.604272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439807825368:2109] [node 4] ICP22 created new session: [1:7587241439807825396:2109]
2025-12-24T02:11:47.604280Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439807825396:2109] [node 4] ICS09 handshake done sender: [1:7587241439807825370:2111] self: [1:7587241439807825369:0] peer: [4:7587241439627705218:0] socket: 445 qp: -1
2025-12-24T02:11:47.604288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439807825396:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:47.604297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.604811Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439807825395:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22587
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.614881Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705220:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.617508Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241439627705221:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41599
2025-12-24T02:11:47.617691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241439627705193:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.616515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967748:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.618768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241436504764739:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.618806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.618817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241436504764739:2111] [node 6] ICP22 created new session: [2:7587241436504764761:2111]
2025-12-24T02:11:47.618827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241436504764761:2111] [node 6] ICS09 handshake done sender: [2:7587241436504764744:2115] self: [2:7587241436504764743:0] peer: [6:7587241437178660529:0] socket: 472 qp: -1
2025-12-24T02:11:47.618838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241436504764761:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:47.618850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.619142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241436504764740:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.619148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.619153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241436504764740:2112] [node 4] ICP22 created new session: [2:7587241436504764763:2112]
2025-12-24T02:11:47.619158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241436504764763:2112] [node 4] ICS09 handshake done sender: [2:7587241436504764742:2114] self: [2:7587241436504764741:0] peer: [4:7587241439627705219:0] socket: 458 qp: -1
2025-12-24T02:11:47.619163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241436504764763:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:47.619168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.619703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241437178660536:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58731
2025-12-24T02:11:47.618336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241439627705203:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.620720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241439627705131:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.620734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.620741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241439627705131:2048] [node 1] ICP22 created new session: [4:7587241439627705222:2048]
2025-12-24T02:11:47.620748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241439627705222:2048] [node 1] ICS09 handshake done sender: [4:7587241439627705193:2109] self: [4:7587241439627705218:0] peer: [1:7587241439807825369:0] socket: 447 qp: -1
2025-12-24T02:11:47.620754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241439627705222:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.620762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.622926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241437178660522:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.624682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241437178660439:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.624709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.624718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241437178660439:2049] [node 2] ICP22 created new session: [6:7587241437178660537:2049]
2025-12-24T02:11:47.624727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241437178660537:2049] [node 2] ICS09 handshake done sender: [6:7587241437178660522:2117] self: [6:7587241437178660529:0] peer: [2:7587241436504764743:0] socket: 474 qp: -1
2025-12-24T02:11:47.624736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241437178660537:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.624748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.626781Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241437178660439:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:47.627337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241439627705132:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.627360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.627369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241439627705132:2049] [node 2] ICP22 created new session: [4:7587241439627705224:2049]
2025-12-24T02:11:47.627377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241439627705224:2049] [node 2] ICS09 handshake done sender: [4:7587241439627705203:2115] self: [4:7587241439627705219:0] peer: [2:7587241436504764741:0] socket: 476 qp: -1
2025-12-24T02:11:47.627384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241439627705224:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.627391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.635170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437735967748:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.635234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437735967722:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.635655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241436504764758:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.635723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241436504764740:2112] [node 4] ICP18 handshake (actor [2:7587241436504764758:2118]) is too old
2025-12-24T02:11:47.636215Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241439807825368:2109] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:47.636440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705209:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.636452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241439627705209:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.637073Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241439627705210:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.637487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437735967750:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49738
2025-12-24T02:11:47.637618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241439627705209:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.637786Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437735967748:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.637865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437735967722:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.637881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.637891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437735967722:2111] [node 4] ICP22 created new session: [5:7587241437735967751:2111]
2025-12-24T02:11:47.637898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437735967751:2111] [node 4] ICS09 handshake done sender: [5:7587241437735967748:2121] self: [5:7587241437735967749:0] peer: [4:7587241439627705208:0] socket: 503 qp: -1
2025-12-24T02:11:47.637906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437735967751:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:47.637930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.639155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241439627705205:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.639175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.639189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241439627705205:2112] [node 5] ICP22 created new session: [4:7587241439627705228:2112]
2025-12-24T02:11:47.639195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241439627705228:2112] [node 5] ICS09 handshake done sender: [4:7587241439627705209:2118] self: [4:7587241439627705208:0] peer: [5:7587241437735967749:0] socket: 485 qp: -1
2025-12-24T02:11:47.639201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241439627705228:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:47.639209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.670869Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:47.670988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241436504764728:2048] [node 1] ICS07 socket disconnect 436 reason# EndOfStream
2025-12-24T02:11:47.670996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241436504764728:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.671045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.671064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241436504764728:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:47.671691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:47.671766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241436329184177:2048] [node 1] ICS01 socket: 446 reason# EndOfStream
2025-12-24T02:11:47.671774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241436329184089:2048] [node 1] ICP30 unregister session Session# [3:7587241436329184177:2048] VirtualId# [3:7587241436329184155:0]
2025-12-24T02:11:47.671782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241436329184177:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.671809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.671845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.672360Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241436504764665:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:47.672373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241436504764728:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:47.672378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241436504764665:2048] [node 1] ICP30 unregister session Session# [2:7587241436504764728:2048] VirtualId# [2:7587241436504764726:0]
2025-12-24T02:11:47.672404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.672411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241436504764665:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.672414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.678224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:47.678315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241439627705222:2048] [node 1] ICS01 socket: 447 reason# EndOfStream
2025-12-24T02:11:47.678325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241439627705131:2048] [node 1] ICP30 unregister session Session# [4:7587241439627705222:2048] VirtualId# [4:7587241439627705218:0]
2025-12-24T02:11:47.678332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241439627705222:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.678384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.678410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.678448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:47.678519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437735967746:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-12-24T02:11:47.678525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437735967654:2048] [node 1] ICP30 unregister session Session# [5:7587241437735967746:2048] VirtualId# [5:7587241437735967741:0]
2025-12-24T02:11:47.678530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437735967746:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.678557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.678579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.678611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:47.678687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241437178660531:2048] [node 1] ICS07 socket disconnect 473 reason# EndOfStream
2025-12-24T02:11:47.678719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241437178660531:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.678744Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:47.678764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241437178660531:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:47.683277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241437178660438:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:47.683296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241437178660531:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:47.683301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241437178660438:2048] [node 1] ICP30 unregister session Session# [6:7587241437178660531:2048] VirtualId# [6:7587241437178660528:0]
2025-12-24T02:11:47.683350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:47.683357Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241437178660438:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.683361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15192) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.690832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:47.690929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241436329184179:2049] [node 2] ICS01 socket: 460 reason# EndOfStream
2025-12-24T02:11:47.690936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241436329184090:2049] [node 2] ICP30 unregister session Session# [3:7587241436329184179:2049] VirtualId# [3:7587241436329184175:0]
2025-12-24T02:11:47.690943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241436329184179:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.690981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.691005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.694063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241437178660438:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:47.698780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:47.698871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437735967730:2049] [node 2] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:47.698879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437735967655:2049] [node 2] ICP30 unregister session Session# [5:7587241437735967730:2049] VirtualId# [5:7587241437735967720:0]
2025-12-24T02:11:47.698886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437735967730:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.698937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.698959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.699000Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:47.699068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241439627705224:2049] [node 2] ICS01 socket: 476 reason# EndOfStream
2025-12-24T02:11:47.699073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241439627705132:2049] [node 2] ICP30 unregister session Session# [4:7587241439627705224:2049] VirtualId# [4:7587241439627705219:0]
2025-12-24T02:11:47.699078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241439627705224:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.699103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.699138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.699523Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:47.699594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241437178660537:2049] [node 2] ICS01 socket: 474 reason# EndOfStream
2025-12-24T02:11:47.699599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241437178660439:2049] [node 2] ICP30 unregister session Session# [6:7587241437178660537:2049] VirtualId# [6:7587241437178660529:0]
2025-12-24T02:11:47.699604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241437178660537:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.699637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:47.699657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:47.734036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:47.734150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241439627705214:2050] [node 3] ICS01 socket: 449 reason# EndOfStream
2025-12-24T02:11:47.734165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241439627705133:2050] [node 3] ICP30 unregister session Session# [4:7587241439627705214:2050] VirtualId# [4:7587241439627705198:0]
2025-12-24T02:11:47.734174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241439627705214:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.734221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:47.734246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:47.734286Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:47.734354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437735967744:2050] [node 3] ICS01 socket: 469 reason# EndOfStream
2025-12-24T02:11:47.734359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437735967656:2050] [node 3] ICP30 unregister session Session# [5:7587241437735967744:2050] VirtualId# [5:7587241437735967740:0]
2025-12-24T02:11:47.734365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437735967744:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.734387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:47.734405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:47.741818Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:47.741928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241437178660534:2050] [node 3] ICS01 socket: 466 reason# EndOfStream
2025-12-24T02:11:47.741939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241437178660440:2050] [node 3] ICP30 unregister session Session# [6:7587241437178660534:2050] VirtualId# [6:7587241437178660527:0]
2025-12-24T02:11:47.741947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241437178660534:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.742010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:47.742039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:47.753200Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:47.753340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241437178660523:2068] [node 4] ICS07 socket disconnect 434 reason# EndOfStream
2025-12-24T02:11:47.753349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241437178660523:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:47.753429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:47.753464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241437178660523:2068] [node 4] ICS15 start handshake
2025-12-24T02:11:47.755313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241437178660466:2068] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32674) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:47.755331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241437178660523:2068] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:47.755338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241437178660466:2068] [node 4] ICP30 unregister session Session# [6:7587241437178660523:2068] VirtualId# [6:7587241437178660471:0]
2025-12-24T02:11:47.755359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:47.755367Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241437178660466:2068] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32674) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.755371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32674) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:47.764891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241437178660466:2068] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:47.815054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439786677417:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:47.815067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439786677418:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:47.814792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436925147291:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.814810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436925147292:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.811570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241438323845371:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.811585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241438323845372:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:47.831477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437703023015:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.831485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437703023016:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.831491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437703023017:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.826925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241438688329308:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.826937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241438688329309:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.826942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241438688329310:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.847079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241436638131068:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:47.847090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241436638131069:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:47.847095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241436638131070:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:47.874057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241438323845417:2092] [node 6] ICP01 ready to work
2025-12-24T02:11:47.874114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241438688329324:2062] [node 6] ICP01 ready to work
2025-12-24T02:11:47.880056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436925147332:2084] [node 6] ICP01 ready to work
2025-12-24T02:11:47.882867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439786677482:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:47.883003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439786677483:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:47.883531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241436638131109:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:47.884363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241438688329370:2077] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.887663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241439786677486:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:47.888611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439786677489:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5620
2025-12-24T02:11:47.888625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439786677490:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.894988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329377:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.899197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329377:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.899243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241438688329308:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.913454Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241436925147332:2084] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(33268665ns) {Configure:104(33267725ns) {ProcessConfigured:135(32846503ns) {StartInitialHandshake:157(32845495ns) {DropHandshakes:470(692ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(324ns) PrepareNewSessionHandshake:477(2947ns) {SwitchToState:179(1665ns)}}}}}
2025-12-24T02:11:47.914384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436925147364:2093] [node 4] ICP01 ready to work
2025-12-24T02:11:47.914783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147361:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-12-24T02:11:47.915214Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241436638131068:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34315806ns) {Configure:104(34315337ns) {ProcessConfigured:135(34025393ns) {StartInitialHandshake:157(34025016ns) {DropHandshakes:470(424ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(767ns) {SwitchToState:179(230ns)}}}}}
2025-12-24T02:11:47.915728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241436638131131:2080] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.918955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329388:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.919032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131141:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.919118Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131142:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.919131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131139:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.919144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131140:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.920460Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241438323845371:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(37656032ns) {Configure:104(37654838ns) {ProcessConfigured:135(37232425ns) {StartInitialHandshake:157(37231916ns) {DropHandshakes:470(531ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(75ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1520ns) {SwitchToState:179(895ns)}}}}}
2025-12-24T02:11:47.920676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241438323845440:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:47.920906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241438323845441:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:47.922440Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241439786677482:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39370545ns) {Configure:104(39370022ns) {ProcessConfigured:135(38992057ns) {StartInitialHandshake:157(38991832ns) {DropHandshakes:470(396ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(309ns) PrepareNewSessionHandshake:477(1338ns) {SwitchToState:179(962ns)}}}}}
2025-12-24T02:11:47.923681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437703023072:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:47.924462Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241436638131141:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.924501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241436638131068:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.925279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241437703023063:2063] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.926880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677492:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.926900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439786677492:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.930571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241436925147358:2090] [node 5] ICP01 ready to work
2025-12-24T02:11:47.927817Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845450:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.927862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845451:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.927882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845452:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.927902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845453:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.927981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845454:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.934829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677488:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.934852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439786677488:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.939095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241436638131140:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.939143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241436638131109:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.941355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439786677500:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5620
2025-12-24T02:11:47.941377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439786677498:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5620
2025-12-24T02:11:47.941389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241439786677499:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5620
2025-12-24T02:11:47.942994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329329:2064] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.943012Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241438688329329:2064] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.944112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147361:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.945701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845453:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.945742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241438323845371:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.946908Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241436638131070:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(63267290ns) {Configure:104(63266834ns) {ProcessConfigured:135(62934622ns) {StartInitialHandshake:157(62934145ns) {DropHandshakes:470(242ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(845ns) {SwitchToState:179(242ns)}}}}}
2025-12-24T02:11:47.946801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677485:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.946818Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439786677485:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.948122Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329388:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.949026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845452:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.949375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439786677488:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.950296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677489:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.950351Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677500:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.949242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845450:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.953135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439786677492:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.953287Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439786677485:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.953509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439786677486:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.953533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.953541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439786677486:2107] [node 4] ICP22 created new session: [1:7587241439786677506:2107]
2025-12-24T02:11:47.953548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439786677506:2107] [node 4] ICS09 handshake done sender: [1:7587241439786677492:2114] self: [1:7587241439786677491:0] peer: [4:7587241438688329380:0] socket: 455 qp: -1
2025-12-24T02:11:47.953555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439786677506:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:47.953565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.953847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677490:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.953865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677498:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.953869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439786677417:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.953879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.953883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439786677417:2048] [node 2] ICP22 created new session: [1:7587241439786677508:2048]
2025-12-24T02:11:47.953887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439786677508:2048] [node 2] ICS09 handshake done sender: [1:7587241439786677485:2110] self: [1:7587241439786677479:0] peer: [2:7587241438323845455:0] socket: 453 qp: -1
2025-12-24T02:11:47.953891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439786677508:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:47.953896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.950548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241437703023072:2069] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(26693190ns) {Configure:104(26692329ns) {ProcessConfigured:135(26314670ns) {StartInitialHandshake:157(26314168ns) {DropHandshakes:470(657ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(139ns)} GenerateSessionVirtualId:988(229ns) PrepareNewSessionHandshake:477(531ns) {SwitchToState:179(18ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.954876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147372:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.954876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241438688329329:2064] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.955916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241438688329324:2062] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.955946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.955956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241438688329324:2062] [node 6] ICP22 created new session: [4:7587241438688329392:2062]
2025-12-24T02:11:47.955963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241438688329392:2062] [node 6] ICS09 handshake done sender: [4:7587241438688329329:2064] self: [4:7587241438688329328:0] peer: [6:7587241436638131150:0] socket: 440 qp: -1
2025-12-24T02:11:47.955971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241438688329392:2062] [node 6] ICS10 traffic start
2025-12-24T02:11:47.955981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.955018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677499:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.957929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439786677483:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.957941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.957947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439786677483:2109] [node 6] ICP22 created new session: [1:7587241439786677514:2109]
2025-12-24T02:11:47.957954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439786677514:2109] [node 6] ICS09 handshake done sender: [1:7587241439786677488:2111] self: [1:7587241439786677487:0] peer: [6:7587241436638131144:0] socket: 454 qp: -1
2025-12-24T02:11:47.957959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439786677514:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:47.957968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.954905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241436638131139:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.955026Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.955066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241436638131069:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.957726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131152:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.955158Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845451:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.958931Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241438323845458:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6673
2025-12-24T02:11:47.959298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147372:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.959338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241436925147291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.960562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147374:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-12-24T02:11:47.958782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329390:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.958853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241438688329308:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241438688329377:2106] held: no
2025-12-24T02:11:47.959738Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329394:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.959631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241436638131142:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.959650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241436638131140:2109] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.963759Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677496:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.963773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439786677496:2115] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.964808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-12-24T02:11:47.963002Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241436638131141:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.963148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241436638131109:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.963163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.963171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241436638131109:2060] [node 4] ICP22 created new session: [6:7587241436638131155:2060]
2025-12-24T02:11:47.963177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241436638131155:2060] [node 4] ICS09 handshake done sender: [6:7587241436638131140:2109] self: [6:7587241436638131150:0] peer: [4:7587241438688329328:0] socket: 465 qp: -1
2025-12-24T02:11:47.963184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241436638131155:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:47.963209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.963442Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241438323845371:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241438323845453:2118] held: no
2025-12-24T02:11:47.963585Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241438323845453:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.964500Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437703023093:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:47.964529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437703023094:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:47.967395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439786677496:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.967661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439786677418:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.967686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.967691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439786677418:2049] [node 3] ICP22 created new session: [1:7587241439786677517:2049]
2025-12-24T02:11:47.967697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439786677517:2049] [node 3] ICS09 handshake done sender: [1:7587241439786677496:2115] self: [1:7587241439786677480:0] peer: [3:7587241436925147373:0] socket: 451 qp: -1
2025-12-24T02:11:47.967701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439786677517:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:47.967707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:47.966658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241438323845371:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.966678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.966685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241438323845371:2048] [node 1] ICP22 created new session: [2:7587241438323845460:2048]
2025-12-24T02:11:47.966703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241438323845460:2048] [node 1] ICS09 handshake done sender: [2:7587241438323845453:2118] self: [2:7587241438323845455:0] peer: [1:7587241439786677479:0] socket: 476 qp: -1
2025-12-24T02:11:47.966711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241438323845460:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.966720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.967125Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845419:2093] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.967131Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241438323845419:2093] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.966851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241436638131068:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.966870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.966876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241436638131068:2048] [node 1] ICP22 created new session: [6:7587241436638131157:2048]
2025-12-24T02:11:47.966883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241436638131157:2048] [node 1] ICS09 handshake done sender: [6:7587241436638131141:2110] self: [6:7587241436638131144:0] peer: [1:7587241439786677487:0] socket: 467 qp: -1
2025-12-24T02:11:47.966889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241436638131157:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.966897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.971365Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436925147372:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.971727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241436925147291:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62485) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241436925147372:2116] held: no
2025-12-24T02:11:47.971835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241436925147291:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.971845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.971851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436925147291:2048] [node 1] ICP22 created new session: [3:7587241436925147376:2048]
2025-12-24T02:11:47.971857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436925147376:2048] [node 1] ICS09 handshake done sender: [3:7587241436925147372:2116] self: [3:7587241436925147373:0] peer: [1:7587241439786677480:0] socket: 486 qp: -1
2025-12-24T02:11:47.971863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436925147376:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.971884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.972368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437703023093:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.972411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437703023015:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.973348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241437703023096:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:47.974121Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437703023093:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.974900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241436638131070:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.972640Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241439786677497:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.972650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241439786677497:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.973490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241439786677497:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.973574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241439786677482:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.973587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.973593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241439786677482:2108] [node 5] ICP22 created new session: [1:7587241439786677520:2108]
2025-12-24T02:11:47.973598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241439786677520:2108] [node 5] ICS09 handshake done sender: [1:7587241439786677497:2116] self: [1:7587241439786677484:0] peer: [5:7587241437703023095:0] socket: 479 qp: -1
2025-12-24T02:11:47.973603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241439786677520:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:47.973609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:47.974226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437703023015:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.974242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.974248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437703023015:2048] [node 1] ICP22 created new session: [5:7587241437703023097:2048]
2025-12-24T02:11:47.974252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241437703023097:2048] [node 1] ICS09 handshake done sender: [5:7587241437703023093:2115] self: [5:7587241437703023095:0] peer: [1:7587241439786677484:0] socket: 485 qp: -1
2025-12-24T02:11:47.974258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241437703023097:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.974263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.975850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329390:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.975992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241438688329377:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:47.975922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437703023094:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.975950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437703023072:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.978093Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241439786677483:2109] [node 6] ICP35 event processing took too much time PendingConnection:279(20043123ns) {HandleHandshakeStatus:329(20042298ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(209ns) SwitchToState:179(79ns) ProcessPendingSessionEvents:495(19968383ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19937439ns max# 19860686ns}}}
2025-12-24T02:11:47.978129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241439786677483:2109] [node 6] ICP18 handshake (actor [1:7587241439786677499:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.980502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147379:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.981885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845454:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.981544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131160:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.983571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241438688329308:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.983594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:47.983602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241438688329308:2048] [node 1] ICP22 created new session: [4:7587241438688329397:2048]
2025-12-24T02:11:47.983610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241438688329397:2048] [node 1] ICS09 handshake done sender: [4:7587241438688329377:2106] self: [4:7587241438688329380:0] peer: [1:7587241439786677491:0] socket: 457 qp: -1
2025-12-24T02:11:47.983618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241438688329397:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:47.983636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:47.984064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241438688329309:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.984137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329399:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.985099Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329374:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.985110Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241438688329374:2104] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.986683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241438323845419:2093] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:47.986795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147360:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.986807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436925147360:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.987169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147374:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.987513Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845443:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.987521Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241438323845443:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.988112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241438323845417:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.988128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.988134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241438323845417:2092] [node 6] ICP22 created new session: [2:7587241438323845465:2092]
2025-12-24T02:11:47.988140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241438323845465:2092] [node 6] ICS09 handshake done sender: [2:7587241438323845419:2093] self: [2:7587241438323845418:0] peer: [6:7587241436638131154:0] socket: 439 qp: -1
2025-12-24T02:11:47.988145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241438323845465:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:47.988153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.989887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436925147360:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.990172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147379:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.990236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241436925147382:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-12-24T02:11:47.990250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241436925147292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.990446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436925147332:2084] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.989729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131162:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.989960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241438688329399:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.990045Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241438688329374:2104] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:47.990665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241438688329370:2077] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.990674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:47.990679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241438688329370:2077] [node 5] ICP22 created new session: [4:7587241438688329403:2077]
2025-12-24T02:11:47.990683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241438688329403:2077] [node 5] ICS09 handshake done sender: [4:7587241438688329374:2104] self: [4:7587241438688329373:0] peer: [5:7587241437703023100:0] socket: 447 qp: -1
2025-12-24T02:11:47.990688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241438688329403:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:47.990707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.994833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241436638131163:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:47.995024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241438688329310:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:47.995040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329402:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.995291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241438688329390:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.995223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241436638131069:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241436638131139:2108] held: no
2025-12-24T02:11:47.995343Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241436638131139:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:47.989554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241438323845443:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:47.995004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241438323845440:2096] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.995023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.995033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241438323845440:2096] [node 4] ICP22 created new session: [2:7587241438323845469:2096]
2025-12-24T02:11:47.995038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241438323845469:2096] [node 4] ICS09 handshake done sender: [2:7587241438323845443:2112] self: [2:7587241438323845442:0] peer: [4:7587241438688329400:0] socket: 472 qp: -1
2025-12-24T02:11:47.995045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241438323845469:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:47.995054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.997319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241438688329309:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.997340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:47.997348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241438688329309:2049] [node 2] ICP22 created new session: [4:7587241438688329406:2049]
2025-12-24T02:11:47.997353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241438688329406:2049] [node 2] ICS09 handshake done sender: [4:7587241438688329390:2117] self: [4:7587241438688329400:0] peer: [2:7587241438323845442:0] socket: 473 qp: -1
2025-12-24T02:11:47.997360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241438688329406:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:47.997368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:47.990459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:47.990464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436925147332:2084] [node 6] ICP22 created new session: [3:7587241436925147385:2084]
2025-12-24T02:11:47.990471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436925147385:2084] [node 6] ICS09 handshake done sender: [3:7587241436925147360:2111] self: [3:7587241436925147349:0] peer: [6:7587241436638131159:0] socket: 460 qp: -1
2025-12-24T02:11:47.990476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436925147385:2084] [node 6] ICS10 traffic start
2025-12-24T02:11:47.990493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:47.995371Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147366:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:47.995377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436925147366:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:47.998589Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436925147366:2114] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:47.999023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241438688329408:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58410
2025-12-24T02:11:47.998755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436925147364:2093] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:47.998773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:47.998777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436925147364:2093] [node 4] ICP22 created new session: [3:7587241436925147388:2093]
2025-12-24T02:11:47.998782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241436925147388:2093] [node 4] ICS09 handshake done sender: [3:7587241436925147366:2114] self: [3:7587241436925147365:0] peer: [4:7587241438688329405:0] socket: 449 qp: -1
2025-12-24T02:11:47.998787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241436925147388:2093] [node 4] ICS10 traffic start
2025-12-24T02:11:47.998792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:47.999148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241438688329399:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:47.999204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241438688329310:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:47.999229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:47.999235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241438688329310:2050] [node 3] ICP22 created new session: [4:7587241438688329409:2050]
2025-12-24T02:11:47.999241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241438688329409:2050] [node 3] ICS09 handshake done sender: [4:7587241438688329399:2119] self: [4:7587241438688329405:0] peer: [3:7587241436925147365:0] socket: 497 qp: -1
2025-12-24T02:11:47.999246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241438688329409:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:47.999253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.003448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845445:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.003470Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241438323845445:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.003521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241436638131142:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.004183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990397:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.004244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990398:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.004274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990399:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.005096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.004057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241436638131160:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.004073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241436638131069:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.004085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.004094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241436638131069:2049] [node 2] ICP22 created new session: [6:7587241440933098461:2049]
2025-12-24T02:11:48.004101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241440933098461:2049] [node 2] ICS09 handshake done sender: [6:7587241436638131139:2108] self: [6:7587241436638131154:0] peer: [2:7587241438323845418:0] socket: 445 qp: -1
2025-12-24T02:11:48.004107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241440933098461:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.004115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.004358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241436638131070:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.004371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.004379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241436638131070:2050] [node 3] ICP22 created new session: [6:7587241440933098463:2050]
2025-12-24T02:11:48.004386Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241440933098463:2050] [node 3] ICS09 handshake done sender: [6:7587241436638131142:2111] self: [6:7587241436638131159:0] peer: [3:7587241436925147349:0] socket: 468 qp: -1
2025-12-24T02:11:48.004392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241440933098463:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.004398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.005005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241436638131131:2080] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.005521Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437703023094:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.005601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437703023072:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.005617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.005623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437703023072:2069] [node 4] ICP22 created new session: [5:7587241441997990401:2069]
2025-12-24T02:11:48.005630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241441997990401:2069] [node 4] ICS09 handshake done sender: [5:7587241437703023094:2116] self: [5:7587241437703023100:0] peer: [4:7587241438688329373:0] socket: 492 qp: -1
2025-12-24T02:11:48.005636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241441997990401:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:48.005681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.007594Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241437703023091:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.007606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241437703023091:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.008154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241441997990400:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.008301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437703023017:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.011360Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241438323845445:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.011517Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241437703023091:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.010996Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241436925147370:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.011008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241436925147370:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.011741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241441220114686:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50698
2025-12-24T02:11:48.011841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241441997990398:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.011880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241437703023016:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.012086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436925147370:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.012174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241436925147358:2090] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.012193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.012200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436925147358:2090] [node 5] ICP22 created new session: [3:7587241441220114688:2090]
2025-12-24T02:11:48.012207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241441220114688:2090] [node 5] ICS09 handshake done sender: [3:7587241436925147370:2115] self: [3:7587241436925147369:0] peer: [5:7587241441997990403:0] socket: 506 qp: -1
2025-12-24T02:11:48.012213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241441220114688:2090] [node 5] ICS10 traffic start
2025-12-24T02:11:48.012220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.012349Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241441997990397:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.012199Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241438323845447:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.012209Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241438323845447:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.013158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241438323845447:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.013178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241438323845372:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.013193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.013199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241438323845372:2049] [node 3] ICP22 created new session: [2:7587241442618812769:2049]
2025-12-24T02:11:48.013205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241442618812769:2049] [node 3] ICS09 handshake done sender: [2:7587241438323845445:2113] self: [2:7587241438323845444:0] peer: [3:7587241436925147383:0] socket: 452 qp: -1
2025-12-24T02:11:48.013210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241442618812769:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:48.013217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.013806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241438323845441:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.013819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.013824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241438323845441:2102] [node 5] ICP22 created new session: [2:7587241442618812771:2102]
2025-12-24T02:11:48.013829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241442618812771:2102] [node 5] ICS09 handshake done sender: [2:7587241438323845447:2114] self: [2:7587241438323845446:0] peer: [5:7587241441997990405:0] socket: 501 qp: -1
2025-12-24T02:11:48.013833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241442618812771:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:48.013838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.011795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241440933098466:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62969
2025-12-24T02:11:48.013697Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241436638131160:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.014177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241436638131131:2080] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.014185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.014189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241436638131131:2080] [node 5] ICP22 created new session: [6:7587241440933098467:2080]
2025-12-24T02:11:48.014194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241440933098467:2080] [node 5] ICS09 handshake done sender: [6:7587241436638131160:2121] self: [6:7587241440933098465:0] peer: [5:7587241437703023090:0] socket: 471 qp: -1
2025-12-24T02:11:48.014198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241440933098467:2080] [node 5] ICS10 traffic start
2025-12-24T02:11:48.014205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.014854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241437703023063:2063] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.014874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.014880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437703023063:2063] [node 6] ICP22 created new session: [5:7587241441997990407:2063]
2025-12-24T02:11:48.014885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241441997990407:2063] [node 6] ICS09 handshake done sender: [5:7587241437703023091:2113] self: [5:7587241437703023090:0] peer: [6:7587241440933098465:0] socket: 495 qp: -1
2025-12-24T02:11:48.014889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241441997990407:2063] [node 6] ICS10 traffic start
2025-12-24T02:11:48.014896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.021477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241436925147379:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.021595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.021701Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241436925147292:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241436925147379:2119] held: no
2025-12-24T02:11:48.021793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241436925147292:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.021804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.021809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241436925147292:2049] [node 2] ICP22 created new session: [3:7587241441220114690:2049]
2025-12-24T02:11:48.021814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241441220114690:2049] [node 2] ICS09 handshake done sender: [3:7587241436925147379:2119] self: [3:7587241436925147383:0] peer: [2:7587241438323845444:0] socket: 461 qp: -1
2025-12-24T02:11:48.021819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241441220114690:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.021826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.022284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241436925147332:2084] [node 6] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.022417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241441997990410:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33911
2025-12-24T02:11:48.022534Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241437703023016:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4378) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241441997990398:2119] held: no
2025-12-24T02:11:48.022628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241441997990398:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.022680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437703023016:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.022704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.022710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437703023016:2049] [node 2] ICP22 created new session: [5:7587241441997990411:2049]
2025-12-24T02:11:48.022715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241441997990411:2049] [node 2] ICS09 handshake done sender: [5:7587241441997990398:2119] self: [5:7587241441997990405:0] peer: [2:7587241438323845446:0] socket: 438 qp: -1
2025-12-24T02:11:48.022720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241441997990411:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.022725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.023794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241437703023017:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2758) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241441997990400:2121] held: no
2025-12-24T02:11:48.023883Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241441997990400:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.024168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241437703023017:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.024180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.024184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241437703023017:2050] [node 3] ICP22 created new session: [5:7587241441997990413:2050]
2025-12-24T02:11:48.024188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241441997990413:2050] [node 3] ICS09 handshake done sender: [5:7587241441997990400:2121] self: [5:7587241441997990403:0] peer: [3:7587241436925147369:0] socket: 490 qp: -1
2025-12-24T02:11:48.024194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241441997990413:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.024199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.045357Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:48.045462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241438323845460:2048] [node 1] ICS01 socket: 476 reason# EndOfStream
2025-12-24T02:11:48.045473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241438323845371:2048] [node 1] ICP30 unregister session Session# [2:7587241438323845460:2048] VirtualId# [2:7587241438323845455:0]
2025-12-24T02:11:48.045483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241438323845460:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.045523Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.045546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.045975Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:48.046043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241436925147376:2048] [node 1] ICS01 socket: 486 reason# EndOfStream
2025-12-24T02:11:48.046049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241436925147291:2048] [node 1] ICP30 unregister session Session# [3:7587241436925147376:2048] VirtualId# [3:7587241436925147373:0]
2025-12-24T02:11:48.046054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241436925147376:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.046083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.046101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.050635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:48.050747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241436638131157:2048] [node 1] ICS01 socket: 467 reason# EndOfStream
2025-12-24T02:11:48.050756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241436638131068:2048] [node 1] ICP30 unregister session Session# [6:7587241436638131157:2048] VirtualId# [6:7587241436638131144:0]
2025-12-24T02:11:48.050763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241436638131157:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.050800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.050822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.050752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:48.050832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241437703023097:2048] [node 1] ICS01 socket: 485 reason# EndOfStream
2025-12-24T02:11:48.050839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437703023015:2048] [node 1] ICP30 unregister session Session# [5:7587241437703023097:2048] VirtualId# [5:7587241437703023095:0]
2025-12-24T02:11:48.050845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241437703023097:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.050892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.050922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.051066Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:48.051150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241438688329397:2048] [node 1] ICS01 socket: 457 reason# EndOfStream
2025-12-24T02:11:48.051157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241438688329308:2048] [node 1] ICP30 unregister session Session# [4:7587241438688329397:2048] VirtualId# [4:7587241438688329380:0]
2025-12-24T02:11:48.051162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241438688329397:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.051204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.051228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.071749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:48.071872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241441220114690:2049] [node 2] ICS01 socket: 461 reason# EndOfStream
2025-12-24T02:11:48.071884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241436925147292:2049] [node 2] ICP30 unregister session Session# [3:7587241441220114690:2049] VirtualId# [3:7587241436925147383:0]
2025-12-24T02:11:48.071893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241441220114690:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.071935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.071958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.078827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:48.078913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241440933098461:2049] [node 2] ICS01 socket: 445 reason# EndOfStream
2025-12-24T02:11:48.078922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241436638131069:2049] [node 2] ICP30 unregister session Session# [6:7587241440933098461:2049] VirtualId# [6:7587241436638131154:0]
2025-12-24T02:11:48.078929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241440933098461:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.078980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.079006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.078771Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:48.078911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241438688329406:2049] [node 2] ICS01 socket: 473 reason# EndOfStream
2025-12-24T02:11:48.078919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241438688329309:2049] [node 2] ICP30 unregister session Session# [4:7587241438688329406:2049] VirtualId# [4:7587241438688329400:0]
2025-12-24T02:11:48.078928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241438688329406:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.078983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.079007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.079814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:48.079915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241441997990411:2049] [node 2] ICS01 socket: 438 reason# EndOfStream
2025-12-24T02:11:48.079923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437703023016:2049] [node 2] ICP30 unregister session Session# [5:7587241441997990411:2049] VirtualId# [5:7587241441997990405:0]
2025-12-24T02:11:48.079929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241441997990411:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.079974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.079997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.106876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:48.106877Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:48.107016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241441997990413:2050] [node 3] ICS01 socket: 490 reason# EndOfStream
2025-12-24T02:11:48.107028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241437703023017:2050] [node 3] ICP30 unregister session Session# [5:7587241441997990413:2050] VirtualId# [5:7587241441997990403:0]
2025-12-24T02:11:48.107036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241441997990413:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.107091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.107114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.107021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241440933098463:2050] [node 3] ICS01 socket: 468 reason# EndOfStream
2025-12-24T02:11:48.107033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241436638131070:2050] [node 3] ICP30 unregister session Session# [6:7587241440933098463:2050] VirtualId# [6:7587241436638131159:0]
2025-12-24T02:11:48.107039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241440933098463:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.107094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.107121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.130974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:48.131075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241436638131155:2060] [node 4] ICS01 socket: 465 reason# EndOfStream
2025-12-24T02:11:48.131085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241436638131109:2060] [node 4] ICP30 unregister session Session# [6:7587241436638131155:2060] VirtualId# [6:7587241436638131150:0]
2025-12-24T02:11:48.131093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241436638131155:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.131142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:48.131167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:48.182950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241442049659493:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:48.182963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241442049659494:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.192802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241440918784622:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.192811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241440918784623:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.192816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241440918784624:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.191541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241442449115541:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.191551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241442449115542:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.191849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241442980119005:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.191858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241442980119006:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.191864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241442980119007:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.186803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241440608760577:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.186817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241440608760578:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.210816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241442966696901:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.210826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241442966696902:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.210831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241442966696903:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.223199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241440918784635:2059] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.224197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241442049659543:2092] [node 4] ICP01 ready to work
2025-12-24T02:11:48.224300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241442049659544:2096] [node 5] ICP01 ready to work
2025-12-24T02:11:48.225274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241440918784643:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:48.225476Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696918:2063] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.225497Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696919:2064] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.223604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241442449115583:2083] [node 5] ICP01 ready to work
2025-12-24T02:11:48.223713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241442449115584:2085] [node 6] ICP01 ready to work
2025-12-24T02:11:48.223802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241442449115585:2088] [node 4] ICP01 ready to work
2025-12-24T02:11:48.227752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241440608760632:2094] [node 6] ICP01 ready to work
2025-12-24T02:11:48.227853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241440608760633:2097] [node 5] ICP01 ready to work
2025-12-24T02:11:48.228069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241442966696928:2066] [node 4] ICP01 ready to work
2025-12-24T02:11:48.228177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241442966696929:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:48.238112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241440608760617:2086] [node 4] ICP01 ready to work
2025-12-24T02:11:48.238187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241442966696918:2063] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.247368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119053:2090] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.247403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119054:2091] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.247998Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241440918784623:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20661491ns) {Configure:104(20660798ns) {ProcessConfigured:135(20428220ns) {StartInitialHandshake:157(20427956ns) {DropHandshakes:470(281ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(765ns) {SwitchToState:179(321ns)}}}}}
2025-12-24T02:11:48.249941Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241442049659543:2092] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(25173099ns) {Configure:104(25172611ns) {ProcessConfigured:135(24958395ns) {StartInitialHandshake:157(24958208ns) {DropHandshakes:470(181ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1136ns) {SwitchToState:179(60ns)}}}}}
2025-12-24T02:11:48.250923Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241442449115542:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(23293668ns) {Configure:104(23293141ns) {ProcessConfigured:135(22964568ns) {StartInitialHandshake:157(22964098ns) {DropHandshakes:470(428ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(789ns) {SwitchToState:179(515ns)}}}}}
2025-12-24T02:11:48.254439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241440608760577:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26527432ns) {Configure:104(26526919ns) {ProcessConfigured:135(26240652ns) {StartInitialHandshake:157(26239738ns) {DropHandshakes:470(374ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(52ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:477(1104ns) {SwitchToState:179(660ns)}}}}}
2025-12-24T02:11:48.257655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241440608760632:2094] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(29682638ns) {Configure:104(29681839ns) {ProcessConfigured:135(29409695ns) {StartInitialHandshake:157(29409122ns) {DropHandshakes:470(222ns) {DropIncomingHandshake:442(34ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(213ns) PrepareNewSessionHandshake:477(562ns) {SwitchToState:179(159ns)}}}}}
2025-12-24T02:11:48.255330Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119054:2091] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.255363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119053:2090] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.255396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241442980119007:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.260727Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119062:2096] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.260784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119063:2097] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.261116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784687:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.261189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784688:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.258938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241442966696928:2066] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30661668ns) {Configure:104(30660887ns) {ProcessConfigured:135(30327804ns) {StartInitialHandshake:157(30327571ns) {DropHandshakes:470(299ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(91ns) PrepareNewSessionHandshake:477(1146ns) {SwitchToState:179(924ns)}}}}}
2025-12-24T02:11:48.262854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241442449115591:2091] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.262871Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241442449115591:2091] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.262448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241442049659559:2100] [node 6] ICP01 ready to work
2025-12-24T02:11:48.261493Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241442980119005:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30570243ns) {Configure:104(30569671ns) {ProcessConfigured:135(30229507ns) {StartInitialHandshake:157(30229226ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(771ns) {SwitchToState:179(378ns)}}}}}
2025-12-24T02:11:48.262928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241442980119036:2074] [node 5] ICP01 ready to work
2025-12-24T02:11:48.263600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119062:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.265081Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241440918784687:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.265376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241442966696919:2064] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.266826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241442980119006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.267131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241440918784622:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(43322314ns) {Configure:104(43321970ns) {ProcessConfigured:135(43080530ns) {StartInitialHandshake:157(43080339ns) {DropHandshakes:470(309ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(699ns) {SwitchToState:179(31ns)}}}}}
2025-12-24T02:11:48.270609Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241440918784624:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32866416ns) {Configure:104(32865967ns) {ProcessConfigured:135(32583463ns) {StartInitialHandshake:157(32582697ns) {DropHandshakes:470(335ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(480ns)}}}}}
2025-12-24T02:11:48.270677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241440918784624:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.271624Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784703:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.271969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241440918784687:2102] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.272074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241440918784624:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.272096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.272102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241440918784624:2050] [node 3] ICP22 created new session: [5:7587241440918784704:2050]
2025-12-24T02:11:48.272108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241440918784704:2050] [node 3] ICS09 handshake done sender: [5:7587241440918784687:2102] self: [5:7587241440918784702:0] peer: [3:7587241442449115586:0] socket: 452 qp: -1
2025-12-24T02:11:48.272115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241440918784704:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.272137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.270977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241442449115587:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.270988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241442449115587:2089] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.271782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241442449115587:2089] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.273938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241442449115583:2083] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.273949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.273955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241442449115583:2083] [node 5] ICP22 created new session: [3:7587241442449115620:2083]
2025-12-24T02:11:48.273961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241442449115620:2083] [node 5] ICS09 handshake done sender: [3:7587241442449115587:2089] self: [3:7587241442449115586:0] peer: [5:7587241440918784702:0] socket: 448 qp: -1
2025-12-24T02:11:48.273966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241442449115620:2083] [node 5] ICS10 traffic start
2025-12-24T02:11:48.273974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.277672Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241440608760633:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(49434884ns) {Configure:104(39359740ns) {ProcessConfigured:135(38950248ns) {StartInitialHandshake:157(38949826ns) {DropHandshakes:470(378ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(764ns) {SwitchToState:179(284ns)}}}}}
2025-12-24T02:11:48.277215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241442049659574:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10079
2025-12-24T02:11:48.277297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241442049659571:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10079
2025-12-24T02:11:48.277324Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241442049659572:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10079
2025-12-24T02:11:48.277365Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241442049659573:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10079
2025-12-24T02:11:48.278834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760644:2105] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.278847Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241440608760644:2105] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.280875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241442980119051:2079] [node 6] ICP01 ready to work
2025-12-24T02:11:48.279120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659574:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.279150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659572:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.287806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241442449115591:2091] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.288079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241442449115585:2088] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.288093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.288101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241442449115585:2088] [node 4] ICP22 created new session: [3:7587241442449115624:2088]
2025-12-24T02:11:48.288107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241442449115624:2088] [node 4] ICS09 handshake done sender: [3:7587241442449115591:2091] self: [3:7587241442449115590:0] peer: [4:7587241442980119059:0] socket: 450 qp: -1
2025-12-24T02:11:48.288114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241442449115624:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:48.288123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.287591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241442049659559:2100] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24979260ns) {Configure:104(24978593ns) {ProcessConfigured:135(24565430ns) {StartInitialHandshake:157(24564883ns) {DropHandshakes:470(829ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(793ns) {SwitchToState:179(62ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.290742Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.290940Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241442980119053:2090] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.291386Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241442980119036:2074] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28320066ns) {Configure:104(28319856ns) {ProcessConfigured:135(27928980ns) {StartInitialHandshake:157(27915328ns) {DropHandshakes:470(359ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(895ns) {SwitchToState:179(162ns)}} IncomingHandshake:261(5853ns) {DropIncomingHandshake:442(151ns) PrepareNewSessionHandshake:477(188ns) {SwitchToState:179(148ns)} GenerateSessionVirtualId:988(675ns) IssueIncomingHandshakeReply:193(211ns)}}}}
2025-12-24T02:11:48.299219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241442980119007:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.299246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.299254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241442980119007:2050] [node 3] ICP22 created new session: [4:7587241442980119090:2050]
2025-12-24T02:11:48.299262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241442980119090:2050] [node 3] ICS09 handshake done sender: [4:7587241442980119053:2090] self: [4:7587241442980119059:0] peer: [3:7587241442449115590:0] socket: 451 qp: -1
2025-12-24T02:11:48.299271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241442980119090:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.299294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.299596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241440918784688:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.299651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241440918784622:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.299986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659571:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.305920Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241442449115626:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56074
2025-12-24T02:11:48.306001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241442449115627:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56074
2025-12-24T02:11:48.306037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241442449115628:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56074
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.307103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241442449115628:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.307186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241442449115627:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.307216Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241442449115542:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.307140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241440608760644:2105] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.310946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659558:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.310964Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241442049659558:2106] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.311786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241442449115541:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(70258056ns) {Configure:104(70257411ns) {ProcessConfigured:135(69882768ns) {StartInitialHandshake:157(69882154ns) {DropHandshakes:470(440ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(536ns) PrepareNewSessionHandshake:477(879ns) {SwitchToState:179(65ns)}}}}}
2025-12-24T02:11:48.311892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241442449115541:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.312415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119063:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.313748Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241442049659558:2106] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.313957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659573:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.313970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241442049659544:2096] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.313987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.314008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241442049659544:2096] [node 5] ICP22 created new session: [1:7587241442049659580:2096]
2025-12-24T02:11:48.314014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241442049659580:2096] [node 5] ICS09 handshake done sender: [1:7587241442049659558:2106] self: [1:7587241442049659557:0] peer: [5:7587241440918784707:0] socket: 461 qp: -1
2025-12-24T02:11:48.314020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241442049659580:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:48.314028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.312591Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241442966696929:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(68121174ns) {Configure:104(68119258ns) {ProcessConfigured:135(67360696ns) {StartInitialHandshake:157(67360034ns) {DropHandshakes:470(158ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(687ns) {SwitchToState:179(322ns)}}}}}
2025-12-24T02:11:48.312686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241442966696929:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.314420Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241442966696903:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(67613935ns) {Configure:104(67613120ns) {ProcessConfigured:135(67347713ns) {StartInitialHandshake:157(67347052ns) {DropHandshakes:470(375ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(227ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(403ns)}}}}}
2025-12-24T02:11:48.314520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241442966696903:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.314295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760659:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.314355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241440608760617:2086] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.314365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.314371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241440608760617:2086] [node 4] ICP22 created new session: [2:7587241440608760660:2086]
2025-12-24T02:11:48.314378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241440608760660:2086] [node 4] ICS09 handshake done sender: [2:7587241440608760644:2105] self: [2:7587241440608760643:0] peer: [4:7587241442980119084:0] socket: 472 qp: -1
2025-12-24T02:11:48.314384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241440608760660:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:48.314402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.314598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760620:2088] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.314603Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241440608760620:2088] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.315332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760657:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.315376Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760658:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.316209Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241442980119007:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(16900290ns) {HandleHandshakeStatus:329(16899152ns) {DropIncomingHandshake:442(638ns) DropOutgoingHandshake:457(5669ns) SwitchToState:179(97ns) ProcessPendingSessionEvents:495(5354704ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5333321ns max# 5254025ns}}}
2025-12-24T02:11:48.317355Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241440608760620:2088] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.319375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241442449115633:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56074
2025-12-24T02:11:48.319541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241440918784638:2061] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.319554Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241440918784638:2061] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.319603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659554:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.319612Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241442049659554:2103] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.317828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760659:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.317839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241440608760578:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.317848Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.317854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241440608760578:2049] [node 3] ICP22 created new session: [2:7587241440608760664:2049]
2025-12-24T02:11:48.317860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241440608760664:2049] [node 3] ICS09 handshake done sender: [2:7587241440608760620:2088] self: [2:7587241440608760619:0] peer: [3:7587241442449115629:0] socket: 458 qp: -1
2025-12-24T02:11:48.317865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241440608760664:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:48.317870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.319723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241442449115589:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.319729Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241442449115589:2090] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.320463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241442049659582:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10079
2025-12-24T02:11:48.319782Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119092:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.320822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241442980119062:2096] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.321479Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241442980119051:2079] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40454000ns) {Configure:104(40453464ns) {ProcessConfigured:135(40022626ns) {StartInitialHandshake:157(40022011ns) {DropHandshakes:470(652ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(528ns) {SwitchToState:179(193ns)}}}}}
2025-12-24T02:11:48.321869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241440918784622:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24359) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241440918784688:2103] held: no
2025-12-24T02:11:48.322180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241442449115627:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.322270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241442980119006:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.322284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.322289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241442980119006:2049] [node 2] ICP22 created new session: [4:7587241442980119097:2049]
2025-12-24T02:11:48.322296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241442980119097:2049] [node 2] ICS09 handshake done sender: [4:7587241442980119062:2096] self: [4:7587241442980119084:0] peer: [2:7587241440608760643:0] socket: 475 qp: -1
2025-12-24T02:11:48.322302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241442980119097:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.322308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.319893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760658:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.321335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760668:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.321378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760669:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.322989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760657:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.323005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760669:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.323023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760668:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.323046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241440608760577:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.323056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241440608760578:2049] [node 3] ICP18 handshake (actor [2:7587241440608760669:2119]) is too old
2025-12-24T02:11:48.324039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241442449115542:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241442449115627:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.322574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784708:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.327237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784709:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.327285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784710:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.328853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659582:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.328910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659546:2097] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.328915Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241442049659546:2097] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.327591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241442449115542:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.327606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.327612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241442449115542:2049] [node 2] ICP22 created new session: [3:7587241442449115634:2049]
2025-12-24T02:11:48.327618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241442449115634:2049] [node 2] ICS09 handshake done sender: [3:7587241442449115627:2115] self: [3:7587241442449115629:0] peer: [2:7587241440608760619:0] socket: 488 qp: -1
2025-12-24T02:11:48.327625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241442449115634:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.327631Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.330855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696983:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.330913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696984:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.330933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696985:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.331226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241440608760617:2086] [node 4] ICP35 event processing took too much time PendingConnection:279(16731801ns) {HandleHandshakeStatus:329(16731195ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(175ns) SwitchToState:179(76ns) ProcessPendingSessionEvents:495(11322197ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11298826ns max# 11216230ns}}}
2025-12-24T02:11:48.331271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241440608760617:2086] [node 4] ICP18 handshake (actor [2:7587241440608760658:2116]) is too old
2025-12-24T02:11:48.331286Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241440608760672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41807
2025-12-24T02:11:48.331736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760647:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.331741Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241440608760647:2106] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.332082Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241440608760657:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.332240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119094:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.329651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241440918784688:2103] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.332510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241440918784638:2061] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.331739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241442049659554:2103] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.331849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241442049659546:2097] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.331995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241442049659494:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.332007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.332014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241442049659494:2049] [node 3] ICP22 created new session: [1:7587241442049659588:2049]
2025-12-24T02:11:48.332019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241442049659588:2049] [node 3] ICS09 handshake done sender: [1:7587241442049659554:2103] self: [1:7587241442049659553:0] peer: [3:7587241442449115631:0] socket: 460 qp: -1
2025-12-24T02:11:48.332024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241442049659588:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:48.332031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.332173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241442449115589:2090] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.332149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241440608760577:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.332162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.332166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241440608760577:2048] [node 1] ICP22 created new session: [2:7587241440608760673:2048]
2025-12-24T02:11:48.332171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241440608760673:2048] [node 1] ICS09 handshake done sender: [2:7587241440608760657:2115] self: [2:7587241440608760670:0] peer: [1:7587241442049659545:0] socket: 464 qp: -1
2025-12-24T02:11:48.332176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241440608760673:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.332181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.333772Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241442449115585:2088] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:48.333783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241442449115584:2085] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.333791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.333797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241442449115584:2085] [node 6] ICP22 created new session: [3:7587241442449115638:2085]
2025-12-24T02:11:48.333802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241442449115638:2085] [node 6] ICS09 handshake done sender: [3:7587241442449115589:2090] self: [3:7587241442449115588:0] peer: [6:7587241442966696982:0] socket: 449 qp: -1
2025-12-24T02:11:48.333808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241442449115638:2085] [node 6] ICS10 traffic start
2025-12-24T02:11:48.333813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.333557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241440918784709:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.333595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241440918784635:2059] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.333603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.333610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241440918784635:2059] [node 6] ICP22 created new session: [5:7587241440918784712:2059]
2025-12-24T02:11:48.333615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241440918784712:2059] [node 6] ICS09 handshake done sender: [5:7587241440918784638:2061] self: [5:7587241440918784637:0] peer: [6:7587241442966696981:0] socket: 444 qp: -1
2025-12-24T02:11:48.333620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241440918784712:2059] [node 6] ICS10 traffic start
2025-12-24T02:11:48.333627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.333826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241440918784622:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.333834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.333838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241440918784622:2048] [node 1] ICP22 created new session: [5:7587241440918784714:2048]
2025-12-24T02:11:48.333843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241440918784714:2048] [node 1] ICS09 handshake done sender: [5:7587241440918784688:2103] self: [5:7587241440918784707:0] peer: [1:7587241442049659557:0] socket: 477 qp: -1
2025-12-24T02:11:48.333847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241440918784714:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.333851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.334108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241440918784623:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.334580Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241442449115637:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:56074
2025-12-24T02:11:48.331599Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241442966696983:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.332835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241442966696919:2064] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.334687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241442966696902:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.334754Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241442966696903:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.334764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.334769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241442966696903:2050] [node 3] ICP22 created new session: [6:7587241442966696987:2050]
2025-12-24T02:11:48.334774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241442966696987:2050] [node 3] ICS09 handshake done sender: [6:7587241442966696919:2064] self: [6:7587241442966696982:0] peer: [3:7587241442449115588:0] socket: 454 qp: -1
2025-12-24T02:11:48.334778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241442966696987:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.334784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.334892Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241442449115628:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.334942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241442449115541:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.334954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.334959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241442449115541:2048] [node 1] ICP22 created new session: [3:7587241442449115640:2048]
2025-12-24T02:11:48.334964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241442449115640:2048] [node 1] ICS09 handshake done sender: [3:7587241442449115628:2116] self: [3:7587241442449115631:0] peer: [1:7587241442049659553:0] socket: 489 qp: -1
2025-12-24T02:11:48.334969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241442449115640:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.334974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.335337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241440608760655:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.335347Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241440608760655:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.335548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241440918784710:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.335573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241440918784643:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.336825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241442049659493:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.336841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.336848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241442049659493:2048] [node 2] ICP22 created new session: [1:7587241442049659590:2048]
2025-12-24T02:11:48.336853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241442049659590:2048] [node 2] ICS09 handshake done sender: [1:7587241442049659546:2097] self: [1:7587241442049659545:0] peer: [2:7587241440608760670:0] socket: 459 qp: -1
2025-12-24T02:11:48.336857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241442049659590:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:48.336865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.338267Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241442966696918:2063] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.338397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241442966696929:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.338406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.338413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241442966696929:2068] [node 5] ICP22 created new session: [6:7587241442966696989:2068]
2025-12-24T02:11:48.338418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241442966696989:2068] [node 5] ICS09 handshake done sender: [6:7587241442966696918:2063] self: [6:7587241442966696981:0] peer: [5:7587241440918784637:0] socket: 453 qp: -1
2025-12-24T02:11:48.338423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241442966696989:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:48.338438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.339253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784718:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.339414Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241440608760655:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.339609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241440918784709:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.340390Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119094:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.340425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241442980119005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.339839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241440918784623:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.339853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.339860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241440918784623:2049] [node 2] ICP22 created new session: [5:7587241440918784719:2049]
2025-12-24T02:11:48.339865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241440918784719:2049] [node 2] ICS09 handshake done sender: [5:7587241440918784709:2119] self: [5:7587241440918784716:0] peer: [2:7587241440608760639:0] socket: 485 qp: -1
2025-12-24T02:11:48.339871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241440918784719:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.339878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.341146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119088:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.341156Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241442980119088:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.341278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241440608760633:2097] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.341282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241440608760647:2106] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.341294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.341300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241440608760633:2097] [node 5] ICP22 created new session: [2:7587241440608760677:2097]
2025-12-24T02:11:48.341306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241440608760677:2097] [node 5] ICS09 handshake done sender: [2:7587241440608760655:2114] self: [2:7587241440608760639:0] peer: [5:7587241440918784716:0] socket: 501 qp: -1
2025-12-24T02:11:48.341311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241440608760677:2097] [node 5] ICS10 traffic start
2025-12-24T02:11:48.341327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.341558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659566:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.341566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241442049659566:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.342120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241442980119101:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42076
2025-12-24T02:11:48.342207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241442049659566:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.342380Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241442980119094:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.342429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241442980119005:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.342442Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.342448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241442980119005:2048] [node 1] ICP22 created new session: [4:7587241442980119102:2048]
2025-12-24T02:11:48.342453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241442980119102:2048] [node 1] ICS09 handshake done sender: [4:7587241442980119094:2120] self: [4:7587241442980119100:0] peer: [1:7587241442049659551:0] socket: 506 qp: -1
2025-12-24T02:11:48.342458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241442980119102:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.342464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.343331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241442049659543:2092] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.343348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.343353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241442049659543:2092] [node 4] ICP22 created new session: [1:7587241442049659594:2092]
2025-12-24T02:11:48.343358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241442049659594:2092] [node 4] ICS09 handshake done sender: [1:7587241442049659566:2110] self: [1:7587241442049659551:0] peer: [4:7587241442980119100:0] socket: 505 qp: -1
2025-12-24T02:11:48.343364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241442049659594:2092] [node 4] ICS10 traffic start
2025-12-24T02:11:48.343370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.344515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241440608760632:2094] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.344534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.344540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241440608760632:2094] [node 6] ICP22 created new session: [2:7587241440608760679:2094]
2025-12-24T02:11:48.344545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241440608760679:2094] [node 6] ICS09 handshake done sender: [2:7587241440608760647:2106] self: [2:7587241440608760635:0] peer: [6:7587241442966696986:0] socket: 500 qp: -1
2025-12-24T02:11:48.344550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241440608760679:2094] [node 6] ICS10 traffic start
2025-12-24T02:11:48.344555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.345319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241442966696902:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241442966696983:2117] held: no
2025-12-24T02:11:48.348373Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696991:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.348389Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696992:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.348398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696993:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.351938Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241442449115541:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16877722ns) {HandleHandshakeStatus:329(16877301ns) {DropIncomingHandshake:442(171ns) DropOutgoingHandshake:457(1331ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(16811569ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16788988ns max# 16710402ns}}}
2025-12-24T02:11:48.353019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241442966696983:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.353374Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241442966696991:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.353416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241442966696928:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.353558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241442980119095:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.353567Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241442980119095:2121] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.354221Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241442980119095:2121] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.354370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241442966696902:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.354386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.354391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241442966696902:2049] [node 2] ICP22 created new session: [6:7587241442966696996:2049]
2025-12-24T02:11:48.354396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241442966696996:2049] [node 2] ICS09 handshake done sender: [6:7587241442966696983:2117] self: [6:7587241442966696986:0] peer: [2:7587241440608760635:0] socket: 503 qp: -1
2025-12-24T02:11:48.354403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241442966696996:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.354436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.354760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966696995:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.354895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241442980119051:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.354915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.354921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241442980119051:2079] [node 6] ICP22 created new session: [4:7587241442980119105:2079]
2025-12-24T02:11:48.354926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241442980119105:2079] [node 6] ICS09 handshake done sender: [4:7587241442980119095:2121] self: [4:7587241442980119086:0] peer: [6:7587241442966696994:0] socket: 445 qp: -1
2025-12-24T02:11:48.354931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241442980119105:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:48.354950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.356433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241442966696992:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.356456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241442966696901:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.356629Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241442049659577:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.356646Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241442049659577:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.358541Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241442966696903:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(23675381ns) {HandleHandshakeStatus:329(23674943ns) {DropIncomingHandshake:442(239ns) DropOutgoingHandshake:457(1044ns) SwitchToState:179(37ns) ProcessPendingSessionEvents:495(17190761ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17168660ns max# 17095906ns}}}
2025-12-24T02:11:48.358561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241442966696991:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.361134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241442980119088:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.361322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241442966696928:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.361339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.361346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241442966696928:2066] [node 4] ICP22 created new session: [6:7587241442966696999:2066]
2025-12-24T02:11:48.361353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241442966696999:2066] [node 4] ICS09 handshake done sender: [6:7587241442966696991:2120] self: [6:7587241442966696994:0] peer: [4:7587241442980119086:0] socket: 514 qp: -1
2025-12-24T02:11:48.361359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241442966696999:2066] [node 4] ICS10 traffic start
2025-12-24T02:11:48.361367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.361869Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241442966696928:2066] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:48.361364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241442980119036:2074] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.361385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.361392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241442980119036:2074] [node 5] ICP22 created new session: [4:7587241442980119108:2074]
2025-12-24T02:11:48.361398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241442980119108:2074] [node 5] ICS09 handshake done sender: [4:7587241442980119088:2118] self: [4:7587241442980119081:0] peer: [5:7587241440918784717:0] socket: 504 qp: -1
2025-12-24T02:11:48.361404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241442980119108:2074] [node 5] ICS10 traffic start
2025-12-24T02:11:48.361410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.363803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241440608760633:2097] [node 5] ICP35 event processing took too much time PendingConnection:279(22415683ns) {HandleHandshakeStatus:329(22415226ns) {DropIncomingHandshake:442(3027ns) DropOutgoingHandshake:457(130ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(5045803ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 5027111ns max# 4951220ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.366806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241442966697001:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40304
2025-12-24T02:11:48.364228Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241440918784643:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241440918784710:2120] held: no
2025-12-24T02:11:48.365398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241440918784721:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30781
2025-12-24T02:11:48.365844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241440918784710:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.365934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241440918784643:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.365942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.365949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241440918784643:2063] [node 4] ICP22 created new session: [5:7587241440918784722:2063]
2025-12-24T02:11:48.365955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241440918784722:2063] [node 4] ICS09 handshake done sender: [5:7587241440918784710:2120] self: [5:7587241440918784717:0] peer: [4:7587241442980119081:0] socket: 508 qp: -1
2025-12-24T02:11:48.365961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241440918784722:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:48.365968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.367173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241442966696992:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.366829Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241442049659577:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.367491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241442049659559:2100] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.367509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.367515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241442049659559:2100] [node 6] ICP22 created new session: [1:7587241442049659597:2100]
2025-12-24T02:11:48.367521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241442049659597:2100] [node 6] ICS09 handshake done sender: [1:7587241442049659577:2118] self: [1:7587241442049659567:0] peer: [6:7587241442966696998:0] socket: 516 qp: -1
2025-12-24T02:11:48.367526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241442049659597:2100] [node 6] ICS10 traffic start
2025-12-24T02:11:48.367533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.367282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241442966696901:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.367299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.367305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241442966696901:2048] [node 1] ICP22 created new session: [6:7587241442966697002:2048]
2025-12-24T02:11:48.367312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241442966697002:2048] [node 1] ICS09 handshake done sender: [6:7587241442966696992:2121] self: [6:7587241442966696998:0] peer: [1:7587241442049659567:0] socket: 478 qp: -1
2025-12-24T02:11:48.367318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241442966697002:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.367328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.396790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:48.396905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241442449115640:2048] [node 1] ICS07 socket disconnect 489 reason# EndOfStream
2025-12-24T02:11:48.396913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241442449115640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.396944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.396961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241442449115640:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:48.398870Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:48.398955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241440608760673:2048] [node 1] ICS01 socket: 464 reason# EndOfStream
2025-12-24T02:11:48.398963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241440608760577:2048] [node 1] ICP30 unregister session Session# [2:7587241440608760673:2048] VirtualId# [2:7587241440608760670:0]
2025-12-24T02:11:48.398970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241440608760673:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.399007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.399037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.399531Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241442449115541:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24359) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:48.399543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241442449115640:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:48.399824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241442449115541:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24359) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.399846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241442449115640:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.399851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241442449115541:2048] [node 1] ICP30 unregister session Session# [3:7587241442449115640:2048] VirtualId# [3:7587241442449115631:0]
2025-12-24T02:11:48.399906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.399912Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241442449115541:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:24359) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.399915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:24359) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.401661Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:48.401734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241442980119102:2048] [node 1] ICS01 socket: 506 reason# EndOfStream
2025-12-24T02:11:48.401740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241442980119005:2048] [node 1] ICP30 unregister session Session# [4:7587241442980119102:2048] VirtualId# [4:7587241442980119100:0]
2025-12-24T02:11:48.401745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241442980119102:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.401780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.401799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.401831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:48.401901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241440918784714:2048] [node 1] ICS01 socket: 477 reason# EndOfStream
2025-12-24T02:11:48.401906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241440918784622:2048] [node 1] ICP30 unregister session Session# [5:7587241440918784714:2048] VirtualId# [5:7587241440918784707:0]
2025-12-24T02:11:48.401911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241440918784714:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.401934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.401952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.402890Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:48.402965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241442966697002:2048] [node 1] ICS01 socket: 478 reason# EndOfStream
2025-12-24T02:11:48.402972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241442966696901:2048] [node 1] ICP30 unregister session Session# [6:7587241442966697002:2048] VirtualId# [6:7587241442966696998:0]
2025-12-24T02:11:48.402978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241442966697002:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.403016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.403037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.409588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241442449115541:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:48.415989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:48.416071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241442449115634:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:48.416079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241442449115542:2049] [node 2] ICP30 unregister session Session# [3:7587241442449115634:2049] VirtualId# [3:7587241442449115629:0]
2025-12-24T02:11:48.416085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241442449115634:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.416120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.416140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.420670Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:48.420756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241442980119097:2049] [node 2] ICS07 socket disconnect 475 reason# EndOfStream
2025-12-24T02:11:48.420762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241442980119097:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.420799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.420819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241442980119097:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:48.421027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:48.421096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241442966696996:2049] [node 2] ICS01 socket: 503 reason# EndOfStream
2025-12-24T02:11:48.421101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241442966696902:2049] [node 2] ICP30 unregister session Session# [6:7587241442966696996:2049] VirtualId# [6:7587241442966696986:0]
2025-12-24T02:11:48.421106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241442966696996:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.421130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.421151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.421182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:48.421247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241440918784719:2049] [node 2] ICS01 socket: 485 reason# EndOfStream
2025-12-24T02:11:48.421251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241440918784623:2049] [node 2] ICP30 unregister session Session# [5:7587241440918784719:2049] VirtualId# [5:7587241440918784716:0]
2025-12-24T02:11:48.421256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241440918784719:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.421278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.421296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.421812Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241442980119006:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20387) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.421823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241442980119097:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.421827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241442980119006:2049] [node 2] ICP30 unregister session Session# [4:7587241442980119097:2049] VirtualId# [4:7587241442980119084:0]
2025-12-24T02:11:48.421871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.421878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241442980119006:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.421881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20387) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.434741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241442980119006:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:48.441388Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:48.441496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241442966696987:2050] [node 3] ICS07 socket disconnect 454 reason# EndOfStream
2025-12-24T02:11:48.441504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241442966696987:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.441554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.441576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241442966696987:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:48.443075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:48.443140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241440918784704:2050] [node 3] ICS01 socket: 452 reason# EndOfStream
2025-12-24T02:11:48.443148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241440918784624:2050] [node 3] ICP30 unregister session Session# [5:7587241440918784704:2050] VirtualId# [5:7587241440918784702:0]
2025-12-24T02:11:48.443156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241440918784704:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.443199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.443223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.443885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241442966696903:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2779) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.443899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241442966696987:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.443904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241442966696903:2050] [node 3] ICP30 unregister session Session# [6:7587241442966696987:2050] VirtualId# [6:7587241442966696982:0]
2025-12-24T02:11:48.443955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.443962Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241442966696903:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:2779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.443965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2779) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.453689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241442966696903:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:48.458628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:48.458738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241442966696999:2066] [node 4] ICS07 socket disconnect 514 reason# EndOfStream
2025-12-24T02:11:48.458745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241442966696999:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.458787Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:48.458805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241442966696999:2066] [node 4] ICS15 start handshake
2025-12-24T02:11:48.459267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241442966696928:2066] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.459282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241442966696999:2066] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.459287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241442966696928:2066] [node 4] ICP30 unregister session Session# [6:7587241442966696999:2066] VirtualId# [6:7587241442966696994:0]
2025-12-24T02:11:48.459338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:48.459345Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241442966696928:2066] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.459347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23716) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.473867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241442966696928:2066] [node 4] ICP33 wake up from error state
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:48.522327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443530017248:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:48.522335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443530017249:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.530822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441113498655:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.530835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441113498656:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.527130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440393728219:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.527139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440393728220:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.536783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444152493020:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.536790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444152493021:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.536795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444152493022:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.544633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241441712103505:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.544640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241441712103506:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.544646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241441712103507:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.535063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241441243680919:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.535072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241441243680920:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.535076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241441243680921:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.566122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441113498702:2089] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.566711Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728267:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.566900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443530017307:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:48.566248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441113498703:2093] [node 6] ICP01 ready to work
2025-12-24T02:11:48.571343Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241441243680971:2096] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.576911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103548:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.587251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440393728291:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:48.606928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241441243680921:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36047946ns) {Configure:104(36046958ns) {ProcessConfigured:135(35654667ns) {StartInitialHandshake:157(35653626ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(2585ns) {SwitchToState:179(1139ns)}}}}}
2025-12-24T02:11:48.608023Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241443530017249:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40922678ns) {Configure:104(36902407ns) {ProcessConfigured:135(36573193ns) {StartInitialHandshake:157(36573008ns) {DropHandshakes:470(483ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(83ns)} GenerateSessionVirtualId:988(271ns) PrepareNewSessionHandshake:477(1131ns) {SwitchToState:179(564ns)}}}}}
2025-12-24T02:11:48.608787Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241441113498702:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(42478158ns) {Configure:104(42477284ns) {ProcessConfigured:135(36419522ns) {StartInitialHandshake:157(36419257ns) {DropHandshakes:470(640ns) {DropIncomingHandshake:442(216ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(703ns) {SwitchToState:179(383ns)}}}}}
2025-12-24T02:11:48.608325Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241441243680971:2096] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(36925014ns) {Configure:104(36924708ns) {ProcessConfigured:135(36662091ns) {StartInitialHandshake:157(36661879ns) {DropHandshakes:470(224ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(74ns) PrepareNewSessionHandshake:477(777ns) {SwitchToState:179(305ns)}}}}}
2025-12-24T02:11:48.609591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241441243680966:2091] [node 5] ICP01 ready to work
2025-12-24T02:11:48.608571Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241441712103507:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(36586561ns) {Configure:104(36585588ns) {ProcessConfigured:135(36249678ns) {StartInitialHandshake:157(36249184ns) {DropHandshakes:470(365ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1045ns) {SwitchToState:179(292ns)}}}}}
2025-12-24T02:11:48.611105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443530017311:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.619066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241443530017326:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9067
2025-12-24T02:11:48.619089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241443530017327:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9067
2025-12-24T02:11:48.619111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241443530017324:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9067
2025-12-24T02:11:48.619131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241443530017325:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9067
2025-12-24T02:11:48.619340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498731:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
2025-12-24T02:11:48.619361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498732:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
2025-12-24T02:11:48.619372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498733:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
2025-12-24T02:11:48.619386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498734:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
2025-12-24T02:11:48.619398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498735:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.624406Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728295:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.624430Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.624462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728293:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.624486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728294:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.625480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444152493089:2099] [node 6] ICP01 ready to work
2025-12-24T02:11:48.626082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441113498728:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:48.626562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443530017320:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:48.627821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440393728286:2097] [node 5] ICP01 ready to work
2025-12-24T02:11:48.628913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440393728290:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:48.628077Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498732:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.627018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241441712103569:2068] [node 5] ICP01 ready to work
2025-12-24T02:11:48.627506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241441712103570:2065] [node 4] ICP01 ready to work
2025-12-24T02:11:48.628406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444152493092:2103] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.654975Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241441243680966:2091] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45239310ns) {Configure:104(45238270ns) {ProcessConfigured:135(44842542ns) {StartInitialHandshake:157(44841482ns) {DropHandshakes:470(1470ns) {DropIncomingHandshake:442(201ns) DropOutgoingHandshake:457(440ns)} GenerateSessionVirtualId:988(451ns) PrepareNewSessionHandshake:477(2047ns) {SwitchToState:179(1472ns)}}}}}
2025-12-24T02:11:48.655191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243680997:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.658855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493096:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.658957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241443530017311:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(47734520ns) {Configure:104(47734041ns) {ProcessConfigured:135(47378844ns) {StartInitialHandshake:157(47378004ns) {DropHandshakes:470(565ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(489ns) PrepareNewSessionHandshake:477(2178ns) {SwitchToState:179(1608ns)}}}}}
2025-12-24T02:11:48.663006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498733:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.664035Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241441113498728:2097] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(37810912ns) {Configure:104(37810248ns) {ProcessConfigured:135(37459331ns) {StartInitialHandshake:157(37458989ns) {DropHandshakes:470(307ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(814ns) {SwitchToState:179(71ns)}}}}}
2025-12-24T02:11:48.666686Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241441712103569:2068] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39534442ns) {Configure:104(39533850ns) {ProcessConfigured:135(39158853ns) {StartInitialHandshake:157(39158565ns) {DropHandshakes:470(222ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(983ns) {SwitchToState:179(240ns)}}}}}
2025-12-24T02:11:48.666824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241441712103570:2065] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39206948ns) {Configure:104(39206689ns) {ProcessConfigured:135(38985469ns) {StartInitialHandshake:157(38985179ns) {DropHandshakes:470(285ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(45ns) PrepareNewSessionHandshake:477(652ns) {SwitchToState:179(337ns)}}}}}
2025-12-24T02:11:48.667094Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241440393728286:2097] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39027254ns) {Configure:104(39026879ns) {ProcessConfigured:135(38675322ns) {StartInitialHandshake:157(38675141ns) {DropHandshakes:470(203ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(37ns)} GenerateSessionVirtualId:988(155ns) PrepareNewSessionHandshake:477(863ns) {SwitchToState:179(178ns)}}}}}
2025-12-24T02:11:48.667350Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243680997:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.667394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241441243680919:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.663929Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241444152493089:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38280274ns) {Configure:104(38278881ns) {ProcessConfigured:135(37845768ns) {StartInitialHandshake:157(37844868ns) {DropHandshakes:470(1510ns) {DropIncomingHandshake:442(306ns) DropOutgoingHandshake:457(220ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(1179ns) {SwitchToState:179(861ns)}}}}}
2025-12-24T02:11:48.667319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241444152493092:2103] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(38790466ns) {Configure:104(38790176ns) {ProcessConfigured:135(38487819ns) {StartInitialHandshake:157(38487685ns) {DropHandshakes:470(269ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(24ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(920ns) {SwitchToState:179(581ns)}}}}}
2025-12-24T02:11:48.664124Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241443530017320:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(37426650ns) {Configure:104(37426380ns) {ProcessConfigured:135(37091549ns) {StartInitialHandshake:157(37091258ns) {DropHandshakes:470(236ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(694ns) PrepareNewSessionHandshake:477(1225ns) {SwitchToState:179(558ns)}}}}}
2025-12-24T02:11:48.667215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017324:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.667239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017326:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.667663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017313:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.667667Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241443530017313:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.668120Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017321:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.668123Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241443530017321:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.667430Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241440393728290:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(38406058ns) {Configure:104(38405766ns) {ProcessConfigured:135(38157339ns) {StartInitialHandshake:157(38157256ns) {DropHandshakes:470(197ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(406ns) {SwitchToState:179(186ns)}}}}}
2025-12-24T02:11:48.667469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728295:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.667578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241440393728219:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.675025Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241443530017313:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.675106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241443530017321:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.674799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241441113498655:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(107902583ns) {Configure:104(107901278ns) {ProcessConfigured:135(102356869ns) {StartInitialHandshake:157(102356182ns) {DropHandshakes:470(369ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(13ns)} GenerateSessionVirtualId:988(179ns) PrepareNewSessionHandshake:477(1342ns) {SwitchToState:179(604ns)}}}}}
2025-12-24T02:11:48.682068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498734:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.682123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241441113498655:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.682490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017315:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.682500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241443530017315:2109] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.680917Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241440393728291:2104] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(93512383ns) {Configure:104(93511148ns) {ProcessConfigured:135(93084982ns) {StartInitialHandshake:157(93084429ns) {DropHandshakes:470(1094ns) {DropIncomingHandshake:442(209ns) DropOutgoingHandshake:457(286ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1870ns) {SwitchToState:179(1116ns)}}}}}
2025-12-24T02:11:48.682321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728293:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.682353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241440393728220:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.683291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498723:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.683298Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241441113498723:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.683714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241443530017315:2109] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.685412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493099:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.685526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728303:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.685914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241440393728295:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.686151Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493096:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.686175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493099:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.686209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444152493021:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.686220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444152493022:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.686994Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498729:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.687003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241441113498729:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.686377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728299:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.686383Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241440393728299:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.687872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017327:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.688273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241441712103548:2085] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.688316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241441712103506:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.688061Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441113498729:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.687738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241440393728299:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.687773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241440393728219:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.687790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.687799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440393728219:2048] [node 1] ICP22 created new session: [3:7587241440393728307:2048]
2025-12-24T02:11:48.687807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241440393728307:2048] [node 1] ICS09 handshake done sender: [3:7587241440393728295:2115] self: [3:7587241440393728302:0] peer: [1:7587241443530017312:0] socket: 488 qp: -1
2025-12-24T02:11:48.687814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241440393728307:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.687822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.688879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443530017248:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.688898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.688906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443530017248:2048] [node 2] ICP22 created new session: [1:7587241443530017337:2048]
2025-12-24T02:11:48.688913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241443530017337:2048] [node 2] ICS09 handshake done sender: [1:7587241443530017315:2109] self: [1:7587241443530017314:0] peer: [2:7587241441113498741:0] socket: 467 qp: -1
2025-12-24T02:11:48.688919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241443530017337:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:48.688937Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.690327Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728296:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.690879Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103585:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.690535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241440393728286:2097] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.690547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.690553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440393728286:2097] [node 5] ICP22 created new session: [3:7587241440393728311:2097]
2025-12-24T02:11:48.690559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241440393728311:2097] [node 5] ICS09 handshake done sender: [3:7587241440393728299:2117] self: [3:7587241440393728297:0] peer: [5:7587241444152493101:0] socket: 490 qp: -1
2025-12-24T02:11:48.690565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241440393728311:2097] [node 5] ICS10 traffic start
2025-12-24T02:11:48.690570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.692342Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493103:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.693695Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.693737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444152493096:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.693820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498721:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.693827Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241441113498721:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.694424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103586:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.694685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241441712103585:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.694515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444152493021:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.694540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.694545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444152493021:2049] [node 2] ICP22 created new session: [5:7587241444152493105:2049]
2025-12-24T02:11:48.694551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444152493105:2049] [node 2] ICS09 handshake done sender: [5:7587241444152493096:2113] self: [5:7587241444152493100:0] peer: [2:7587241441113498712:0] socket: 479 qp: -1
2025-12-24T02:11:48.694789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241441712103507:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.694556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444152493105:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.694573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.694813Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493102:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.694979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728301:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.694988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241440393728301:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.695203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441113498721:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.695298Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241441712103548:2085] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.695401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241441712103506:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.695419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.695425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241441712103506:2049] [node 2] ICP22 created new session: [6:7587241441712103588:2049]
2025-12-24T02:11:48.695431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241441712103588:2049] [node 2] ICS09 handshake done sender: [6:7587241441712103548:2085] self: [6:7587241441712103584:0] peer: [2:7587241441113498720:0] socket: 469 qp: -1
2025-12-24T02:11:48.695436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241441712103588:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.695471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.697960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441113498703:2093] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.697979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.697985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441113498703:2093] [node 6] ICP22 created new session: [2:7587241441113498745:2093]
2025-12-24T02:11:48.697991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241441113498745:2093] [node 6] ICS09 handshake done sender: [2:7587241441113498721:2108] self: [2:7587241441113498720:0] peer: [6:7587241441712103584:0] socket: 463 qp: -1
2025-12-24T02:11:48.697996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241441113498745:2093] [node 6] ICS10 traffic start
2025-12-24T02:11:48.698002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.698792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443530017249:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.698817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.698823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443530017249:2049] [node 3] ICP22 created new session: [1:7587241443530017340:2049]
2025-12-24T02:11:48.698829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241443530017340:2049] [node 3] ICS09 handshake done sender: [1:7587241443530017321:2112] self: [1:7587241443530017312:0] peer: [3:7587241440393728302:0] socket: 476 qp: -1
2025-12-24T02:11:48.698834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241443530017340:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:48.698840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.698983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443530017307:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.698997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.699004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443530017307:2104] [node 4] ICP22 created new session: [1:7587241443530017343:2104]
2025-12-24T02:11:48.699011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241443530017343:2104] [node 4] ICS09 handshake done sender: [1:7587241443530017313:2108] self: [1:7587241443530017308:0] peer: [4:7587241441243680998:0] socket: 466 qp: -1
2025-12-24T02:11:48.699017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241443530017343:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:48.699035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.699363Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.699656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493104:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.699721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444152493099:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.700039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241441243680919:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27427) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241441243680997:2115] held: no
2025-12-24T02:11:48.701074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444152493020:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.703329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493107:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.703536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.703732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493107:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.704974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444152493022:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.704995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.705003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444152493022:2050] [node 3] ICP22 created new session: [5:7587241444152493110:2050]
2025-12-24T02:11:48.705008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444152493110:2050] [node 3] ICS09 handshake done sender: [5:7587241444152493099:2116] self: [5:7587241444152493101:0] peer: [3:7587241440393728297:0] socket: 491 qp: -1
2025-12-24T02:11:48.705015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444152493110:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.705034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.705680Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017329:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.705690Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241443530017329:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.706168Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241441712103591:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.706217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241441712103505:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-12-24T02:11:48.707831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498735:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:48.709507Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017330:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.709519Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241443530017330:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.709584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241441113498749:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:50422
2025-12-24T02:11:48.709789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441113498723:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.709875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441113498656:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.709895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.709901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441113498656:2049] [node 3] ICP22 created new session: [2:7587241441113498751:2049]
2025-12-24T02:11:48.709911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241441113498751:2049] [node 3] ICS09 handshake done sender: [2:7587241441113498723:2109] self: [2:7587241441113498722:0] peer: [3:7587241440393728305:0] socket: 464 qp: -1
2025-12-24T02:11:48.709917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241441113498751:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:48.709924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.710155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441113498734:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.710381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241441113498655:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.710396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.710402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441113498655:2048] [node 1] ICP22 created new session: [2:7587241441113498753:2048]
2025-12-24T02:11:48.710408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241441113498753:2048] [node 1] ICS09 handshake done sender: [2:7587241441113498734:2116] self: [2:7587241441113498741:0] peer: [1:7587241443530017314:0] socket: 485 qp: -1
2025-12-24T02:11:48.710412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241441113498753:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.710418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.710808Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241440393728286:2097] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:48.710861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728294:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.711953Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241440393728220:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241440393728293:2113] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.712265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103594:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.712473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241440393728301:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.712618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103595:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.712800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241441712103585:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.712936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241441712103507:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.712957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.712964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241441712103507:2050] [node 3] ICP22 created new session: [6:7587241441712103596:2050]
2025-12-24T02:11:48.712970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241441712103596:2050] [node 3] ICS09 handshake done sender: [6:7587241441712103585:2116] self: [6:7587241441712103587:0] peer: [3:7587241440393728298:0] socket: 502 qp: -1
2025-12-24T02:11:48.712976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241441712103596:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.712985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.712744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241443530017345:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9067
2025-12-24T02:11:48.713387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241443530017329:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.713150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241440393728290:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.713166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.713170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440393728290:2103] [node 6] ICP22 created new session: [3:7587241440393728316:2103]
2025-12-24T02:11:48.713175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241440393728316:2103] [node 6] ICS09 handshake done sender: [3:7587241440393728301:2118] self: [3:7587241440393728298:0] peer: [6:7587241441712103587:0] socket: 501 qp: -1
2025-12-24T02:11:48.713179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241440393728316:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:48.713185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.713767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241441712103592:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.713793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241441712103569:2068] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.713713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443530017311:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.713725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.713732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443530017311:2106] [node 5] ICP22 created new session: [1:7587241443530017347:2106]
2025-12-24T02:11:48.713738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241443530017347:2106] [node 5] ICS09 handshake done sender: [1:7587241443530017329:2117] self: [1:7587241443530017323:0] peer: [5:7587241444152493108:0] socket: 500 qp: -1
2025-12-24T02:11:48.713744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241443530017347:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:48.713750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.714465Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241443530017325:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.714489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241443530017311:2106] [node 5] ICP18 handshake (actor [1:7587241443530017325:2114]) is too old
2025-12-24T02:11:48.714319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493097:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.714330Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241444152493097:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.714899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441113498702:2089] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.714914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.714922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441113498702:2089] [node 5] ICP22 created new session: [2:7587241441113498755:2089]
2025-12-24T02:11:48.714928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241441113498755:2089] [node 5] ICS09 handshake done sender: [2:7587241441113498729:2112] self: [2:7587241441113498712:0] peer: [5:7587241444152493100:0] socket: 478 qp: -1
2025-12-24T02:11:48.714933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241441113498755:2089] [node 5] ICS10 traffic start
2025-12-24T02:11:48.714943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.715225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444152493097:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.715315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241444152493089:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.715329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.715333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444152493089:2099] [node 6] ICP22 created new session: [5:7587241444152493115:2099]
2025-12-24T02:11:48.715339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444152493115:2099] [node 6] ICS09 handshake done sender: [5:7587241444152493097:2114] self: [5:7587241444152493094:0] peer: [6:7587241441712103598:0] socket: 450 qp: -1
2025-12-24T02:11:48.715342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444152493115:2099] [node 6] ICS10 traffic start
2025-12-24T02:11:48.715348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.718894Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103599:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.719986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241441712103592:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.719827Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241440393728318:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8182
2025-12-24T02:11:48.720335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241441712103569:2068] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.720351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.720359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241441712103569:2068] [node 5] ICP22 created new session: [6:7587241441712103600:2068]
2025-12-24T02:11:48.720365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241441712103600:2068] [node 5] ICS09 handshake done sender: [6:7587241441712103592:2119] self: [6:7587241441712103598:0] peer: [5:7587241444152493094:0] socket: 484 qp: -1
2025-12-24T02:11:48.720371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241441712103600:2068] [node 5] ICS10 traffic start
2025-12-24T02:11:48.720405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.721837Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241443530017330:2118] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.721950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443530017320:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.721968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.721976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443530017320:2107] [node 6] ICP22 created new session: [1:7587241443530017350:2107]
2025-12-24T02:11:48.721982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241443530017350:2107] [node 6] ICS09 handshake done sender: [1:7587241443530017330:2118] self: [1:7587241443530017328:0] peer: [6:7587241441712103593:0] socket: 460 qp: -1
2025-12-24T02:11:48.721987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241443530017350:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:48.721994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.724249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681000:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.721915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103603:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.723708Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493117:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.724499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444152493104:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.727037Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241440393728293:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.730924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681001:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.730972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681002:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.731005Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681003:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.731035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681004:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.731067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241441243680997:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.730909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241441712103591:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:48.728365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444152493020:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.728384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.728391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444152493020:2048] [node 1] ICP22 created new session: [5:7587241444152493118:2048]
2025-12-24T02:11:48.728401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444152493118:2048] [node 1] ICS09 handshake done sender: [5:7587241444152493104:2119] self: [5:7587241444152493108:0] peer: [1:7587241443530017323:0] socket: 506 qp: -1
2025-12-24T02:11:48.728407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444152493118:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.728413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.732142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241440393728220:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.732158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.732165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440393728220:2049] [node 2] ICP22 created new session: [3:7587241440393728319:2049]
2025-12-24T02:11:48.732181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241440393728319:2049] [node 2] ICS09 handshake done sender: [3:7587241440393728293:2113] self: [3:7587241440393728305:0] peer: [2:7587241441113498722:0] socket: 472 qp: -1
2025-12-24T02:11:48.732188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241440393728319:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.732196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.732266Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441113498739:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.732277Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241441113498739:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.733412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441113498739:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.733586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441113498728:2097] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.733604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.733610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441113498728:2097] [node 4] ICP22 created new session: [2:7587241441113498758:2097]
2025-12-24T02:11:48.733616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241441113498758:2097] [node 4] ICS09 handshake done sender: [2:7587241441113498739:2118] self: [2:7587241441113498736:0] peer: [4:7587241441243681005:0] socket: 508 qp: -1
2025-12-24T02:11:48.733621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241441113498758:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:48.733627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.731951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241441712103595:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.733259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241441712103570:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.734874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241441712103605:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30735
2025-12-24T02:11:48.732021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243681002:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.732043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243681001:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.732061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243681004:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.732095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241441243680920:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.732106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241441243680921:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.733683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243680991:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.733687Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241441243680991:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.734386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681008:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
2025-12-24T02:11:48.734499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241441243680991:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:48.734566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241441243680971:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.734581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:48.734587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241441243680971:2096] [node 6] ICP22 created new session: [4:7587241441243681010:2096]
2025-12-24T02:11:48.734593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241441243681010:2096] [node 6] ICS09 handshake done sender: [4:7587241441243680991:2110] self: [4:7587241441243680972:0] peer: [6:7587241441712103604:0] socket: 465 qp: -1
2025-12-24T02:11:48.734598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241441243681010:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:48.734603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:48.735262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243681003:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.735276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241441243680919:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.735290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.735296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241441243680919:2048] [node 1] ICP22 created new session: [4:7587241441243681013:2048]
2025-12-24T02:11:48.735302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241441243681013:2048] [node 1] ICS09 handshake done sender: [4:7587241441243680997:2115] self: [4:7587241441243680998:0] peer: [1:7587241443530017308:0] socket: 473 qp: -1
2025-12-24T02:11:48.735306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241441243681013:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.735321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.735939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241441712103505:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.735959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:48.735965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241441712103505:2048] [node 1] ICP22 created new session: [6:7587241441712103606:2048]
2025-12-24T02:11:48.735972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241441712103606:2048] [node 1] ICS09 handshake done sender: [6:7587241441712103591:2118] self: [6:7587241441712103593:0] peer: [1:7587241443530017328:0] socket: 510 qp: -1
2025-12-24T02:11:48.735978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241441712103606:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:48.735988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:48.737287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241441712103570:2065] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7540) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241441712103595:2121] held: no
2025-12-24T02:11:48.737387Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241441712103595:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.737053Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241441243680920:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241441243681002:2118] held: no
2025-12-24T02:11:48.735609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241440393728304:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.735620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241440393728304:2120] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:48.737889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241441243681002:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:48.737941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241441243680920:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.737949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:48.737955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241441243680920:2049] [node 2] ICP22 created new session: [4:7587241441243681016:2049]
2025-12-24T02:11:48.737962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241441243681016:2049] [node 2] ICS09 handshake done sender: [4:7587241441243681002:2118] self: [4:7587241441243681005:0] peer: [2:7587241441113498736:0] socket: 482 qp: -1
2025-12-24T02:11:48.737967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241441243681016:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:48.737982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:48.741668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241441712103570:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.741685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.741694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241441712103570:2065] [node 4] ICP22 created new session: [6:7587241441712103608:2065]
2025-12-24T02:11:48.741701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241441712103608:2065] [node 4] ICS09 handshake done sender: [6:7587241441712103595:2121] self: [6:7587241441712103604:0] peer: [4:7587241441243680972:0] socket: 514 qp: -1
2025-12-24T02:11:48.741707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241441712103608:2065] [node 4] ICS10 traffic start
2025-12-24T02:11:48.741717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.743295Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241441113498702:2089] [node 5] ICP35 event processing took too much time PendingConnection:279(54934479ns) {HandleHandshakeStatus:329(54933938ns) {DropIncomingHandshake:442(381ns) DropOutgoingHandshake:457(195ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(15997910ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15973400ns max# 15902945ns}}}
2025-12-24T02:11:48.743321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241441113498702:2089] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.748327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241441243681018:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29725
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.748499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493120:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.748828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444152493120:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.748865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444152493092:2103] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:48.749017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241441243680996:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:48.749026Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241441243680996:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.750463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241441243680921:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241441243681001:2117] held: no
2025-12-24T02:11:48.750985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241441243680996:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:48.751100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241443530017248:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:48.747987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241440393728304:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.748692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241440393728291:2104] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.748714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.748719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440393728291:2104] [node 4] ICP22 created new session: [3:7587241440393728322:2104]
2025-12-24T02:11:48.748725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241440393728322:2104] [node 4] ICS09 handshake done sender: [3:7587241440393728304:2120] self: [3:7587241440393728292:0] peer: [4:7587241441243681006:0] socket: 503 qp: -1
2025-12-24T02:11:48.748730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241440393728322:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:48.748737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.751091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241441243680966:2091] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:48.751108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:48.751113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241441243680966:2091] [node 5] ICP22 created new session: [4:7587241441243681020:2091]
2025-12-24T02:11:48.751119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241441243681020:2091] [node 5] ICS09 handshake done sender: [4:7587241441243680996:2114] self: [4:7587241441243680992:0] peer: [5:7587241444152493121:0] socket: 524 qp: -1
2025-12-24T02:11:48.751125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241441243681020:2091] [node 5] ICS10 traffic start
2025-12-24T02:11:48.751132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:48.753266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241441243681001:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:48.753358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241441243680921:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.753370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:48.753375Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241441243680921:2050] [node 3] ICP22 created new session: [4:7587241441243681022:2050]
2025-12-24T02:11:48.753381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241441243681022:2050] [node 3] ICS09 handshake done sender: [4:7587241441243681001:2117] self: [4:7587241441243681006:0] peer: [3:7587241440393728292:0] socket: 517 qp: -1
2025-12-24T02:11:48.753387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241441243681022:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:48.753394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:48.750841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444152493122:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44401
2025-12-24T02:11:48.751599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444152493120:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:48.751669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444152493092:2103] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:48.751677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:48.751682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444152493092:2103] [node 4] ICP22 created new session: [5:7587241444152493123:2103]
2025-12-24T02:11:48.751687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444152493123:2103] [node 4] ICS09 handshake done sender: [5:7587241444152493120:2122] self: [5:7587241444152493121:0] peer: [4:7587241441243680992:0] socket: 486 qp: -1
2025-12-24T02:11:48.751692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444152493123:2103] [node 4] ICS10 traffic start
2025-12-24T02:11:48.751708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:48.780428Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:48.780538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241441113498753:2048] [node 1] ICS01 socket: 485 reason# EndOfStream
2025-12-24T02:11:48.780549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241441113498655:2048] [node 1] ICP30 unregister session Session# [2:7587241441113498753:2048] VirtualId# [2:7587241441113498741:0]
2025-12-24T02:11:48.780558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241441113498753:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.780597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.780621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.781138Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:48.781212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241440393728307:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:48.781218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241440393728219:2048] [node 1] ICP30 unregister session Session# [3:7587241440393728307:2048] VirtualId# [3:7587241440393728302:0]
2025-12-24T02:11:48.781224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241440393728307:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.781254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.781272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.791158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:48.791267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241441712103606:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-12-24T02:11:48.791277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241441712103505:2048] [node 1] ICP30 unregister session Session# [6:7587241441712103606:2048] VirtualId# [6:7587241441712103593:0]
2025-12-24T02:11:48.791285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241441712103606:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.791330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.791354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.792219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:48.792305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241441243681013:2048] [node 1] ICS01 socket: 473 reason# EndOfStream
2025-12-24T02:11:48.792313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241441243680919:2048] [node 1] ICP30 unregister session Session# [4:7587241441243681013:2048] VirtualId# [4:7587241441243680998:0]
2025-12-24T02:11:48.792319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241441243681013:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.792372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.792401Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.792440Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:48.792511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444152493118:2048] [node 1] ICS01 socket: 506 reason# EndOfStream
2025-12-24T02:11:48.792517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444152493020:2048] [node 1] ICP30 unregister session Session# [5:7587241444152493118:2048] VirtualId# [5:7587241444152493108:0]
2025-12-24T02:11:48.792522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444152493118:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.792553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:48.792575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:48.806030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:48.806160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241440393728319:2049] [node 2] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:48.806172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241440393728220:2049] [node 2] ICP30 unregister session Session# [3:7587241440393728319:2049] VirtualId# [3:7587241440393728305:0]
2025-12-24T02:11:48.806180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241440393728319:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.806232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.806255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.817711Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:48.817809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241441243681016:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-12-24T02:11:48.817818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241441243680920:2049] [node 2] ICP30 unregister session Session# [4:7587241441243681016:2049] VirtualId# [4:7587241441243681005:0]
2025-12-24T02:11:48.817826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241441243681016:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.817877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.817900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.818102Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:48.818159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444152493105:2049] [node 2] ICS01 socket: 479 reason# EndOfStream
2025-12-24T02:11:48.818164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444152493021:2049] [node 2] ICP30 unregister session Session# [5:7587241444152493105:2049] VirtualId# [5:7587241444152493100:0]
2025-12-24T02:11:48.818169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444152493105:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.818199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.818219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.822737Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:48.822835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241441712103588:2049] [node 2] ICS01 socket: 469 reason# EndOfStream
2025-12-24T02:11:48.822844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241441712103506:2049] [node 2] ICP30 unregister session Session# [6:7587241441712103588:2049] VirtualId# [6:7587241441712103584:0]
2025-12-24T02:11:48.822853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241441712103588:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.822912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:48.822940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:48.849714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:48.849828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241441712103596:2050] [node 3] ICS07 socket disconnect 502 reason# EndOfStream
2025-12-24T02:11:48.849835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241441712103596:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.849891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.849913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241441712103596:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:48.850575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241441712103507:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.850588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241441712103596:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.850593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241441712103507:2050] [node 3] ICP30 unregister session Session# [6:7587241441712103596:2050] VirtualId# [6:7587241441712103587:0]
2025-12-24T02:11:48.850645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.850651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241441712103507:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.850654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.854754Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:48.854839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241444152493110:2050] [node 3] ICS07 socket disconnect 491 reason# EndOfStream
2025-12-24T02:11:48.854846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444152493110:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.854902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:48.854925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241444152493110:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:48.855524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241444152493022:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:48.855536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444152493110:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:48.855541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444152493022:2050] [node 3] ICP30 unregister session Session# [5:7587241444152493110:2050] VirtualId# [5:7587241444152493101:0]
2025-12-24T02:11:48.855599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:48.855605Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241444152493022:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.855608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63007) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:48.860845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241441712103507:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:48.867639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241444152493022:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:48.875601Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:48.875680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241441712103608:2065] [node 4] ICS01 socket: 514 reason# EndOfStream
2025-12-24T02:11:48.875687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241441712103570:2065] [node 4] ICP30 unregister session Session# [6:7587241441712103608:2065] VirtualId# [6:7587241441712103604:0]
2025-12-24T02:11:48.875692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241441712103608:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:48.875734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:48.875755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:48.926883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441197204934:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.926899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441197204935:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.926876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440362129360:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.926887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241440362129361:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.937148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443771273448:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:48.937165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241443771273449:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:48.931191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241440090997964:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.931203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241440090997965:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.931208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241440090997966:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.950403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241443312700449:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.967526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241443312700450:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.967555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241443312700451:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.975015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444077752565:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:48.975049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444077752566:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:48.975056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444077752567:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:48.993537Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444077752615:2096] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:48.994649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241443312700510:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:48.995280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241441197204995:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:49.007624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241448372719922:2104] [node 4] ICP01 ready to work
2025-12-24T02:11:49.011611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241445492172297:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:49.011823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241445492172298:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:49.011679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241448066240809:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:49.011870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241448066240810:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:49.011980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241448066240811:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:49.031065Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241448066240809:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18983630ns) {Configure:104(18982968ns) {ProcessConfigured:135(18594640ns) {StartInitialHandshake:157(18594078ns) {DropHandshakes:470(532ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(1903ns) {SwitchToState:179(1289ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.039193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241444657096719:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:49.039365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241444657096720:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:49.039428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096718:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.042327Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241444077752566:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34609730ns) {Configure:104(34608787ns) {ProcessConfigured:135(34135531ns) {StartInitialHandshake:157(34134964ns) {DropHandshakes:470(1420ns) {DropIncomingHandshake:442(246ns) DropOutgoingHandshake:457(324ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(2221ns) {SwitchToState:179(1423ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.043099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444077752619:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.047364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241444657096733:2114] [node 5] ICP01 ready to work
2025-12-24T02:11:49.047218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447607667810:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:49.047361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447607667811:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:49.047427Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241443312700510:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.047482Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667809:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.058908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719940:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.059002Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241443771273449:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27864355ns) {Configure:104(27863012ns) {ProcessConfigured:135(27478218ns) {StartInitialHandshake:157(27476958ns) {DropHandshakes:470(1449ns) {DropIncomingHandshake:442(368ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(1387ns) {SwitchToState:179(838ns)}}}}}
2025-12-24T02:11:49.060119Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241448066240821:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6463
2025-12-24T02:11:49.060133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241448066240822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6463
2025-12-24T02:11:49.069235Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241440090997964:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35913865ns) {Configure:104(35912877ns) {ProcessConfigured:135(35478814ns) {StartInitialHandshake:157(35477948ns) {DropHandshakes:470(708ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(320ns) PrepareNewSessionHandshake:477(1931ns) {SwitchToState:179(1227ns)}}}}}
2025-12-24T02:11:49.069625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241444385965325:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.073737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667826:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.073057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172309:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.073078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172310:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.073111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172311:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.073142Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172312:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.074052Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444077752619:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.074144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444077752566:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.075183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240822:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.075345Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241448372719940:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.075373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444077752615:2096] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.077503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172310:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.077535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172311:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.077555Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172312:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.077833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096736:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.077878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096737:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.076382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447607667826:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.078765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241441197204997:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.078774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241441197204997:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.080623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447607667817:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.080638Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241447607667817:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.081011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096737:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.081092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096736:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.081122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241440362129360:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.081242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240821:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.081448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240815:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.081462Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241448066240815:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.086574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241441197204934:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.086936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096718:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.086962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241440362129361:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.087112Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719943:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.087914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241441197204997:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.088155Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444077752619:2098] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.088234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444077752566:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.088256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.088263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444077752566:2049] [node 2] ICP22 created new session: [6:7587241448372719944:2049]
2025-12-24T02:11:49.088271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241448372719944:2049] [node 2] ICS09 handshake done sender: [6:7587241444077752619:2098] self: [6:7587241448372719941:0] peer: [2:7587241441197204996:0] socket: 459 qp: -1
2025-12-24T02:11:49.088279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241448372719944:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.088287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.089583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172309:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.094690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241444385965327:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:49.095860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965326:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.096649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172302:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.096662Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241445492172302:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.097270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441197204995:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.097286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.097292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441197204995:2107] [node 6] ICP22 created new session: [2:7587241445492172317:2107]
2025-12-24T02:11:49.097298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241445492172317:2107] [node 6] ICS09 handshake done sender: [2:7587241441197204997:2108] self: [2:7587241441197204996:0] peer: [6:7587241448372719941:0] socket: 458 qp: -1
2025-12-24T02:11:49.097305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241445492172317:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:49.097316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.098379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447607667809:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.098500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241443312700449:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.098570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241443312700450:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.098815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447607667817:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.100646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241447607667810:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.100660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.100666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447607667810:2109] [node 6] ICP22 created new session: [5:7587241447607667830:2109]
2025-12-24T02:11:49.100672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447607667830:2109] [node 6] ICS09 handshake done sender: [5:7587241447607667817:2114] self: [5:7587241447607667816:0] peer: [6:7587241448372719942:0] socket: 474 qp: -1
2025-12-24T02:11:49.100684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447607667830:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:49.100692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.101531Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667832:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.099124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240808:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.101177Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096741:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.099136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241448066240808:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.101207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096742:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.099762Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240813:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.099765Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241448066240813:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.100532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241448066240827:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6463
2025-12-24T02:11:49.100546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241448066240828:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6463
2025-12-24T02:11:49.100592Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241448066240808:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.100754Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241448066240813:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.100968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443771273448:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.100979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.100985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443771273448:2048] [node 2] ICP22 created new session: [1:7587241448066240832:2048]
2025-12-24T02:11:49.100990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241448066240832:2048] [node 2] ICS09 handshake done sender: [1:7587241448066240808:2110] self: [1:7587241443771273508:0] peer: [2:7587241445492172316:0] socket: 481 qp: -1
2025-12-24T02:11:49.100996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241448066240832:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:49.101003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.101240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241448066240809:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.101252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.101256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241448066240809:2107] [node 5] ICP22 created new session: [1:7587241448066240834:2107]
2025-12-24T02:11:49.101260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241448066240834:2107] [node 5] ICS09 handshake done sender: [1:7587241448066240813:2111] self: [1:7587241448066240812:0] peer: [5:7587241447607667828:0] socket: 482 qp: -1
2025-12-24T02:11:49.101264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241448066240834:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:49.101268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.099247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241445492172302:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.101760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172300:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.101763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241445492172300:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.103039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241441197204935:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.104354Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447607667826:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.104419Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241443312700449:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241447607667826:2117] held: no
2025-12-24T02:11:49.104509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241443312700449:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.104517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.104523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241443312700449:2048] [node 1] ICP22 created new session: [5:7587241447607667834:2048]
2025-12-24T02:11:49.104528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447607667834:2048] [node 1] ICS09 handshake done sender: [5:7587241447607667826:2117] self: [5:7587241447607667828:0] peer: [1:7587241448066240812:0] socket: 489 qp: -1
2025-12-24T02:11:49.104535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447607667834:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.104541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.103162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965326:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.104418Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965339:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.104436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241440090997966:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.103063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.103084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441197204935:2049] [node 3] ICP22 created new session: [2:7587241445492172323:2049]
2025-12-24T02:11:49.103089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241445492172323:2049] [node 3] ICS09 handshake done sender: [2:7587241445492172302:2112] self: [2:7587241445492172301:0] peer: [3:7587241444657096740:0] socket: 463 qp: -1
2025-12-24T02:11:49.103095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241445492172323:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.103101Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.106418Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241448066240815:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.109866Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241444657096718:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.112322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241443771273449:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.112342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.112349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241443771273449:2049] [node 3] ICP22 created new session: [1:7587241448066240838:2049]
2025-12-24T02:11:49.112356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241448066240838:2049] [node 3] ICS09 handshake done sender: [1:7587241448066240815:2112] self: [1:7587241448066240814:0] peer: [3:7587241444657096738:0] socket: 483 qp: -1
2025-12-24T02:11:49.112363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241448066240838:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.112370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.115344Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241444077752615:2096] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31804) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241448372719940:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.120855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965341:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.120886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965342:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.120962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241440362129361:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.120985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.120992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440362129361:2049] [node 2] ICP22 created new session: [3:7587241444657096745:2049]
2025-12-24T02:11:49.121000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241444657096745:2049] [node 2] ICS09 handshake done sender: [3:7587241444657096718:2109] self: [3:7587241444657096740:0] peer: [2:7587241445492172301:0] socket: 464 qp: -1
2025-12-24T02:11:49.121007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241444657096745:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.121017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.122653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096722:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.122658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241444657096722:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.122368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719947:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.124294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965339:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.125805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240827:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.125843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241448066240840:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6463
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.126113Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241440362129360:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16153) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241444657096737:2117] held: no
2025-12-24T02:11:49.126238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096741:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.126383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096743:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.126206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.124080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719948:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.124235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241448372719940:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.124422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444077752615:2096] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.124431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.124436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444077752615:2096] [node 5] ICP22 created new session: [6:7587241448372719950:2096]
2025-12-24T02:11:49.124441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241448372719950:2096] [node 5] ICS09 handshake done sender: [6:7587241448372719940:2116] self: [6:7587241448372719942:0] peer: [5:7587241447607667816:0] socket: 476 qp: -1
2025-12-24T02:11:49.124446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241448372719950:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:49.124453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.127859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.127649Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241445492172325:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7178
2025-12-24T02:11:49.128862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241445492172300:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.127941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965342:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.127967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965341:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.128073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241440090997964:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.128152Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965344:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.128164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965345:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.130358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240817:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.130367Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241448066240817:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.131061Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172320:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.131113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241445492172297:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.131125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.131129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241445492172297:2109] [node 5] ICP22 created new session: [2:7587241445492172328:2109]
2025-12-24T02:11:49.131134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241445492172328:2109] [node 5] ICS09 handshake done sender: [2:7587241445492172300:2111] self: [2:7587241445492172299:0] peer: [5:7587241447607667829:0] socket: 462 qp: -1
2025-12-24T02:11:49.131139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241445492172328:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:49.131146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.131454Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241445492172312:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.130953Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965348:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.131710Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241448066240817:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.131871Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241444385965342:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.131920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241440090997964:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.131931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.131936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241440090997964:2048] [node 1] ICP22 created new session: [4:7587241444385965349:2048]
2025-12-24T02:11:49.131940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241444385965349:2048] [node 1] ICS09 handshake done sender: [4:7587241444385965342:2118] self: [4:7587241444385965346:0] peer: [1:7587241448066240816:0] socket: 514 qp: -1
2025-12-24T02:11:49.131945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241444385965349:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.131951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.133071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240828:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.133084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241448066240810:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.133094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.133099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241448066240810:2108] [node 4] ICP22 created new session: [1:7587241448066240843:2108]
2025-12-24T02:11:49.133103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241448066240843:2108] [node 4] ICS09 handshake done sender: [1:7587241448066240817:2113] self: [1:7587241448066240816:0] peer: [4:7587241444385965346:0] socket: 479 qp: -1
2025-12-24T02:11:49.133107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241448066240843:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:49.133113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.129022Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447607667809:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.129074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241443312700450:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.129084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.129092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241443312700450:2049] [node 2] ICP22 created new session: [5:7587241447607667838:2049]
2025-12-24T02:11:49.129098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447607667838:2049] [node 2] ICS09 handshake done sender: [5:7587241447607667809:2111] self: [5:7587241447607667829:0] peer: [2:7587241445492172299:0] socket: 465 qp: -1
2025-12-24T02:11:49.129104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447607667838:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.129112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.134031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241441197204934:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.134042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.134046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241441197204934:2048] [node 1] ICP22 created new session: [2:7587241445492172330:2048]
2025-12-24T02:11:49.134051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241445492172330:2048] [node 1] ICS09 handshake done sender: [2:7587241445492172312:2118] self: [2:7587241445492172316:0] peer: [1:7587241443771273508:0] socket: 488 qp: -1
2025-12-24T02:11:49.134055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241445492172330:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.134060Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.136929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241444657096744:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18187
2025-12-24T02:11:49.137261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241443771273448:2048] [node 2] ICP18 handshake (actor [1:7587241448066240828:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.137834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240840:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.137859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241448066240819:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.137862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241448066240819:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.138545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241448066240819:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.138750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241448066240811:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.138762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.138766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241448066240811:2109] [node 6] ICP22 created new session: [1:7587241448066240847:2109]
2025-12-24T02:11:49.138770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241448066240847:2109] [node 6] ICS09 handshake done sender: [1:7587241448066240819:2114] self: [1:7587241448066240818:0] peer: [6:7587241448372719953:0] socket: 512 qp: -1
2025-12-24T02:11:49.138775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241448066240847:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:49.138780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.137526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241448372719948:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.137638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444077752565:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.137703Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719952:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.138639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719954:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.139200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241448372719948:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.139385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444077752565:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.139394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.139398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444077752565:2048] [node 1] ICP22 created new session: [6:7587241448372719955:2048]
2025-12-24T02:11:49.139402Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241448372719955:2048] [node 1] ICS09 handshake done sender: [6:7587241448372719948:2119] self: [6:7587241448372719953:0] peer: [1:7587241448066240818:0] socket: 456 qp: -1
2025-12-24T02:11:49.139406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241448372719955:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.139411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.150396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241444657096722:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.150279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667840:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.151995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241444657096719:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.152018Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.152025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241444657096719:2107] [node 4] ICP22 created new session: [3:7587241444657096751:2107]
2025-12-24T02:11:49.152031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241444657096751:2107] [node 4] ICS09 handshake done sender: [3:7587241444657096722:2110] self: [3:7587241444657096721:0] peer: [4:7587241444385965340:0] socket: 466 qp: -1
2025-12-24T02:11:49.152038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241444657096751:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:49.152047Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.152314Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096744:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.152324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241444657096737:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.153964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241444385965326:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.154382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241440362129360:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.154396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.154402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241440362129360:2048] [node 1] ICP22 created new session: [3:7587241444657096753:2048]
2025-12-24T02:11:49.154407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241444657096753:2048] [node 1] ICS09 handshake done sender: [3:7587241444657096737:2117] self: [3:7587241444657096738:0] peer: [1:7587241448066240814:0] socket: 490 qp: -1
2025-12-24T02:11:49.154411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241444657096753:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.154418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.156368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667841:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.158270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447607667840:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.158299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241447607667811:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.158619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965345:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.158634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241440090997966:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.158646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.158651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241440090997966:2050] [node 3] ICP22 created new session: [4:7587241444385965352:2050]
2025-12-24T02:11:49.158657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241444385965352:2050] [node 3] ICS09 handshake done sender: [4:7587241444385965326:2111] self: [4:7587241444385965340:0] peer: [3:7587241444657096721:0] socket: 485 qp: -1
2025-12-24T02:11:49.158662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241444385965352:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.158668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.158887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241440090997966:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:49.159133Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965333:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.159137Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241444385965333:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.160107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241440090997965:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.160281Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241444385965333:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.160374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241444385965325:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.160397Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.160412Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241444385965325:2108] [node 5] ICP22 created new session: [4:7587241444385965357:2108]
2025-12-24T02:11:49.160427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241444385965357:2108] [node 5] ICS09 handshake done sender: [4:7587241444385965333:2114] self: [4:7587241444385965332:0] peer: [5:7587241447607667842:0] socket: 501 qp: -1
2025-12-24T02:11:49.160441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241444385965357:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:49.160457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.160741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667843:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.160997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447607667840:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.167572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241445492172306:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.167585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241445492172306:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.168199Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241445492172306:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.168352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241445492172298:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.168368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.168373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241445492172298:2110] [node 4] ICP22 created new session: [2:7587241445492172334:2110]
2025-12-24T02:11:49.168378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241445492172334:2110] [node 4] ICS09 handshake done sender: [2:7587241445492172306:2114] self: [2:7587241445492172305:0] peer: [4:7587241444385965356:0] socket: 516 qp: -1
2025-12-24T02:11:49.168383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241445492172334:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:49.168389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.169130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241448372719952:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.169167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719957:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.169205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444077752567:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.169429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096728:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.169437Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241444657096728:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.170008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719959:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.170392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241444657096728:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.170644Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241448372719952:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.167480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241447607667811:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.167498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.167505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447607667811:2110] [node 4] ICP22 created new session: [5:7587241447607667844:2110]
2025-12-24T02:11:49.167510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447607667844:2110] [node 4] ICS09 handshake done sender: [5:7587241447607667840:2120] self: [5:7587241447607667842:0] peer: [4:7587241444385965332:0] socket: 515 qp: -1
2025-12-24T02:11:49.167515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447607667844:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:49.167521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.168717Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241447607667811:2110] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:49.170138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447607667841:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.170160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241443312700451:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.170722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444077752567:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.170736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.170740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444077752567:2050] [node 3] ICP22 created new session: [6:7587241448372719960:2050]
2025-12-24T02:11:49.170745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241448372719960:2050] [node 3] ICS09 handshake done sender: [6:7587241448372719952:2120] self: [6:7587241448372719958:0] peer: [3:7587241444657096727:0] socket: 487 qp: -1
2025-12-24T02:11:49.170750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241448372719960:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.170755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.172107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241444657096735:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.172115Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241444657096735:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.172550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241444657096720:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.172561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.172566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241444657096720:2108] [node 6] ICP22 created new session: [3:7587241444657096757:2108]
2025-12-24T02:11:49.172571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241444657096757:2108] [node 6] ICS09 handshake done sender: [3:7587241444657096728:2113] self: [3:7587241444657096727:0] peer: [6:7587241448372719958:0] socket: 469 qp: -1
2025-12-24T02:11:49.172576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241444657096757:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:49.172582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.174520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241440090997965:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2588) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241444385965345:2120] held: no
2025-12-24T02:11:49.174661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241444385965359:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:61794
2025-12-24T02:11:49.181121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241448372719957:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.181156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241448372719922:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.182415Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241444657096735:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.182334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447607667847:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15484
2025-12-24T02:11:49.182567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447607667841:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.182627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241443312700451:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.182636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.182641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241443312700451:2050] [node 3] ICP22 created new session: [5:7587241447607667848:2050]
2025-12-24T02:11:49.182647Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447607667848:2050] [node 3] ICS09 handshake done sender: [5:7587241447607667841:2121] self: [5:7587241447607667846:0] peer: [3:7587241444657096734:0] socket: 521 qp: -1
2025-12-24T02:11:49.182653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447607667848:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.182661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.181402Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241444385965335:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.181411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241444385965335:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.182007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241444385965345:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.183830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241444657096733:2114] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.183857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.183862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241444657096733:2114] [node 5] ICP22 created new session: [3:7587241444657096760:2114]
2025-12-24T02:11:49.183867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241444657096760:2114] [node 5] ICS09 handshake done sender: [3:7587241444657096735:2115] self: [3:7587241444657096734:0] peer: [5:7587241447607667846:0] socket: 499 qp: -1
2025-12-24T02:11:49.183872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241444657096760:2114] [node 5] ICS10 traffic start
2025-12-24T02:11:49.183878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.185196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241448372719963:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57392
2025-12-24T02:11:49.185572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241448372719957:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.185682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241448372719922:2104] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.185694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.185698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241448372719922:2104] [node 4] ICP22 created new session: [6:7587241448372719964:2104]
2025-12-24T02:11:49.185703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241448372719964:2104] [node 4] ICS09 handshake done sender: [6:7587241448372719957:2122] self: [6:7587241448372719962:0] peer: [4:7587241444385965334:0] socket: 472 qp: -1
2025-12-24T02:11:49.185707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241448372719964:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:49.185714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.184760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241440090997965:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.184773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.184779Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241440090997965:2049] [node 2] ICP22 created new session: [4:7587241444385965361:2049]
2025-12-24T02:11:49.184784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241444385965361:2049] [node 2] ICS09 handshake done sender: [4:7587241444385965345:2120] self: [4:7587241444385965356:0] peer: [2:7587241445492172305:0] socket: 519 qp: -1
2025-12-24T02:11:49.184788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241444385965361:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.184794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.185400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241444385965335:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.187565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241444385965327:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.187576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.187581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241444385965327:2109] [node 6] ICP22 created new session: [4:7587241444385965363:2109]
2025-12-24T02:11:49.187586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241444385965363:2109] [node 6] ICS09 handshake done sender: [4:7587241444385965335:2115] self: [4:7587241444385965334:0] peer: [6:7587241448372719962:0] socket: 493 qp: -1
2025-12-24T02:11:49.187590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241444385965363:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:49.187597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.198298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:49.198397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241445492172330:2048] [node 1] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:49.198419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241441197204934:2048] [node 1] ICP30 unregister session Session# [2:7587241445492172330:2048] VirtualId# [2:7587241445492172316:0]
2025-12-24T02:11:49.198427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241445492172330:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.198465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.198489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.200838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:49.200915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241444385965349:2048] [node 1] ICS01 socket: 514 reason# EndOfStream
2025-12-24T02:11:49.200921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241440090997964:2048] [node 1] ICP30 unregister session Session# [4:7587241444385965349:2048] VirtualId# [4:7587241444385965346:0]
2025-12-24T02:11:49.200927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241444385965349:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.200967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.200993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.202847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:49.202920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241444657096753:2048] [node 1] ICS01 socket: 490 reason# EndOfStream
2025-12-24T02:11:49.202927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241440362129360:2048] [node 1] ICP30 unregister session Session# [3:7587241444657096753:2048] VirtualId# [3:7587241444657096738:0]
2025-12-24T02:11:49.202933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241444657096753:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.202972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.202994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.203026Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:49.203093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447607667834:2048] [node 1] ICS01 socket: 489 reason# EndOfStream
2025-12-24T02:11:49.203098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241443312700449:2048] [node 1] ICP30 unregister session Session# [5:7587241447607667834:2048] VirtualId# [5:7587241447607667828:0]
2025-12-24T02:11:49.203102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447607667834:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.203132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.203151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.203178Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:49.203241Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241448372719955:2048] [node 1] ICS01 socket: 456 reason# EndOfStream
2025-12-24T02:11:49.203246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444077752565:2048] [node 1] ICP30 unregister session Session# [6:7587241448372719955:2048] VirtualId# [6:7587241448372719953:0]
2025-12-24T02:11:49.203250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241448372719955:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.203273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.203291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.209131Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:49.209207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241444657096745:2049] [node 2] ICS01 socket: 464 reason# EndOfStream
2025-12-24T02:11:49.209213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241440362129361:2049] [node 2] ICP30 unregister session Session# [3:7587241444657096745:2049] VirtualId# [3:7587241444657096740:0]
2025-12-24T02:11:49.209219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241444657096745:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.209253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.209270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.239083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:49.239190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241444385965361:2049] [node 2] ICS01 socket: 519 reason# EndOfStream
2025-12-24T02:11:49.239201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241440090997965:2049] [node 2] ICP30 unregister session Session# [4:7587241444385965361:2049] VirtualId# [4:7587241444385965356:0]
2025-12-24T02:11:49.239208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241444385965361:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.239255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.239279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.242837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:49.242932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447607667838:2049] [node 2] ICS01 socket: 465 reason# EndOfStream
2025-12-24T02:11:49.242940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241443312700450:2049] [node 2] ICP30 unregister session Session# [5:7587241447607667838:2049] VirtualId# [5:7587241447607667829:0]
2025-12-24T02:11:49.242947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447607667838:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.242992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.243026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.243119Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:49.243213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241448372719944:2049] [node 2] ICS01 socket: 459 reason# EndOfStream
2025-12-24T02:11:49.243218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444077752566:2049] [node 2] ICP30 unregister session Session# [6:7587241448372719944:2049] VirtualId# [6:7587241448372719941:0]
2025-12-24T02:11:49.243222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241448372719944:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.243248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.243278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.266686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:49.266852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241448372719960:2050] [node 3] ICS01 socket: 487 reason# EndOfStream
2025-12-24T02:11:49.266862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444077752567:2050] [node 3] ICP30 unregister session Session# [6:7587241448372719960:2050] VirtualId# [6:7587241448372719958:0]
2025-12-24T02:11:49.266870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241448372719960:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.266922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.266945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.266986Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:49.267046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447607667848:2050] [node 3] ICS01 socket: 521 reason# EndOfStream
2025-12-24T02:11:49.267050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241443312700451:2050] [node 3] ICP30 unregister session Session# [5:7587241447607667848:2050] VirtualId# [5:7587241447607667846:0]
2025-12-24T02:11:49.267054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447607667848:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.267078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.267096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.297367Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:49.297471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241448372719964:2104] [node 4] ICS01 socket: 472 reason# EndOfStream
2025-12-24T02:11:49.297481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241448372719922:2104] [node 4] ICP30 unregister session Session# [6:7587241448372719964:2104] VirtualId# [6:7587241448372719962:0]
2025-12-24T02:11:49.297491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241448372719964:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.297564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:49.297589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:49.346816Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447904606715:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.343004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241445888398651:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:49.343018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241445888398652:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:49.346840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447904606716:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:49.346828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241446550554908:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.346844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241446550554909:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.343145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241445856799090:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.343155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241445856799091:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.343160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241445856799092:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.360307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444472742282:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.360330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444472742283:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.360336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444472742284:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.350313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444627915582:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.350322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444627915583:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.350327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444627915584:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.373251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241445888398704:2097] [node 4] ICP01 ready to work
2025-12-24T02:11:49.375249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444627915609:2073] [node 6] ICP01 ready to work
2025-12-24T02:11:49.375825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241444627915616:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:49.375485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447904606764:2091] [node 4] ICP01 ready to work
2025-12-24T02:11:49.375627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241445888398712:2101] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.376372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742293:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.375516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241446550554950:2086] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.381586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241445888398723:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.381081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241446550554963:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:49.382255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444472742310:2062] [node 5] ICP01 ready to work
2025-12-24T02:11:49.387571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241445888398717:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35572
2025-12-24T02:11:49.390043Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915630:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.391782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241445856799148:2073] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.394957Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241447904606764:2091] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19410248ns) {Configure:104(19409893ns) {ProcessConfigured:135(19197455ns) {StartInitialHandshake:157(19196910ns) {DropHandshakes:470(192ns) {DropIncomingHandshake:442(39ns) DropOutgoingHandshake:457(26ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:477(996ns) {SwitchToState:179(737ns)}}}}}
2025-12-24T02:11:49.395089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447904606782:2094] [node 5] ICP01 ready to work
2025-12-24T02:11:49.395471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447904606786:2104] [node 6] ICP01 ready to work
2025-12-24T02:11:49.396260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606784:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
2025-12-24T02:11:49.396273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606785:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
2025-12-24T02:11:49.396372Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606783:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.392616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799145:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.392638Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799147:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.392652Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799146:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.393625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241446550554977:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:49.398480Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550554983:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.400467Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241445888398730:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35572
2025-12-24T02:11:49.400491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241445888398731:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35572
2025-12-24T02:11:49.400503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241445888398732:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35572
2025-12-24T02:11:49.400727Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241446550554963:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19596194ns) {Configure:104(19595266ns) {ProcessConfigured:135(19365817ns) {StartInitialHandshake:157(19365287ns) {DropHandshakes:470(169ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(27ns)} GenerateSessionVirtualId:988(85ns) PrepareNewSessionHandshake:477(685ns) {SwitchToState:179(280ns)}}}}}
2025-12-24T02:11:49.399293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241445856799158:2075] [node 6] ICP01 ready to work
2025-12-24T02:11:49.402729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241444472742309:2071] [node 4] ICP01 ready to work
2025-12-24T02:11:49.407368Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241447904606716:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(33735745ns) {Configure:104(33733928ns) {ProcessConfigured:135(33281753ns) {StartInitialHandshake:157(33281392ns) {DropHandshakes:470(455ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(318ns) PrepareNewSessionHandshake:477(1222ns) {SwitchToState:179(568ns)}}}}}
2025-12-24T02:11:49.409172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606784:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.409373Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606783:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.409390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241447904606715:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.414739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241444627915583:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34465394ns) {Configure:104(34464920ns) {ProcessConfigured:135(34170145ns) {StartInitialHandshake:157(34169671ns) {DropHandshakes:470(348ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(577ns)}}}}}
2025-12-24T02:11:49.415812Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915656:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.418545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606799:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
2025-12-24T02:11:49.417783Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799146:2102] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.421261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799145:2101] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.421323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241445856799092:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.421378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241445856799090:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.421838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799159:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.421879Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398730:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.419515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444627915630:2089] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.423381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554983:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.423500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398703:2098] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.423517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241445888398703:2098] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.430450Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241444472742284:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(48105235ns) {Configure:104(48104214ns) {ProcessConfigured:135(47827251ns) {StartInitialHandshake:157(47826822ns) {DropHandshakes:470(412ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(195ns) PrepareNewSessionHandshake:477(1198ns) {SwitchToState:179(685ns)}}}}}
2025-12-24T02:11:49.434147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444472742293:2058] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.434182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444472742310:2062] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.433867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398707:2100] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.433888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241445888398707:2100] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.434735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554952:2087] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.434753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241446550554952:2087] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.436940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606785:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.438267Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606800:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
2025-12-24T02:11:49.438704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606800:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.438925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606781:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.438927Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447904606781:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.438784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799159:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.438815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241445856799091:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.450967Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799175:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.450986Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444627915615:2078] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.451003Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241444627915615:2078] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.451645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444627915582:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.454864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799147:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.455025Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742362:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.456960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444472742293:2058] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.457070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444472742310:2062] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.457091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.457097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444472742310:2062] [node 5] ICP22 created new session: [6:7587241444472742363:2062]
2025-12-24T02:11:49.457105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241444472742363:2062] [node 5] ICS09 handshake done sender: [6:7587241444472742293:2058] self: [6:7587241444472742361:0] peer: [5:7587241444627915614:0] socket: 469 qp: -1
2025-12-24T02:11:49.455202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444627915615:2078] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.455818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398714:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.455830Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241445888398714:2102] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.457116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241444472742363:2062] [node 5] ICS10 traffic start
2025-12-24T02:11:49.457142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.458304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550554989:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
2025-12-24T02:11:49.459046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447904606781:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.461104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447904606764:2091] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.461130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.461144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447904606764:2091] [node 4] ICP22 created new session: [2:7587241447904606805:2091]
2025-12-24T02:11:49.461150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447904606805:2091] [node 4] ICS09 handshake done sender: [2:7587241447904606781:2109] self: [2:7587241447904606767:0] peer: [4:7587241445856799174:0] socket: 487 qp: -1
2025-12-24T02:11:49.461157Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447904606805:2091] [node 4] ICS10 traffic start
2025-12-24T02:11:49.461167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.461198Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606799:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.461465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447904606804:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:2858
2025-12-24T02:11:49.460938Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241446550554952:2087] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.461198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241446550554950:2086] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.461214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.461220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241446550554950:2086] [node 4] ICP22 created new session: [3:7587241446550554991:2086]
2025-12-24T02:11:49.461226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241446550554991:2086] [node 4] ICS09 handshake done sender: [3:7587241446550554952:2087] self: [3:7587241446550554951:0] peer: [4:7587241445856799172:0] socket: 465 qp: -1
2025-12-24T02:11:49.461231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241446550554991:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:49.461239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.459323Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241445888398703:2098] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.459434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241445888398714:2102] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.461649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398732:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.461683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398717:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.461727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398731:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.461736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241445888398651:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.461746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.461752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241445888398651:2048] [node 2] ICP22 created new session: [1:7587241445888398738:2048]
2025-12-24T02:11:49.461757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241445888398738:2048] [node 2] ICS09 handshake done sender: [1:7587241445888398703:2098] self: [1:7587241445888398702:0] peer: [2:7587241447904606798:0] socket: 462 qp: -1
2025-12-24T02:11:49.461762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241445888398738:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:49.461781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.462129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241445888398651:2048] [node 2] ICP18 handshake (actor [1:7587241445888398731:2116]) is too old
2025-12-24T02:11:49.462176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241445888398707:2100] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.462451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241445888398704:2097] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.462466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.462473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241445888398704:2097] [node 4] ICP22 created new session: [1:7587241445888398743:2097]
2025-12-24T02:11:49.462478Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241445888398743:2097] [node 4] ICS09 handshake done sender: [1:7587241445888398707:2100] self: [1:7587241445888398706:0] peer: [4:7587241445856799173:0] socket: 464 qp: -1
2025-12-24T02:11:49.462483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241445888398743:2097] [node 4] ICS10 traffic start
2025-12-24T02:11:49.462489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.459526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241444627915609:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.459544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.459549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444627915609:2073] [node 6] ICP22 created new session: [5:7587241444627915666:2073]
2025-12-24T02:11:49.459555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444627915666:2073] [node 6] ICS09 handshake done sender: [5:7587241444627915615:2078] self: [5:7587241444627915614:0] peer: [6:7587241444472742361:0] socket: 467 qp: -1
2025-12-24T02:11:49.459561Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444627915666:2073] [node 6] ICS10 traffic start
2025-12-24T02:11:49.459571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.463315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550554994:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.463836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554989:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.464227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241445888398712:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.464243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.464250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241445888398712:2101] [node 5] ICP22 created new session: [1:7587241445888398745:2101]
2025-12-24T02:11:49.464255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241445888398745:2101] [node 5] ICS09 handshake done sender: [1:7587241445888398714:2102] self: [1:7587241445888398713:0] peer: [5:7587241444627915663:0] socket: 466 qp: -1
2025-12-24T02:11:49.464261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241445888398745:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:49.464268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.464673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241447904606715:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241447904606783:2110] held: no
2025-12-24T02:11:49.464903Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447904606783:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.463720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742366:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.464449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742367:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.464606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444472742367:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.464632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444472742284:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.464980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444472742366:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.464998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444472742283:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.468466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554979:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.468480Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241446550554979:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.469241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241446550554979:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.468330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915664:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.468431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915665:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.469674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241444627915609:2073] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:49.469764Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444627915656:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.469855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444627915630:2089] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.469964Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742370:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.470125Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444472742367:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.470220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444472742284:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.470236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.470245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444472742284:2050] [node 3] ICP22 created new session: [6:7587241444472742371:2050]
2025-12-24T02:11:49.470251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241444472742371:2050] [node 3] ICS09 handshake done sender: [6:7587241444472742367:2118] self: [6:7587241444472742368:0] peer: [3:7587241446550554978:0] socket: 518 qp: -1
2025-12-24T02:11:49.470259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241444472742371:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.470266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.470884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241446550554977:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.470904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.470928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241446550554977:2105] [node 6] ICP22 created new session: [3:7587241446550554997:2105]
2025-12-24T02:11:49.470933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241446550554997:2105] [node 6] ICS09 handshake done sender: [3:7587241446550554979:2108] self: [3:7587241446550554978:0] peer: [6:7587241444472742368:0] socket: 514 qp: -1
2025-12-24T02:11:49.470938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241446550554997:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:49.470945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.471691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444627915583:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.472031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241447904606715:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.472051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.472058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447904606715:2048] [node 1] ICP22 created new session: [2:7587241447904606809:2048]
2025-12-24T02:11:49.472064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447904606809:2048] [node 1] ICS09 handshake done sender: [2:7587241447904606783:2110] self: [2:7587241447904606798:0] peer: [1:7587241445888398702:0] socket: 482 qp: -1
2025-12-24T02:11:49.472070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447904606809:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.472077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.472346Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606790:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.472351Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447904606790:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.472861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606788:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.472865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447904606788:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.474868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550555000:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
2025-12-24T02:11:49.476484Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447904606790:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.476614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447904606788:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.476709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241445888398748:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35572
2025-12-24T02:11:49.477974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241445856799092:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22117) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241445856799146:2102] held: no
2025-12-24T02:11:49.478139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241445856799091:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241445856799159:2107] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.479872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554994:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.479905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241446550554908:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.480691Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398716:2103] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.480702Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241445888398716:2103] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.481542Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550555002:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
2025-12-24T02:11:49.479556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742373:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.479572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742374:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.481747Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241445856799090:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241445856799145:2101] held: no
2025-12-24T02:11:49.481890Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241445856799159:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.481993Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241445888398716:2103] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.482133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241446550554994:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.482194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241446550554908:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.482208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.482213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241446550554908:2048] [node 1] ICP22 created new session: [3:7587241446550555003:2048]
2025-12-24T02:11:49.482219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241446550555003:2048] [node 1] ICS09 handshake done sender: [3:7587241446550554994:2117] self: [3:7587241446550555001:0] peer: [1:7587241445888398709:0] socket: 516 qp: -1
2025-12-24T02:11:49.482225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241446550555003:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.482232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.483398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444627915582:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.483419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.483425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444627915582:2048] [node 1] ICP22 created new session: [5:7587241444627915670:2048]
2025-12-24T02:11:49.483432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444627915670:2048] [node 1] ICS09 handshake done sender: [5:7587241444627915630:2089] self: [5:7587241444627915663:0] peer: [1:7587241445888398713:0] socket: 473 qp: -1
2025-12-24T02:11:49.483437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444627915670:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.483443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.484998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241445888398652:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.485017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.485021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241445888398652:2049] [node 3] ICP22 created new session: [1:7587241445888398750:2049]
2025-12-24T02:11:49.485026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241445888398750:2049] [node 3] ICS09 handshake done sender: [1:7587241445888398716:2103] self: [1:7587241445888398709:0] peer: [3:7587241446550555001:0] socket: 491 qp: -1
2025-12-24T02:11:49.485030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241445888398750:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.485037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.479655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444472742366:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.479809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444472742283:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.479822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.479828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444472742283:2049] [node 2] ICP22 created new session: [6:7587241444472742376:2049]
2025-12-24T02:11:49.479833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241444472742376:2049] [node 2] ICS09 handshake done sender: [6:7587241444472742366:2117] self: [6:7587241444472742369:0] peer: [2:7587241447904606789:0] socket: 517 qp: -1
2025-12-24T02:11:49.479838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241444472742376:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.479843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.486307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915672:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.486985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447904606786:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.487007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.487013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447904606786:2104] [node 6] ICP22 created new session: [2:7587241447904606814:2104]
2025-12-24T02:11:49.487019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447904606814:2104] [node 6] ICS09 handshake done sender: [2:7587241447904606790:2114] self: [2:7587241447904606789:0] peer: [6:7587241444472742369:0] socket: 484 qp: -1
2025-12-24T02:11:49.487024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447904606814:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:49.487032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.487448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447904606782:2094] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.487459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.487463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447904606782:2094] [node 5] ICP22 created new session: [2:7587241447904606816:2094]
2025-12-24T02:11:49.487468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447904606816:2094] [node 5] ICS09 handshake done sender: [2:7587241447904606788:2113] self: [2:7587241447904606787:0] peer: [5:7587241444627915669:0] socket: 488 qp: -1
2025-12-24T02:11:49.487472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447904606816:2094] [node 5] ICS10 traffic start
2025-12-24T02:11:49.487488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.489402Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915673:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.489468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444627915656:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.490308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444627915583:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.490319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.490325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444627915583:2049] [node 2] ICP22 created new session: [5:7587241444627915676:2049]
2025-12-24T02:11:49.490331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444627915676:2049] [node 2] ICS09 handshake done sender: [5:7587241444627915656:2114] self: [5:7587241444627915669:0] peer: [2:7587241447904606787:0] socket: 496 qp: -1
2025-12-24T02:11:49.490336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444627915676:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.490341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.489672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241445856799091:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.489688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.489694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241445856799091:2049] [node 2] ICP22 created new session: [4:7587241445856799182:2049]
2025-12-24T02:11:49.489699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241445856799182:2049] [node 2] ICS09 handshake done sender: [4:7587241445856799159:2107] self: [4:7587241445856799174:0] peer: [2:7587241447904606767:0] socket: 493 qp: -1
2025-12-24T02:11:49.489704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241445856799182:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.489711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.489984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799177:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.492239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799178:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.493444Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241445856799179:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47684
2025-12-24T02:11:49.493504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241445856799146:2102] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.493767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241445856799092:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.493780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.493785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241445856799092:2050] [node 3] ICP22 created new session: [4:7587241445856799184:2050]
2025-12-24T02:11:49.493790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241445856799184:2050] [node 3] ICS09 handshake done sender: [4:7587241445856799146:2102] self: [4:7587241445856799172:0] peer: [3:7587241446550554951:0] socket: 476 qp: -1
2025-12-24T02:11:49.493795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241445856799184:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.493800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.494015Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241445856799145:2101] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.492983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444472742373:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.493020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444472742282:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.495631Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444627915673:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.495672Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915675:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.495692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444627915584:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.496176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241445888398729:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.496190Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241445888398729:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.494185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799179:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.494195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241445856799090:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.494200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.494204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241445856799090:2048] [node 1] ICP22 created new session: [4:7587241445856799186:2048]
2025-12-24T02:11:49.494207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241445856799186:2048] [node 1] ICS09 handshake done sender: [4:7587241445856799145:2101] self: [4:7587241445856799173:0] peer: [1:7587241445888398706:0] socket: 474 qp: -1
2025-12-24T02:11:49.494211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241445856799186:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.494215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.496756Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241445888398712:2101] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:49.496929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550555000:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.496966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241446550554984:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.496969Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241446550554984:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.497401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241446550554909:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.497826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915679:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.497913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241445888398729:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.498080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241445888398723:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.498095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.498101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241445888398723:2107] [node 6] ICP22 created new session: [1:7587241445888398753:2107]
2025-12-24T02:11:49.498106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241445888398753:2107] [node 6] ICS09 handshake done sender: [1:7587241445888398729:2114] self: [1:7587241445888398728:0] peer: [6:7587241444472742378:0] socket: 521 qp: -1
2025-12-24T02:11:49.498110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241445888398753:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:49.498117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.497979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742379:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.499235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241446550554984:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.499406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444627915673:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.499459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444627915584:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.499470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.499476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444627915584:2050] [node 3] ICP22 created new session: [5:7587241444627915680:2050]
2025-12-24T02:11:49.499481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444627915680:2050] [node 3] ICS09 handshake done sender: [5:7587241444627915673:2120] self: [5:7587241444627915678:0] peer: [3:7587241446550554964:0] socket: 529 qp: -1
2025-12-24T02:11:49.499486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444627915680:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.499493Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.500362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241446550554963:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.500376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.500381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241446550554963:2095] [node 5] ICP22 created new session: [3:7587241446550555007:2095]
2025-12-24T02:11:49.500387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241446550555007:2095] [node 5] ICS09 handshake done sender: [3:7587241446550554984:2112] self: [3:7587241446550554964:0] peer: [5:7587241444627915678:0] socket: 526 qp: -1
2025-12-24T02:11:49.500392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241446550555007:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:49.500398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.500910Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241444472742282:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62602) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241444472742373:2120] held: no
2025-12-24T02:11:49.502930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241444472742379:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.502454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447904606796:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.502467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447904606796:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.503433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799161:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.503442Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241445856799161:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.504450Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550555009:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
2025-12-24T02:11:49.504738Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241446550554963:2095] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:49.504760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241446550555010:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25942
2025-12-24T02:11:49.505856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241445856799161:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.506162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447904606796:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.506315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241446550555000:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.505973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241445856799158:2075] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.505989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.505993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241445856799158:2075] [node 6] ICP22 created new session: [4:7587241445856799190:2075]
2025-12-24T02:11:49.505997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241445856799190:2075] [node 6] ICS09 handshake done sender: [4:7587241445856799161:2108] self: [4:7587241445856799160:0] peer: [6:7587241444472742380:0] socket: 497 qp: -1
2025-12-24T02:11:49.506002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241445856799190:2075] [node 6] ICS10 traffic start
2025-12-24T02:11:49.506008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.506409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241444627915675:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.506441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241444627915616:2075] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.502956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241444472742309:2071] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.504709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742381:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.504984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241444472742382:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29805
2025-12-24T02:11:49.504997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444472742373:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.505042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444472742282:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.505051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.505057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444472742282:2048] [node 1] ICP22 created new session: [6:7587241444472742383:2048]
2025-12-24T02:11:49.505063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241444472742383:2048] [node 1] ICS09 handshake done sender: [6:7587241444472742373:2120] self: [6:7587241444472742378:0] peer: [1:7587241445888398728:0] socket: 523 qp: -1
2025-12-24T02:11:49.505069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241444472742383:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.505076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.506035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241444472742379:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.506576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241444472742309:2071] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.506585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.506589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241444472742309:2071] [node 4] ICP22 created new session: [6:7587241444472742385:2071]
2025-12-24T02:11:49.506594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241444472742385:2071] [node 4] ICS09 handshake done sender: [6:7587241444472742379:2122] self: [6:7587241444472742380:0] peer: [4:7587241445856799160:0] socket: 531 qp: -1
2025-12-24T02:11:49.506599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241444472742385:2071] [node 4] ICS10 traffic start
2025-12-24T02:11:49.506605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.506397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241446550554909:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.506410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.506415Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241446550554909:2049] [node 2] ICP22 created new session: [3:7587241446550555011:2049]
2025-12-24T02:11:49.506420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241446550555011:2049] [node 2] ICS09 handshake done sender: [3:7587241446550555000:2118] self: [3:7587241446550555006:0] peer: [2:7587241447904606756:0] socket: 525 qp: -1
2025-12-24T02:11:49.506426Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241446550555011:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.506432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.507960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447904606716:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.507979Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.507985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447904606716:2049] [node 3] ICP22 created new session: [2:7587241447904606819:2049]
2025-12-24T02:11:49.507991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447904606819:2049] [node 3] ICS09 handshake done sender: [2:7587241447904606796:2117] self: [2:7587241447904606756:0] peer: [3:7587241446550555006:0] socket: 479 qp: -1
2025-12-24T02:11:49.507997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447904606819:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.508004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.509518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241445856799154:2106] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.509527Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241445856799154:2106] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.511579Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241445856799154:2106] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.511655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241445856799148:2073] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.511667Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.511674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241445856799148:2073] [node 5] ICP22 created new session: [4:7587241445856799193:2073]
2025-12-24T02:11:49.511681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241445856799193:2073] [node 5] ICS09 handshake done sender: [4:7587241445856799154:2106] self: [4:7587241445856799153:0] peer: [5:7587241444627915682:0] socket: 480 qp: -1
2025-12-24T02:11:49.511686Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241445856799193:2073] [node 5] ICS10 traffic start
2025-12-24T02:11:49.511692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.515270Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241444627915616:2075] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241444627915675:2121] held: no
2025-12-24T02:11:49.515388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241444627915683:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:57194
2025-12-24T02:11:49.515821Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241444627915675:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.515880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241444627915616:2075] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.515892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.515899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241444627915616:2075] [node 4] ICP22 created new session: [5:7587241444627915684:2075]
2025-12-24T02:11:49.515904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241444627915684:2075] [node 4] ICS09 handshake done sender: [5:7587241444627915675:2121] self: [5:7587241444627915682:0] peer: [4:7587241445856799153:0] socket: 530 qp: -1
2025-12-24T02:11:49.515909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241444627915684:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:49.515914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.522062Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:49.522141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241447904606809:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-12-24T02:11:49.522149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241447904606715:2048] [node 1] ICP30 unregister session Session# [2:7587241447904606809:2048] VirtualId# [2:7587241447904606798:0]
2025-12-24T02:11:49.522155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241447904606809:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.522185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.522203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.522549Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:49.522621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241446550555003:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:49.522627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241446550554908:2048] [node 1] ICP30 unregister session Session# [3:7587241446550555003:2048] VirtualId# [3:7587241446550555001:0]
2025-12-24T02:11:49.522631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241446550555003:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.522655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.522671Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.530299Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:49.530373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444627915670:2048] [node 1] ICS01 socket: 473 reason# EndOfStream
2025-12-24T02:11:49.530381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444627915582:2048] [node 1] ICP30 unregister session Session# [5:7587241444627915670:2048] VirtualId# [5:7587241444627915663:0]
2025-12-24T02:11:49.530387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444627915670:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.530430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.530453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.530831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:49.530885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241445856799186:2048] [node 1] ICS01 socket: 474 reason# EndOfStream
2025-12-24T02:11:49.530889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241445856799090:2048] [node 1] ICP30 unregister session Session# [4:7587241445856799186:2048] VirtualId# [4:7587241445856799173:0]
2025-12-24T02:11:49.530893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241445856799186:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.530915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.530935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.530492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:49.530548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241444472742383:2048] [node 1] ICS01 socket: 523 reason# EndOfStream
2025-12-24T02:11:49.530553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444472742282:2048] [node 1] ICP30 unregister session Session# [6:7587241444472742383:2048] VirtualId# [6:7587241444472742378:0]
2025-12-24T02:11:49.530557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241444472742383:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.530584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.530603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.539218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:49.539288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241446550555011:2049] [node 2] ICS01 socket: 525 reason# EndOfStream
2025-12-24T02:11:49.539294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241446550554909:2049] [node 2] ICP30 unregister session Session# [3:7587241446550555011:2049] VirtualId# [3:7587241446550555006:0]
2025-12-24T02:11:49.539299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241446550555011:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.539335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.539351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.546706Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:49.546796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241445856799182:2049] [node 2] ICS01 socket: 493 reason# EndOfStream
2025-12-24T02:11:49.546804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241445856799091:2049] [node 2] ICP30 unregister session Session# [4:7587241445856799182:2049] VirtualId# [4:7587241445856799174:0]
2025-12-24T02:11:49.546809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241445856799182:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.546853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.546878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.546912Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:49.546981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444627915676:2049] [node 2] ICS01 socket: 496 reason# EndOfStream
2025-12-24T02:11:49.546985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444627915583:2049] [node 2] ICP30 unregister session Session# [5:7587241444627915676:2049] VirtualId# [5:7587241444627915669:0]
2025-12-24T02:11:49.546989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444627915676:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.547009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.547030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.547059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:49.547124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241444472742376:2049] [node 2] ICS01 socket: 517 reason# EndOfStream
2025-12-24T02:11:49.547129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444472742283:2049] [node 2] ICP30 unregister session Session# [6:7587241444472742376:2049] VirtualId# [6:7587241444472742369:0]
2025-12-24T02:11:49.547133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241444472742376:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.547156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.547174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.555251Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:49.555362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241445856799184:2050] [node 3] ICS01 socket: 476 reason# EndOfStream
2025-12-24T02:11:49.555369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241445856799092:2050] [node 3] ICP30 unregister session Session# [4:7587241445856799184:2050] VirtualId# [4:7587241445856799172:0]
2025-12-24T02:11:49.555375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241445856799184:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.555420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.555443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.560918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:49.561015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241444472742371:2050] [node 3] ICS01 socket: 518 reason# EndOfStream
2025-12-24T02:11:49.561022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444472742284:2050] [node 3] ICP30 unregister session Session# [6:7587241444472742371:2050] VirtualId# [6:7587241444472742368:0]
2025-12-24T02:11:49.561028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241444472742371:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.561071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.561093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.567128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:49.567204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241444627915680:2050] [node 3] ICS01 socket: 529 reason# EndOfStream
2025-12-24T02:11:49.567211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241444627915584:2050] [node 3] ICP30 unregister session Session# [5:7587241444627915680:2050] VirtualId# [5:7587241444627915678:0]
2025-12-24T02:11:49.567217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241444627915680:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.567258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.567280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.578199Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:49.578284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241444472742385:2071] [node 4] ICS01 socket: 531 reason# EndOfStream
2025-12-24T02:11:49.578291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241444472742309:2071] [node 4] ICP30 unregister session Session# [6:7587241444472742385:2071] VirtualId# [6:7587241444472742380:0]
2025-12-24T02:11:49.578298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241444472742385:2071] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.578345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:49.578368Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:49.608408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241446243675791:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:49.608417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241446243675792:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:49.615345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447652065990:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.615360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447652065991:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:49.622964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241446348433226:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.622979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241446348433227:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.622985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241446348433228:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.617122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448050187205:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.617131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448050187206:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.630814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241445534555214:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.630824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241445534555215:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.630829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241445534555216:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.640222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447010015311:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:49.640242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447010015312:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:49.640249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447010015313:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:49.719124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241446348433275:2095] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.720238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447010015358:2089] [node 4] ICP01 ready to work
2025-12-24T02:11:49.720347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241447010015359:2091] [node 6] ICP01 ready to work
2025-12-24T02:11:49.721237Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015360:2094] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.722182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241445534555235:2065] [node 4] ICP01 ready to work
2025-12-24T02:11:49.722273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241445534555236:2067] [node 5] ICP01 ready to work
2025-12-24T02:11:49.722425Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187265:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.721725Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433280:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.727206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555244:2072] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.727424Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241446243675855:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53282
2025-12-24T02:11:49.727925Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241446243675856:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53282
2025-12-24T02:11:49.732213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448050187272:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:49.733284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447652066057:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.742802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241446348433295:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:49.743963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015387:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.744305Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015387:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.747367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066059:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
2025-12-24T02:11:49.747392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066060:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
2025-12-24T02:11:49.747676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447652066061:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.751227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241446243675861:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:49.751381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241446243675862:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:49.754227Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241446243675863:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53282
2025-12-24T02:11:49.754257Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241446243675864:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53282
2025-12-24T02:11:49.751340Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241448050187206:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20127704ns) {Configure:104(20126753ns) {ProcessConfigured:135(19769327ns) {StartInitialHandshake:157(19768757ns) {DropHandshakes:470(338ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(194ns) PrepareNewSessionHandshake:477(1671ns) {SwitchToState:179(950ns)}}}}}
2025-12-24T02:11:49.754871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241448050187272:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22560732ns) {Configure:104(22560014ns) {ProcessConfigured:135(22110687ns) {StartInitialHandshake:157(22110325ns) {DropHandshakes:470(498ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(434ns) PrepareNewSessionHandshake:477(801ns) {SwitchToState:179(307ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.755473Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241447652066057:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(22100642ns) {Configure:104(22099593ns) {ProcessConfigured:135(21789467ns) {StartInitialHandshake:157(21789165ns) {DropHandshakes:470(520ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(1490ns) {SwitchToState:179(63ns)}}}}}
2025-12-24T02:11:49.755538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015365:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.755552Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241447010015365:2097] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.755206Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555286:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.755410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241445534555244:2072] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.755444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241445534555236:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.762852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447010015365:2097] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.762736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241446243675860:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.769518Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433305:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.769551Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433306:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.769571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433304:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.769725Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241446348433295:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(26826123ns) {Configure:104(26824789ns) {ProcessConfigured:135(26357610ns) {StartInitialHandshake:157(26357214ns) {DropHandshakes:470(777ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(207ns)} GenerateSessionVirtualId:988(525ns) PrepareNewSessionHandshake:477(465ns) {SwitchToState:179(273ns)}}}}}
2025-12-24T02:11:49.765939Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241446243675792:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(41899774ns) {Configure:104(41898876ns) {ProcessConfigured:135(41482181ns) {StartInitialHandshake:157(41481621ns) {DropHandshakes:470(423ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(1839ns) {SwitchToState:179(241ns)}}}}}
2025-12-24T02:11:49.763995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448050187274:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:49.768915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448050187280:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:49.765288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241447652066056:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:49.766825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066059:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.768627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241447652065990:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.773555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241447010015359:2091] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.773596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.773608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447010015359:2091] [node 6] ICP22 created new session: [5:7587241447010015391:2091]
2025-12-24T02:11:49.773617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447010015391:2091] [node 6] ICS09 handshake done sender: [5:7587241447010015365:2097] self: [5:7587241447010015364:0] peer: [6:7587241445534555287:0] socket: 472 qp: -1
2025-12-24T02:11:49.773627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447010015391:2091] [node 6] ICS10 traffic start
2025-12-24T02:11:49.773655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.774296Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675855:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.774332Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675856:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.774393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675852:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.774398Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241446243675852:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.774499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433304:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.775258Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433280:2098] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.775396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187265:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.775671Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066060:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.779090Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241445534555236:2067] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4502) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241445534555244:2072] held: no
2025-12-24T02:11:49.779330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555294:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.779609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015393:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.783190Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015393:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.784738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241447010015312:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.785211Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433305:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.785246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241446348433226:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.786091Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433311:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.786458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241446348433305:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.786568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241446348433226:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.786592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.786600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241446348433226:2048] [node 1] ICP22 created new session: [4:7587241446348433312:2048]
2025-12-24T02:11:49.786608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241446348433312:2048] [node 1] ICS09 handshake done sender: [4:7587241446348433305:2116] self: [4:7587241446348433310:0] peer: [1:7587241446243675865:0] socket: 510 qp: -1
2025-12-24T02:11:49.786616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241446348433312:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.786627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.784991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675864:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.785030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675863:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.785410Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675866:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.785415Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241446243675866:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.786247Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241446243675866:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.786753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015360:2094] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.786786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241447010015358:2089] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.786981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433277:2096] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.786988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241446348433277:2096] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.787394Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241446243675861:2111] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.787411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.787419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241446243675861:2111] [node 4] ICP22 created new session: [1:7587241446243675879:2111]
2025-12-24T02:11:49.787425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241446243675879:2111] [node 4] ICS09 handshake done sender: [1:7587241446243675866:2115] self: [1:7587241446243675865:0] peer: [4:7587241446348433310:0] socket: 498 qp: -1
2025-12-24T02:11:49.787430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241446243675879:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:49.787449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.792008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066063:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.792029Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447652066063:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.794024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187286:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.795122Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241446243675852:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.795255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187286:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.795303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241448050187206:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.795257Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241445534555215:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(64588818ns) {Configure:104(64587356ns) {ProcessConfigured:135(60015157ns) {StartInitialHandshake:157(60014162ns) {DropHandshakes:470(453ns) {DropIncomingHandshake:442(73ns) DropOutgoingHandshake:457(88ns)} GenerateSessionVirtualId:988(252ns) PrepareNewSessionHandshake:477(2206ns) {SwitchToState:179(643ns)}}}}}
2025-12-24T02:11:49.795358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241445534555244:2072] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.797598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241446348433277:2096] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.798827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066065:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.798845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447652066065:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.799160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066073:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.799205Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066074:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
2025-12-24T02:11:49.799240Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066075:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.800021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066074:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.800046Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447652066059:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.800100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241447652065990:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.800120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.800127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447652065990:2048] [node 1] ICP22 created new session: [2:7587241447652066077:2048]
2025-12-24T02:11:49.800133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447652066077:2048] [node 1] ICS09 handshake done sender: [2:7587241447652066059:2111] self: [2:7587241447652066071:0] peer: [1:7587241446243675851:0] socket: 480 qp: -1
2025-12-24T02:11:49.800141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447652066077:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.800160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.801312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066073:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.801364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447652066063:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.801447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447652066061:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.801457Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.801462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447652066061:2109] [node 5] ICP22 created new session: [2:7587241447652066081:2109]
2025-12-24T02:11:49.801468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447652066081:2109] [node 5] ICS09 handshake done sender: [2:7587241447652066063:2113] self: [2:7587241447652066062:0] peer: [5:7587241447010015397:0] socket: 503 qp: -1
2025-12-24T02:11:49.801472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447652066081:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:49.801478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.802854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187288:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.809971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187289:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.807002Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241447010015312:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241447010015393:2116] held: no
2025-12-24T02:11:49.809771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241446348433275:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.809802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.809812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241446348433275:2095] [node 5] ICP22 created new session: [4:7587241446348433316:2095]
2025-12-24T02:11:49.809819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241446348433316:2095] [node 5] ICS09 handshake done sender: [4:7587241446348433277:2096] self: [4:7587241446348433276:0] peer: [5:7587241447010015398:0] socket: 469 qp: -1
2025-12-24T02:11:49.809825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241446348433316:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:49.809833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.809021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241445534555236:2067] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.809048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.809057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241445534555236:2067] [node 5] ICP22 created new session: [6:7587241445534555296:2067]
2025-12-24T02:11:49.809065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241445534555296:2067] [node 5] ICS09 handshake done sender: [6:7587241445534555244:2072] self: [6:7587241445534555287:0] peer: [5:7587241447010015364:0] socket: 481 qp: -1
2025-12-24T02:11:49.809075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241445534555296:2067] [node 5] ICS10 traffic start
2025-12-24T02:11:49.809099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.809519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555298:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.809540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555299:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.810165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241446243675861:2111] [node 4] ICP35 event processing took too much time PendingConnection:279(22662938ns) {HandleHandshakeStatus:329(22662431ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(189ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(15109453ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15087646ns max# 15003083ns}}}
2025-12-24T02:11:49.810484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241446243675791:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.810498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.810503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241446243675791:2048] [node 2] ICP22 created new session: [1:7587241446243675883:2048]
2025-12-24T02:11:49.810508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241446243675883:2048] [node 2] ICS09 handshake done sender: [1:7587241446243675852:2107] self: [1:7587241446243675851:0] peer: [2:7587241447652066071:0] socket: 468 qp: -1
2025-12-24T02:11:49.810513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241446243675883:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:49.810519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.811949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187289:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.811704Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241445534555286:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.811804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241445534555214:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.815141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675868:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.815153Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241446243675868:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.819209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241446243675868:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.819484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241446243675862:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.819503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.819510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241446243675862:2112] [node 6] ICP22 created new session: [1:7587241446243675887:2112]
2025-12-24T02:11:49.819516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241446243675887:2112] [node 6] ICS09 handshake done sender: [1:7587241446243675868:2116] self: [1:7587241446243675867:0] peer: [6:7587241445534555301:0] socket: 499 qp: -1
2025-12-24T02:11:49.819522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241446243675887:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:49.819530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.821779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187288:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.821834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241448050187205:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.819108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555302:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.822099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241445534555286:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.822187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241445534555214:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.822203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.822210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241445534555214:2048] [node 1] ICP22 created new session: [6:7587241445534555303:2048]
2025-12-24T02:11:49.822217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241445534555303:2048] [node 1] ICS09 handshake done sender: [6:7587241445534555286:2109] self: [6:7587241445534555301:0] peer: [1:7587241446243675867:0] socket: 502 qp: -1
2025-12-24T02:11:49.822223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241445534555303:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.822231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.827204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433306:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.828105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433318:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.828246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241446348433228:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.827010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015401:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.827100Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015399:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.831076Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241446348433275:2095] [node 5] ICP35 event processing took too much time PendingConnection:279(21202066ns) {HandleHandshakeStatus:329(21200945ns) {DropIncomingHandshake:442(7386ns) DropOutgoingHandshake:457(145ns) SwitchToState:179(176ns) ProcessPendingSessionEvents:495(21077326ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 21058560ns max# 20982835ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.836338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433318:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.836384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241446348433227:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.836203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447010015360:2094] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.835671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241447652066085:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4937
2025-12-24T02:11:49.836532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066066:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.836537Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447652066066:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.837995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447652066066:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.837181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433321:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.838158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241446348433318:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.838226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241446348433227:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.838240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.838246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241446348433227:2049] [node 2] ICP22 created new session: [4:7587241446348433322:2049]
2025-12-24T02:11:49.838252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241446348433322:2049] [node 2] ICS09 handshake done sender: [4:7587241446348433318:2120] self: [4:7587241446348433320:0] peer: [2:7587241447652066058:0] socket: 530 qp: -1
2025-12-24T02:11:49.838260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241446348433322:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.838268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.839298Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241446348433307:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.839303Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241446348433307:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.838979Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241445534555298:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.839046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241445534555235:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.840216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066085:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.840232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447652066057:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.840246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.840253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447652066057:2110] [node 4] ICP22 created new session: [2:7587241447652066087:2110]
2025-12-24T02:11:49.840262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447652066087:2110] [node 4] ICS09 handshake done sender: [2:7587241447652066066:2115] self: [2:7587241447652066058:0] peer: [4:7587241446348433320:0] socket: 523 qp: -1
2025-12-24T02:11:49.840266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447652066087:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:49.840274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.843000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187277:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.843017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241448050187277:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.844020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241445534555299:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.844056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241445534555215:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.843331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447652066065:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.843386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447652065991:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.843400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.843407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447652065991:2049] [node 3] ICP22 created new session: [2:7587241447652066091:2049]
2025-12-24T02:11:49.843413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447652066091:2049] [node 3] ICS09 handshake done sender: [2:7587241447652066065:2114] self: [2:7587241447652066064:0] peer: [3:7587241448050187287:0] socket: 504 qp: -1
2025-12-24T02:11:49.843419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447652066091:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.843426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.846774Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675871:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.846796Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241446243675871:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.848449Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241447010015358:2089] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12371) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241447010015360:2094] held: no
2025-12-24T02:11:49.848563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241447010015358:2089] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.848573Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.848580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447010015358:2089] [node 4] ICP22 created new session: [5:7587241447010015402:2089]
2025-12-24T02:11:49.848587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447010015402:2089] [node 4] ICS09 handshake done sender: [5:7587241447010015360:2094] self: [5:7587241447010015398:0] peer: [4:7587241446348433276:0] socket: 470 qp: -1
2025-12-24T02:11:49.848593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447010015402:2089] [node 4] ICS10 traffic start
2025-12-24T02:11:49.848601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.848849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015401:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.849070Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015400:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.851621Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241446243675871:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.852119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241447010015311:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.852150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447010015393:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.851738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241446243675792:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.851761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.851767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241446243675792:2049] [node 3] ICP22 created new session: [1:7587241446243675890:2049]
2025-12-24T02:11:49.851773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241446243675890:2049] [node 3] ICS09 handshake done sender: [1:7587241446243675871:2117] self: [1:7587241446243675854:0] peer: [3:7587241448050187291:0] socket: 515 qp: -1
2025-12-24T02:11:49.851779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241446243675890:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:49.851786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.851055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241446348433307:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.852412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241448050187206:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241448050187286:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.855295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555308:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.856229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555309:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.853651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241448050187205:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241448050187288:2117] held: no
2025-12-24T02:11:49.857937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241447010015312:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.857952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.857959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447010015312:2049] [node 2] ICP22 created new session: [5:7587241447010015406:2049]
2025-12-24T02:11:49.857967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447010015406:2049] [node 2] ICS09 handshake done sender: [5:7587241447010015393:2116] self: [5:7587241447010015397:0] peer: [2:7587241447652066062:0] socket: 508 qp: -1
2025-12-24T02:11:49.857973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447010015406:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.857980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.858307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241448050187277:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.858461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241448050187272:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.858478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.858483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448050187272:2109] [node 4] ICP22 created new session: [3:7587241448050187294:2109]
2025-12-24T02:11:49.858488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241448050187294:2109] [node 4] ICS09 handshake done sender: [3:7587241448050187277:2112] self: [3:7587241448050187273:0] peer: [4:7587241446348433319:0] socket: 501 qp: -1
2025-12-24T02:11:49.858492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241448050187294:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:49.858498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.859307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241446348433295:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.859343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.859353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241446348433295:2107] [node 6] ICP22 created new session: [4:7587241446348433325:2107]
2025-12-24T02:11:49.859359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241446348433325:2107] [node 6] ICS09 handshake done sender: [4:7587241446348433307:2118] self: [4:7587241446348433301:0] peer: [6:7587241445534555306:0] socket: 518 qp: -1
2025-12-24T02:11:49.859365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241446348433325:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:49.859373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.857055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241445534555298:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:49.857116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241445534555235:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.857129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:49.857136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241445534555235:2065] [node 4] ICP22 created new session: [6:7587241445534555310:2065]
2025-12-24T02:11:49.857142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241445534555310:2065] [node 4] ICS09 handshake done sender: [6:7587241445534555298:2118] self: [6:7587241445534555306:0] peer: [4:7587241446348433301:0] socket: 483 qp: -1
2025-12-24T02:11:49.857151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241445534555310:2065] [node 4] ICS10 traffic start
2025-12-24T02:11:49.857159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:49.859035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241445534555309:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.859062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241445534555216:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.859262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187282:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.859272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241448050187282:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.860255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241448050187282:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.860278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241446243675873:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.860291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241446243675873:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.860137Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555313:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
2025-12-24T02:11:49.860769Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241445534555309:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.860850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241445534555216:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.860864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.860873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241445534555216:2050] [node 3] ICP22 created new session: [6:7587241445534555314:2050]
2025-12-24T02:11:49.860881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241445534555314:2050] [node 3] ICS09 handshake done sender: [6:7587241445534555309:2122] self: [6:7587241445534555312:0] peer: [3:7587241448050187281:0] socket: 506 qp: -1
2025-12-24T02:11:49.860886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241445534555314:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.860892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.861021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241448050187280:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.861039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.861044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448050187280:2113] [node 6] ICP22 created new session: [3:7587241448050187301:2113]
2025-12-24T02:11:49.861048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241448050187301:2113] [node 6] ICS09 handshake done sender: [3:7587241448050187282:2114] self: [3:7587241448050187281:0] peer: [6:7587241445534555312:0] socket: 500 qp: -1
2025-12-24T02:11:49.861052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241448050187301:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:49.861057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:49.864735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241446348433228:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8350) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241446348433306:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.866563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241446243675873:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.865081Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187298:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.863143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015408:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.866741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241446243675860:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.866757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.866762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241446243675860:2110] [node 5] ICP22 created new session: [1:7587241446243675893:2110]
2025-12-24T02:11:49.866768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241446243675893:2110] [node 5] ICS09 handshake done sender: [1:7587241446243675873:2118] self: [1:7587241446243675872:0] peer: [5:7587241447010015405:0] socket: 526 qp: -1
2025-12-24T02:11:49.866773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241446243675893:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:49.866782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.866842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241447652066070:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.866858Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241447652066070:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.868122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241447652065991:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(24626534ns) {HandleHandshakeStatus:329(24626159ns) {DropIncomingHandshake:442(3143ns) DropOutgoingHandshake:457(129ns) SwitchToState:179(72ns) ProcessPendingSessionEvents:495(7452089ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7428836ns max# 7347063ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.868417Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187299:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.868527Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241446348433327:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21296
2025-12-24T02:11:49.870395Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241446348433306:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.867685Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241447010015311:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8912) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241447010015401:2120] held: no
2025-12-24T02:11:49.870490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241446348433228:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.870504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.870510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241446348433228:2050] [node 3] ICP22 created new session: [4:7587241446348433328:2050]
2025-12-24T02:11:49.870517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241446348433328:2050] [node 3] ICS09 handshake done sender: [4:7587241446348433306:2117] self: [4:7587241446348433319:0] peer: [3:7587241448050187273:0] socket: 511 qp: -1
2025-12-24T02:11:49.870523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241446348433328:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.870528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.870900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241447652066070:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:49.871004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241447652066056:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.871022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:49.871027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241447652066056:2107] [node 6] ICP22 created new session: [2:7587241447652066094:2107]
2025-12-24T02:11:49.871033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241447652066094:2107] [node 6] ICS09 handshake done sender: [2:7587241447652066070:2116] self: [2:7587241447652066069:0] peer: [6:7587241445534555307:0] socket: 520 qp: -1
2025-12-24T02:11:49.871038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241447652066094:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:49.871044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.872572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015409:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.872769Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241445534555316:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.873190Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241448050187300:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40480
2025-12-24T02:11:49.873554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241447010015408:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.873588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241447010015313:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:49.873782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241445534555299:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.875252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241448050187286:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:49.875339Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241448050187288:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.875492Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447010015401:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:49.875603Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187300:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.875647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241448050187284:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:49.875650Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241448050187284:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:49.876321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241448050187206:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.876333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.876342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448050187206:2049] [node 2] ICP22 created new session: [3:7587241448050187303:2049]
2025-12-24T02:11:49.876348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241448050187303:2049] [node 2] ICS09 handshake done sender: [3:7587241448050187286:2116] self: [3:7587241448050187287:0] peer: [2:7587241447652066064:0] socket: 509 qp: -1
2025-12-24T02:11:49.876356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241448050187303:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.876364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.876577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241448050187205:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.876584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.876589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448050187205:2048] [node 1] ICP22 created new session: [3:7587241448050187305:2048]
2025-12-24T02:11:49.876594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241448050187305:2048] [node 1] ICS09 handshake done sender: [3:7587241448050187288:2117] self: [3:7587241448050187291:0] peer: [1:7587241446243675854:0] socket: 516 qp: -1
2025-12-24T02:11:49.876599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241448050187305:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.876605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:49.877154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241447010015411:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38417
2025-12-24T02:11:49.877245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241445534555215:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.877265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:49.877272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241445534555215:2049] [node 2] ICP22 created new session: [6:7587241445534555317:2049]
2025-12-24T02:11:49.877279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241445534555317:2049] [node 2] ICS09 handshake done sender: [6:7587241445534555299:2119] self: [6:7587241445534555307:0] peer: [2:7587241447652066069:0] socket: 488 qp: -1
2025-12-24T02:11:49.877285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241445534555317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:49.877292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:49.879106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241447010015311:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.879123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:49.879129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447010015311:2048] [node 1] ICP22 created new session: [5:7587241447010015412:2048]
2025-12-24T02:11:49.879134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447010015412:2048] [node 1] ICS09 handshake done sender: [5:7587241447010015401:2120] self: [5:7587241447010015405:0] peer: [1:7587241446243675872:0] socket: 484 qp: -1
2025-12-24T02:11:49.879140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447010015412:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:49.879146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:49.880008Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241448050187284:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:49.880216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241447010015408:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:49.880278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241447010015313:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:49.880294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:49.880300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241447010015313:2050] [node 3] ICP22 created new session: [5:7587241447010015414:2050]
2025-12-24T02:11:49.880307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241447010015414:2050] [node 3] ICS09 handshake done sender: [5:7587241447010015408:2121] self: [5:7587241447010015410:0] peer: [3:7587241448050187283:0] socket: 538 qp: -1
2025-12-24T02:11:49.880313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241447010015414:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:49.880321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:49.881443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241448050187274:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:49.881459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:49.881464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448050187274:2110] [node 5] ICP22 created new session: [3:7587241448050187309:2110]
2025-12-24T02:11:49.881470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241448050187309:2110] [node 5] ICS09 handshake done sender: [3:7587241448050187284:2115] self: [3:7587241448050187283:0] peer: [5:7587241447010015410:0] socket: 476 qp: -1
2025-12-24T02:11:49.881476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241448050187309:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:49.881497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:49.901021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:49.901096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241448050187305:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:49.901103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241448050187205:2048] [node 1] ICP30 unregister session Session# [3:7587241448050187305:2048] VirtualId# [3:7587241448050187291:0]
2025-12-24T02:11:49.901108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241448050187305:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.901143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.901164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.900244Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:49.900359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241447652066077:2048] [node 1] ICS01 socket: 480 reason# EndOfStream
2025-12-24T02:11:49.900370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241447652065990:2048] [node 1] ICP30 unregister session Session# [2:7587241447652066077:2048] VirtualId# [2:7587241447652066071:0]
2025-12-24T02:11:49.900379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241447652066077:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.900435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.900462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.915321Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:49.915427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447010015412:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-12-24T02:11:49.915436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241447010015311:2048] [node 1] ICP30 unregister session Session# [5:7587241447010015412:2048] VirtualId# [5:7587241447010015405:0]
2025-12-24T02:11:49.915444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447010015412:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.915498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.915531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.915575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:49.915644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241446348433312:2048] [node 1] ICS01 socket: 510 reason# EndOfStream
2025-12-24T02:11:49.915649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241446348433226:2048] [node 1] ICP30 unregister session Session# [4:7587241446348433312:2048] VirtualId# [4:7587241446348433310:0]
2025-12-24T02:11:49.915654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241446348433312:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.915685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.915710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.917545Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:49.917636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241445534555303:2048] [node 1] ICS01 socket: 502 reason# EndOfStream
2025-12-24T02:11:49.917643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241445534555214:2048] [node 1] ICP30 unregister session Session# [6:7587241445534555303:2048] VirtualId# [6:7587241445534555301:0]
2025-12-24T02:11:49.917650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241445534555303:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.917707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:49.917734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:49.924366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:49.924460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241448050187303:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-12-24T02:11:49.924468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241448050187206:2049] [node 2] ICP30 unregister session Session# [3:7587241448050187303:2049] VirtualId# [3:7587241448050187287:0]
2025-12-24T02:11:49.924475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241448050187303:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.924522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.924544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.930005Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:49.930102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241445534555317:2049] [node 2] ICS01 socket: 488 reason# EndOfStream
2025-12-24T02:11:49.930109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241445534555215:2049] [node 2] ICP30 unregister session Session# [6:7587241445534555317:2049] VirtualId# [6:7587241445534555307:0]
2025-12-24T02:11:49.930115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241445534555317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.930188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.930213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.930164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:49.930268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447010015406:2049] [node 2] ICS01 socket: 508 reason# EndOfStream
2025-12-24T02:11:49.930275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241447010015312:2049] [node 2] ICP30 unregister session Session# [5:7587241447010015406:2049] VirtualId# [5:7587241447010015397:0]
2025-12-24T02:11:49.930281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447010015406:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.930333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.930358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.938940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:49.939043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241446348433322:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-12-24T02:11:49.939052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241446348433227:2049] [node 2] ICP30 unregister session Session# [4:7587241446348433322:2049] VirtualId# [4:7587241446348433320:0]
2025-12-24T02:11:49.939059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241446348433322:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.939133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:49.939161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:49.951101Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:49.951194Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241447010015414:2050] [node 3] ICS01 socket: 538 reason# EndOfStream
2025-12-24T02:11:49.951205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241447010015313:2050] [node 3] ICP30 unregister session Session# [5:7587241447010015414:2050] VirtualId# [5:7587241447010015410:0]
2025-12-24T02:11:49.951212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241447010015414:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.951266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.951290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.951327Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:49.951385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241445534555314:2050] [node 3] ICS07 socket disconnect 506 reason# EndOfStream
2025-12-24T02:11:49.951389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241445534555314:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.951412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:49.951431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241445534555314:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:49.952028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241445534555216:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8350) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:49.952039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241445534555314:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:49.952044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241445534555216:2050] [node 3] ICP30 unregister session Session# [6:7587241445534555314:2050] VirtualId# [6:7587241445534555312:0]
2025-12-24T02:11:49.952097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:49.952104Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241445534555216:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:8350) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:49.952107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8350) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:49.962740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241445534555216:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:49.971810Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:49.971933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241445534555310:2065] [node 4] ICS01 socket: 483 reason# EndOfStream
2025-12-24T02:11:49.971946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241445534555235:2065] [node 4] ICP30 unregister session Session# [6:7587241445534555310:2065] VirtualId# [6:7587241445534555306:0]
2025-12-24T02:11:49.971955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241445534555310:2065] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:49.972016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:49.972042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:50.007477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241449719696121:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.007505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241449719696122:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.009275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241452723349629:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.009290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241452723349630:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.010474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241449192132428:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.010482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241449192132429:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.010488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241449192132430:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.009632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241452154282366:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:50.009640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241452154282367:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.012530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241452369839766:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.012539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241452369839767:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.012544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241452369839768:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.070991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241452464650905:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.071018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241452464650906:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.074872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241452464650904:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.075929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241449719696182:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.077311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241452154282426:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:50.077897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241452154282427:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53798
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.078937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839777:2058] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
2025-12-24T02:11:50.078705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241452154282432:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:50.078864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241449719696190:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.085108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241452154282436:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53798
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.085462Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349692:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.085485Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349693:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.085585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696197:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.090139Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696198:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
2025-12-24T02:11:50.090312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241452154282440:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53798
2025-12-24T02:11:50.095088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132466:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.099633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241452723349696:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:50.099765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241452723349697:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:50.098777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282436:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.099995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839777:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.100544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349691:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.099276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241452464650920:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:50.102396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282427:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.103328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241452154282445:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:50.104573Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241452723349629:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25023609ns) {Configure:104(25023070ns) {ProcessConfigured:135(24640197ns) {StartInitialHandshake:157(24639673ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(251ns) PrepareNewSessionHandshake:477(406ns) {SwitchToState:179(53ns)}}}}}
2025-12-24T02:11:50.104980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349691:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.106298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696200:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.107934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132499:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.106284Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241452154282450:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53798
2025-12-24T02:11:50.110531Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132466:2085] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.110608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241449192132429:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.112209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132499:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.112237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241449192132430:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.118719Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696200:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.123662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241452369839814:2060] [node 6] ICP01 ready to work
2025-12-24T02:11:50.123850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241452369839767:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.124438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241449192132488:2081] [node 5] ICP01 ready to work
2025-12-24T02:11:50.126678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349703:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.126743Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241452723349703:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.127035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282440:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.125001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696192:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.125019Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241449719696192:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.128768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839839:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.130747Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696202:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
2025-12-24T02:11:50.128895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650975:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.130138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650976:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.130883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282450:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.131108Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132506:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.133533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696197:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.133716Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696202:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.135485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241452464650976:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.135519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241452464650906:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.135684Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349707:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.135696Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241452723349707:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.136265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132506:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.136535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241452723349707:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.136637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241452723349696:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.136659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.136667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241452723349696:2107] [node 6] ICP22 created new session: [3:7587241452723349710:2107]
2025-12-24T02:11:50.136673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241452723349710:2107] [node 6] ICS09 handshake done sender: [3:7587241452723349707:2115] self: [3:7587241452723349698:0] peer: [6:7587241452464650978:0] socket: 505 qp: -1
2025-12-24T02:11:50.136684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241452723349710:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:50.136695Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.137241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650979:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.137412Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241452464650976:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.137468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241452464650906:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.137484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.137489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241452464650906:2050] [node 3] ICP22 created new session: [6:7587241452464650980:2050]
2025-12-24T02:11:50.137495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241452464650980:2050] [node 3] ICS09 handshake done sender: [6:7587241452464650976:2111] self: [6:7587241452464650978:0] peer: [3:7587241452723349698:0] socket: 512 qp: -1
2025-12-24T02:11:50.137500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241452464650980:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.137506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.137996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241449192132489:2084] [node 6] ICP01 ready to work
2025-12-24T02:11:50.138356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241449719696199:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:50.138908Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696184:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.138919Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241449719696184:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.138416Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241452723349700:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:50.139134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241452369839817:2077] [node 4] ICP01 ready to work
2025-12-24T02:11:50.138654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241452464650961:2077] [node 5] ICP01 ready to work
2025-12-24T02:11:50.143308Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241452369839814:2060] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19267079ns) {Configure:104(19266402ns) {ProcessConfigured:135(18867443ns) {StartInitialHandshake:157(18866582ns) {DropHandshakes:470(933ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(303ns)} GenerateSessionVirtualId:988(232ns) PrepareNewSessionHandshake:477(1042ns) {SwitchToState:179(494ns)}}}}}
2025-12-24T02:11:50.143433Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241449192132488:2081] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18859259ns) {Configure:104(18859061ns) {ProcessConfigured:135(18497764ns) {StartInitialHandshake:157(18497672ns) {DropHandshakes:470(352ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(44ns)} GenerateSessionVirtualId:988(185ns) PrepareNewSessionHandshake:477(1129ns) {SwitchToState:179(611ns)}}}}}
2025-12-24T02:11:50.148615Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241449719696192:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.147718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349693:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.148787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241449719696190:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.148811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.148819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241449719696190:2110] [node 4] ICP22 created new session: [2:7587241449719696206:2110]
2025-12-24T02:11:50.148827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241449719696206:2110] [node 4] ICS09 handshake done sender: [2:7587241449719696192:2111] self: [2:7587241449719696191:0] peer: [4:7587241449192132500:0] socket: 481 qp: -1
2025-12-24T02:11:50.148835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241449719696206:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:50.148845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.151500Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241452723349703:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.150007Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241452464650904:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(55961833ns) {Configure:104(55960234ns) {ProcessConfigured:135(54050799ns) {StartInitialHandshake:157(54049773ns) {DropHandshakes:470(254ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(1631ns) {SwitchToState:179(799ns)}}}}}
2025-12-24T02:11:50.154842Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241449719696199:2112] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16343897ns) {Configure:104(16343680ns) {ProcessConfigured:135(15949118ns) {StartInitialHandshake:157(15929525ns) {DropHandshakes:470(368ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(263ns) PrepareNewSessionHandshake:477(1328ns) {SwitchToState:179(447ns)}} IncomingHandshake:261(7675ns) {DropIncomingHandshake:442(24ns) PrepareNewSessionHandshake:477(187ns) {SwitchToState:179(159ns)} GenerateSessionVirtualId:988(907ns) IssueIncomingHandshakeReply:193(260ns)}}}}
2025-12-24T02:11:50.155353Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241449192132429:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241449192132466:2085] held: no
2025-12-24T02:11:50.157193Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241452464650961:2077] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18412865ns) {Configure:104(18412017ns) {ProcessConfigured:135(18045821ns) {StartInitialHandshake:157(18045436ns) {DropHandshakes:470(471ns) {DropIncomingHandshake:442(76ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(233ns) PrepareNewSessionHandshake:477(881ns) {SwitchToState:179(31ns)}}}}}
2025-12-24T02:11:50.156597Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241452723349630:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(55950181ns) {Configure:104(55948779ns) {ProcessConfigured:135(55665075ns) {StartInitialHandshake:157(55664481ns) {DropHandshakes:470(239ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(948ns) {SwitchToState:179(330ns)}}}}}
2025-12-24T02:11:50.156618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241452723349697:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.156634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.156643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241452723349697:2112] [node 4] ICP22 created new session: [3:7587241452723349715:2112]
2025-12-24T02:11:50.156649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241452723349715:2112] [node 4] ICS09 handshake done sender: [3:7587241452723349703:2114] self: [3:7587241452723349702:0] peer: [4:7587241449192132501:0] socket: 503 qp: -1
2025-12-24T02:11:50.156657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241452723349715:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:50.156703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.157771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241452369839817:2077] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18510100ns) {Configure:104(18509598ns) {ProcessConfigured:135(18191459ns) {StartInitialHandshake:157(18191153ns) {DropHandshakes:470(212ns) {DropIncomingHandshake:442(35ns) DropOutgoingHandshake:457(21ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(1024ns) {SwitchToState:179(128ns)}}}}}
2025-12-24T02:11:50.157412Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241452723349700:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18416813ns) {Configure:104(18416289ns) {ProcessConfigured:135(18084593ns) {StartInitialHandshake:157(18084507ns) {DropHandshakes:470(380ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(93ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:477(658ns) {SwitchToState:179(300ns)}}}}}
2025-12-24T02:11:50.157465Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241452723349696:2107] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:50.157510Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349717:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.163941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241449192132430:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14565) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241449192132499:2110] held: no
2025-12-24T02:11:50.166198Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241449719696184:2108] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.168752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241452154282453:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53798
2025-12-24T02:11:50.168983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282453:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.167560Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241449719696190:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(18654694ns) {HandleHandshakeStatus:329(18653807ns) {DropIncomingHandshake:442(5284ns) DropOutgoingHandshake:457(238ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(14717705ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14697514ns max# 14612059ns}}}
2025-12-24T02:11:50.167767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241449719696182:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.167783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.167790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241449719696182:2107] [node 5] ICP22 created new session: [2:7587241449719696213:2107]
2025-12-24T02:11:50.167796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241449719696213:2107] [node 5] ICS09 handshake done sender: [2:7587241449719696184:2108] self: [2:7587241449719696183:0] peer: [5:7587241452369839828:0] socket: 474 qp: -1
2025-12-24T02:11:50.167803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241449719696213:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:50.167811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.168231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.173113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839847:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
2025-12-24T02:11:50.168611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650989:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.173762Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241452369839767:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241452369839777:2058] held: no
2025-12-24T02:11:50.172326Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696212:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.175118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349717:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.175159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241452723349630:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.175527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241452464650989:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.175568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241452464650905:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.175788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839848:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.176244Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132513:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.176123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241452369839777:2058] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.176847Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241449192132466:2085] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.176935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241449192132429:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.176952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.176959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241449192132429:2049] [node 2] ICP22 created new session: [4:7587241449192132515:2049]
2025-12-24T02:11:50.176967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241449192132515:2049] [node 2] ICS09 handshake done sender: [4:7587241449192132466:2085] self: [4:7587241449192132500:0] peer: [2:7587241449719696191:0] socket: 482 qp: -1
2025-12-24T02:11:50.176975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241449192132515:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.176995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.177343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132514:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.175725Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696196:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.175740Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241449719696196:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.178810Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241449192132499:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.177740Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650991:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.178541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696208:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.178555Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241449719696208:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.179248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241449719696198:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.179289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241449719696208:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.179502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241449719696199:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.179518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.179522Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241449719696199:2112] [node 6] ICP22 created new session: [2:7587241449719696218:2112]
2025-12-24T02:11:50.179528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241449719696218:2112] [node 6] ICS09 handshake done sender: [2:7587241449719696208:2118] self: [2:7587241449719696204:0] peer: [6:7587241452464650990:0] socket: 492 qp: -1
2025-12-24T02:11:50.179534Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241449719696218:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:50.180072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282438:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.180081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241452154282438:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.180721Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241452154282438:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.180795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241452154282366:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.180814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.180819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241452154282366:2048] [node 2] ICP22 created new session: [1:7587241452154282456:2048]
2025-12-24T02:11:50.180824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241452154282456:2048] [node 2] ICS09 handshake done sender: [1:7587241452154282438:2112] self: [1:7587241452154282437:0] peer: [2:7587241449719696220:0] socket: 497 qp: -1
2025-12-24T02:11:50.180829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241452154282456:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:50.180834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.182461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241452464650975:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.179551Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.179959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241449719696121:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.182428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241449719696221:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54866
2025-12-24T02:11:50.182038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241449192132430:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.182055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.182062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241449192132430:2050] [node 3] ICP22 created new session: [4:7587241449192132518:2050]
2025-12-24T02:11:50.182070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241449192132518:2050] [node 3] ICS09 handshake done sender: [4:7587241449192132499:2110] self: [4:7587241449192132501:0] peer: [3:7587241452723349702:0] socket: 508 qp: -1
2025-12-24T02:11:50.182076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241449192132518:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.182094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.182175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839848:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.182209Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839839:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.182236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241452369839768:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.182246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241452369839766:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.182398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241452369839767:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.182410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.182417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241452369839767:2049] [node 2] ICP22 created new session: [5:7587241452369839851:2049]
2025-12-24T02:11:50.182422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241452369839851:2049] [node 2] ICS09 handshake done sender: [5:7587241452369839777:2058] self: [5:7587241452369839828:0] peer: [2:7587241449719696183:0] socket: 478 qp: -1
2025-12-24T02:11:50.182428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241452369839851:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.182435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.182562Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349718:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.182575Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241452723349718:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.183308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241452723349718:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.183413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241452723349700:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.183434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.183441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241452723349700:2111] [node 5] ICP22 created new session: [3:7587241452723349724:2111]
2025-12-24T02:11:50.183447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241452723349724:2111] [node 5] ICS09 handshake done sender: [3:7587241452723349718:2118] self: [3:7587241452723349712:0] peer: [5:7587241452369839849:0] socket: 526 qp: -1
2025-12-24T02:11:50.183453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241452723349724:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:50.183460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.184390Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839854:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.182676Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241452464650905:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241452464650989:2118] held: no
2025-12-24T02:11:50.182866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241452464650904:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.186934Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241449719696198:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.187096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650992:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.187237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241452369839848:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.187150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282429:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.187159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241452154282429:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.189097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839853:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
2025-12-24T02:11:50.189174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241452369839768:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.189185Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.189191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241452369839768:2050] [node 3] ICP22 created new session: [5:7587241452369839855:2050]
2025-12-24T02:11:50.189198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241452369839855:2050] [node 3] ICS09 handshake done sender: [5:7587241452369839848:2118] self: [5:7587241452369839849:0] peer: [3:7587241452723349712:0] socket: 514 qp: -1
2025-12-24T02:11:50.189204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241452369839855:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.189221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.190101Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241452464650989:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.190249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241452464650991:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.190279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241452464650920:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.190356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241452464650905:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.190366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.190372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241452464650905:2049] [node 2] ICP22 created new session: [6:7587241452464650995:2049]
2025-12-24T02:11:50.190376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241452464650995:2049] [node 2] ICS09 handshake done sender: [6:7587241452464650989:2118] self: [6:7587241452464650990:0] peer: [2:7587241449719696204:0] socket: 527 qp: -1
2025-12-24T02:11:50.190381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241452464650995:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.190388Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.190801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282434:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.190807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241452154282434:2110] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.192036Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241452154282434:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.192197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241452154282432:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.192213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.192221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241452154282432:2109] [node 5] ICP22 created new session: [1:7587241452154282460:2109]
2025-12-24T02:11:50.192226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241452154282460:2109] [node 5] ICS09 handshake done sender: [1:7587241452154282434:2110] self: [1:7587241452154282433:0] peer: [5:7587241452369839850:0] socket: 479 qp: -1
2025-12-24T02:11:50.192231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241452154282460:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:50.192238Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.193461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349726:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.193489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349727:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.191831Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132520:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.196003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132520:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.196053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241449192132428:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.194838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839853:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.197793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241452723349717:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.197477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241449719696196:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.197509Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241449719696121:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241449719696198:2115] held: no
2025-12-24T02:11:50.197635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241449719696121:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.197645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.197652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241449719696121:2048] [node 1] ICP22 created new session: [2:7587241449719696223:2048]
2025-12-24T02:11:50.197658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241449719696223:2048] [node 1] ICS09 handshake done sender: [2:7587241449719696198:2115] self: [2:7587241449719696220:0] peer: [1:7587241452154282437:0] socket: 498 qp: -1
2025-12-24T02:11:50.197664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241449719696223:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.197676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.197897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241452723349630:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.197909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.197915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241452723349630:2049] [node 2] ICP22 created new session: [3:7587241452723349728:2049]
2025-12-24T02:11:50.197922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241452723349728:2049] [node 2] ICS09 handshake done sender: [3:7587241452723349717:2117] self: [3:7587241452723349722:0] peer: [2:7587241449719696195:0] socket: 524 qp: -1
2025-12-24T02:11:50.197927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241452723349728:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.197933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.198169Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241452154282429:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.198262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241452154282426:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.198275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.198280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241452154282426:2106] [node 6] ICP22 created new session: [1:7587241452154282464:2106]
2025-12-24T02:11:50.198286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241452154282464:2106] [node 6] ICS09 handshake done sender: [1:7587241452154282429:2108] self: [1:7587241452154282428:0] peer: [6:7587241452464650993:0] socket: 477 qp: -1
2025-12-24T02:11:50.198291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241452154282464:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:50.198306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.199723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132510:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.199736Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241449192132510:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.200639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241449719696122:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.200668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.200675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241449719696122:2049] [node 3] ICP22 created new session: [2:7587241449719696225:2049]
2025-12-24T02:11:50.200681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241449719696225:2049] [node 3] ICS09 handshake done sender: [2:7587241449719696196:2113] self: [2:7587241449719696195:0] peer: [3:7587241452723349722:0] socket: 490 qp: -1
2025-12-24T02:11:50.200688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241449719696225:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:50.200705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.202468Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241449192132510:2117] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.202587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241449192132489:2084] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.202609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.202616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241449192132489:2084] [node 6] ICP22 created new session: [4:7587241449192132523:2084]
2025-12-24T02:11:50.202623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241449192132523:2084] [node 6] ICS09 handshake done sender: [4:7587241449192132510:2117] self: [4:7587241449192132508:0] peer: [6:7587241452464650994:0] socket: 487 qp: -1
2025-12-24T02:11:50.202629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241449192132523:2084] [node 6] ICS10 traffic start
2025-12-24T02:11:50.202636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.206848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650999:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.210315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464650998:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.210281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282447:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.210300Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241452154282447:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.207033Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241452369839766:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241452369839839:2110] held: no
2025-12-24T02:11:50.207460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241452369839817:2077] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.207498Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839857:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.212378Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241449192132509:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.212393Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241449192132509:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.213357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132525:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.211313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839858:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
2025-12-24T02:11:50.211546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241452369839839:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.214282Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241452369839768:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(24997591ns) {HandleHandshakeStatus:329(24997129ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(1173ns) SwitchToState:179(66ns) ProcessPendingSessionEvents:495(14574481ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14547585ns max# 14471207ns}}}
2025-12-24T02:11:50.214636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241452369839862:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31502
2025-12-24T02:11:50.215705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839858:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.216148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241449192132509:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.216593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241449192132488:2081] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.216605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.216616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241449192132488:2081] [node 5] ICP22 created new session: [4:7587241449192132527:2081]
2025-12-24T02:11:50.216624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241449192132527:2081] [node 5] ICS09 handshake done sender: [4:7587241449192132509:2116] self: [4:7587241449192132502:0] peer: [5:7587241452369839860:0] socket: 501 qp: -1
2025-12-24T02:11:50.216630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241449192132527:2081] [node 5] ICS10 traffic start
2025-12-24T02:11:50.216638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.215993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241452369839766:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.216012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.216018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241452369839766:2048] [node 1] ICP22 created new session: [5:7587241452369839864:2048]
2025-12-24T02:11:50.216026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241452369839864:2048] [node 1] ICS09 handshake done sender: [5:7587241452369839839:2110] self: [5:7587241452369839850:0] peer: [1:7587241452154282433:0] socket: 484 qp: -1
2025-12-24T02:11:50.216033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241452369839864:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.216041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.218167Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241452369839853:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.215502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241452723349727:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.215782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241452723349629:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.217911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241452723349731:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58680
2025-12-24T02:11:50.218358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241452723349727:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.218418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241452723349629:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.218430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.218438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241452723349629:2048] [node 1] ICP22 created new session: [3:7587241452723349732:2048]
2025-12-24T02:11:50.218444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241452723349732:2048] [node 1] ICS09 handshake done sender: [3:7587241452723349727:2120] self: [3:7587241452723349730:0] peer: [1:7587241452154282442:0] socket: 537 qp: -1
2025-12-24T02:11:50.218450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241452723349732:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.218456Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.220058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241452369839817:2077] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.220073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.220079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241452369839817:2077] [node 4] ICP22 created new session: [5:7587241452369839866:2077]
2025-12-24T02:11:50.220085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241452369839866:2077] [node 4] ICS09 handshake done sender: [5:7587241452369839853:2119] self: [5:7587241452369839860:0] peer: [4:7587241449192132502:0] socket: 528 qp: -1
2025-12-24T02:11:50.220091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241452369839866:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:50.220097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.219452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241452464650904:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241452464650975:2110] held: no
2025-12-24T02:11:50.219575Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241452464650991:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.222890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241452464650920:2059] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241452464650991:2119] held: no
2025-12-24T02:11:50.222990Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241452464650920:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.222999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.223004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241452464650920:2059] [node 4] ICP22 created new session: [6:7587241452464651001:2059]
2025-12-24T02:11:50.223011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241452464651001:2059] [node 4] ICS09 handshake done sender: [6:7587241452464650991:2119] self: [6:7587241452464650994:0] peer: [4:7587241449192132508:0] socket: 530 qp: -1
2025-12-24T02:11:50.223017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241452464651001:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:50.223026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.224453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241449192132529:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8472
2025-12-24T02:11:50.226235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241449192132520:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.216989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241452154282449:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.217000Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241452154282449:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.218263Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241452154282449:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.224758Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241452154282426:2106] [node 6] ICP35 event processing took too much time PendingConnection:279(26392704ns) {HandleHandshakeStatus:329(26392016ns) {DropIncomingHandshake:442(2571ns) DropOutgoingHandshake:457(88ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(14274629ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14256227ns max# 14183027ns}}}
2025-12-24T02:11:50.224835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241452154282447:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.226535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241449192132428:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.226553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.226560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241449192132428:2048] [node 1] ICP22 created new session: [4:7587241449192132530:2048]
2025-12-24T02:11:50.226566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241449192132530:2048] [node 1] ICS09 handshake done sender: [4:7587241449192132520:2120] self: [4:7587241449192132521:0] peer: [1:7587241452154282446:0] socket: 536 qp: -1
2025-12-24T02:11:50.226572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241449192132530:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.226579Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.227677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241452154282367:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.227700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.227708Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241452154282367:2049] [node 3] ICP22 created new session: [1:7587241452154282468:2049]
2025-12-24T02:11:50.227714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241452154282468:2049] [node 3] ICS09 handshake done sender: [1:7587241452154282449:2117] self: [1:7587241452154282442:0] peer: [3:7587241452723349730:0] socket: 499 qp: -1
2025-12-24T02:11:50.227720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241452154282468:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:50.227726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.228419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241452154282445:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.228436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.228442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241452154282445:2115] [node 4] ICP22 created new session: [1:7587241452154282470:2115]
2025-12-24T02:11:50.228447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241452154282470:2115] [node 4] ICS09 handshake done sender: [1:7587241452154282447:2116] self: [1:7587241452154282446:0] peer: [4:7587241449192132521:0] socket: 532 qp: -1
2025-12-24T02:11:50.228452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241452154282470:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:50.228459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.231019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241452464650975:2110] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.231494Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241449192132488:2081] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:50.231611Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464651000:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.231669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241452464650904:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.231679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.231685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241452464650904:2048] [node 1] ICP22 created new session: [6:7587241452464651003:2048]
2025-12-24T02:11:50.231690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241452464651003:2048] [node 1] ICS09 handshake done sender: [6:7587241452464650975:2110] self: [6:7587241452464650993:0] peer: [1:7587241452154282428:0] socket: 500 qp: -1
2025-12-24T02:11:50.231696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241452464651003:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.231710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.232342Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241452464651000:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.232385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241452464650961:2077] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.232473Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241452369839845:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.232484Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241452369839845:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.233160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241452369839845:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.233280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241452369839814:2060] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.233296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.233303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241452369839814:2060] [node 6] ICP22 created new session: [5:7587241452369839869:2060]
2025-12-24T02:11:50.233309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241452369839869:2060] [node 6] ICS09 handshake done sender: [5:7587241452369839845:2115] self: [5:7587241452369839830:0] peer: [6:7587241452464651005:0] socket: 542 qp: -1
2025-12-24T02:11:50.233314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241452369839869:2060] [node 6] ICS10 traffic start
2025-12-24T02:11:50.233325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.234718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241452464650961:2077] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3707) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241452464651000:2123] held: no
2025-12-24T02:11:50.234887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241452464651006:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8987
2025-12-24T02:11:50.235092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241452464651000:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.235154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241452464650961:2077] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.235172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.235180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241452464650961:2077] [node 5] ICP22 created new session: [6:7587241452464651007:2077]
2025-12-24T02:11:50.235186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241452464651007:2077] [node 5] ICS09 handshake done sender: [6:7587241452464651000:2123] self: [6:7587241452464651005:0] peer: [5:7587241452369839830:0] socket: 543 qp: -1
2025-12-24T02:11:50.235192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241452464651007:2077] [node 5] ICS10 traffic start
2025-12-24T02:11:50.235208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.246613Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:50.247067Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:50.247132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241452723349732:2048] [node 1] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:50.247137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241452723349629:2048] [node 1] ICP30 unregister session Session# [3:7587241452723349732:2048] VirtualId# [3:7587241452723349730:0]
2025-12-24T02:11:50.247141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241452723349732:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.247169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.247191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.246896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241449719696223:2048] [node 1] ICS01 socket: 498 reason# EndOfStream
2025-12-24T02:11:50.246912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241449719696121:2048] [node 1] ICP30 unregister session Session# [2:7587241449719696223:2048] VirtualId# [2:7587241449719696220:0]
2025-12-24T02:11:50.246920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241449719696223:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.246975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.247001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.258887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:50.258988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241452464651003:2048] [node 1] ICS01 socket: 500 reason# EndOfStream
2025-12-24T02:11:50.258997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241452464650904:2048] [node 1] ICP30 unregister session Session# [6:7587241452464651003:2048] VirtualId# [6:7587241452464650993:0]
2025-12-24T02:11:50.259005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241452464651003:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.259068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.259097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.259304Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:50.259362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241449192132530:2048] [node 1] ICS01 socket: 536 reason# EndOfStream
2025-12-24T02:11:50.259367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241449192132428:2048] [node 1] ICP30 unregister session Session# [4:7587241449192132530:2048] VirtualId# [4:7587241449192132521:0]
2025-12-24T02:11:50.259372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241449192132530:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.259399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.259419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.259141Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:50.259203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241452369839864:2048] [node 1] ICS01 socket: 484 reason# EndOfStream
2025-12-24T02:11:50.259208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241452369839766:2048] [node 1] ICP30 unregister session Session# [5:7587241452369839864:2048] VirtualId# [5:7587241452369839850:0]
2025-12-24T02:11:50.259212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241452369839864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.259244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.259265Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.265421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:50.265532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241452723349728:2049] [node 2] ICS07 socket disconnect 524 reason# EndOfStream
2025-12-24T02:11:50.265539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241452723349728:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.265596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.265616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241452723349728:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:50.268144Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241452723349630:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:50.268159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241452723349728:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:50.268470Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241452723349630:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:50.268483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241452723349728:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:50.268489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241452723349630:2049] [node 2] ICP30 unregister session Session# [3:7587241452723349728:2049] VirtualId# [3:7587241452723349722:0]
2025-12-24T02:11:50.268546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.268552Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241452723349630:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.268555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21206) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.274071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:50.274163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241452369839851:2049] [node 2] ICS01 socket: 478 reason# EndOfStream
2025-12-24T02:11:50.274172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241452369839767:2049] [node 2] ICP30 unregister session Session# [5:7587241452369839851:2049] VirtualId# [5:7587241452369839828:0]
2025-12-24T02:11:50.274180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241452369839851:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.274245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.274270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.282758Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:50.282850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241449192132515:2049] [node 2] ICS01 socket: 482 reason# EndOfStream
2025-12-24T02:11:50.282859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241449192132429:2049] [node 2] ICP30 unregister session Session# [4:7587241449192132515:2049] VirtualId# [4:7587241449192132500:0]
2025-12-24T02:11:50.282866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241449192132515:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.282930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.282958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.283009Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:50.283073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241452464650995:2049] [node 2] ICS01 socket: 527 reason# EndOfStream
2025-12-24T02:11:50.283078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241452464650905:2049] [node 2] ICP30 unregister session Session# [6:7587241452464650995:2049] VirtualId# [6:7587241452464650990:0]
2025-12-24T02:11:50.283083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241452464650995:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.283116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.283140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.314252Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:50.314374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241452464650980:2050] [node 3] ICS01 socket: 512 reason# EndOfStream
2025-12-24T02:11:50.314386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241452464650906:2050] [node 3] ICP30 unregister session Session# [6:7587241452464650980:2050] VirtualId# [6:7587241452464650978:0]
2025-12-24T02:11:50.314395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241452464650980:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.314464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:50.314492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:50.319033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:50.319143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241452369839855:2050] [node 3] ICS07 socket disconnect 514 reason# EndOfStream
2025-12-24T02:11:50.319150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241452369839855:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.319212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:50.319238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241452369839855:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:50.320287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241452369839768:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14565) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:50.320302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241452369839855:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:50.320307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241452369839768:2050] [node 3] ICP30 unregister session Session# [5:7587241452369839855:2050] VirtualId# [5:7587241452369839849:0]
2025-12-24T02:11:50.320361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:50.320368Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241452369839768:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.320371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14565) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.333898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241452369839768:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:50.334779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:50.334885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241452369839866:2077] [node 4] ICS07 socket disconnect 528 reason# EndOfStream
2025-12-24T02:11:50.334904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241452369839866:2077] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.334947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:50.334960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241452369839866:2077] [node 4] ICS15 start handshake
2025-12-24T02:11:50.335435Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241452369839817:2077] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:50.335450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241452369839866:2077] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:50.335457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241452369839817:2077] [node 4] ICP30 unregister session Session# [5:7587241452369839866:2077] VirtualId# [5:7587241452369839860:0]
2025-12-24T02:11:50.335517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:50.335524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241452369839817:2077] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.335527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.337896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:50.337985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241452464651001:2059] [node 4] ICS01 socket: 530 reason# EndOfStream
2025-12-24T02:11:50.337993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241452464650920:2059] [node 4] ICP30 unregister session Session# [6:7587241452464651001:2059] VirtualId# [6:7587241452464650994:0]
2025-12-24T02:11:50.337999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241452464651001:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.338045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:50.338068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:50.411171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449728653815:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:50.411181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449728653816:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.431028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241452443585006:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.431052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241452443585007:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.435112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241449562771256:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.435122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241449562771257:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.435128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241449562771258:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.426635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241450494977037:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.426643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241450494977038:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.435449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450008012204:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.435459Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450008012205:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.435465Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450008012206:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.458821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241451467775233:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.458837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241451467775234:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.458852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241451467775235:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.518808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241452443585067:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:50.519105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241449562771290:2079] [node 6] ICP01 ready to work
2025-12-24T02:11:50.519429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241452443585069:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:50.519600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241449562771294:2083] [node 4] ICP01 ready to work
2025-12-24T02:11:50.519450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449728653876:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.520443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771295:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.522027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450008012265:2106] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.522590Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012266:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.521100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241451467775250:2061] [node 4] ICP01 ready to work
2025-12-24T02:11:50.521818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977094:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
2025-12-24T02:11:50.521844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977095:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
::ffff:127.1.0.0
2025-12-24T02:11:50.522986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241450494977098:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:50.523254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977093:2103] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
2025-12-24T02:11:50.522895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241451467775258:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:50.521555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449728653884:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.538589Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241452443585067:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19694480ns) {Configure:104(19692819ns) {ProcessConfigured:135(15259045ns) {StartInitialHandshake:157(15258197ns) {DropHandshakes:470(397ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(18ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(1199ns) {SwitchToState:179(1073ns)}}}}}
2025-12-24T02:11:50.538601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977093:2103] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.539032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241450494977037:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.538857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241452443585077:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49172
2025-12-24T02:11:50.539264Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241452443585069:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19778209ns) {Configure:104(19777696ns) {ProcessConfigured:135(15785826ns) {StartInitialHandshake:157(15785132ns) {DropHandshakes:470(278ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(152ns) PrepareNewSessionHandshake:477(548ns) {SwitchToState:179(228ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.540202Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241449562771295:2084] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.540413Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977107:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
2025-12-24T02:11:50.539302Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653878:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.539316Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241449728653878:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.543072Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653881:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.543082Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241449728653881:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.543083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241449728653878:2108] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.543744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977095:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.541016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012272:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.543015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012272:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.543877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241449728653881:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.546807Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241449562771290:2079] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27601268ns) {Configure:104(27600835ns) {ProcessConfigured:135(22677769ns) {StartInitialHandshake:157(22677100ns) {DropHandshakes:470(402ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(146ns) PrepareNewSessionHandshake:477(1190ns) {SwitchToState:179(613ns)}}}}}
2025-12-24T02:11:50.551087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771326:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.551164Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771327:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.551056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449728653889:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:50.551163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449728653816:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.551183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.551191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449728653816:2049] [node 3] ICP22 created new session: [1:7587241449728653893:2049]
2025-12-24T02:11:50.551198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241449728653893:2049] [node 3] ICS09 handshake done sender: [1:7587241449728653878:2108] self: [1:7587241449728653877:0] peer: [3:7587241450494977106:0] socket: 480 qp: -1
2025-12-24T02:11:50.551208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241449728653893:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:50.551232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.552991Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977104:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.553016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241450494977104:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.553729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977094:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.553884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241450494977110:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:50.554220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241450494977110:2114] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.554493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012271:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.563568Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241451467775233:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32176772ns) {Configure:104(32176216ns) {ProcessConfigured:135(31878621ns) {StartInitialHandshake:157(31878396ns) {DropHandshakes:470(429ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(706ns) {SwitchToState:179(340ns)}}}}}
2025-12-24T02:11:50.564143Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775310:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.564168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775311:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.564187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775312:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.560956Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241449728653815:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29798287ns) {Configure:104(29797114ns) {ProcessConfigured:135(29423578ns) {StartInitialHandshake:157(29422805ns) {DropHandshakes:470(509ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(575ns) PrepareNewSessionHandshake:477(1479ns) {SwitchToState:179(674ns)}}}}}
2025-12-24T02:11:50.569899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241449562771326:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.569818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012271:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.570432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450008012206:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.570482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241450008012206:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27393703ns) {Configure:104(27393401ns) {ProcessConfigured:135(27046089ns) {IncomingHandshake:261(27030898ns) {DropIncomingHandshake:442(78ns) PrepareNewSessionHandshake:477(365ns) {SwitchToState:179(153ns)} GenerateSessionVirtualId:988(353ns) IssueIncomingHandshakeReply:193(27025269ns) {SwitchToState:179(747ns)}}}}}
2025-12-24T02:11:50.570517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241451467775246:2059] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.570534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241451467775246:2059] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.570813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241449562771256:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.570895Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241449562771256:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(30595879ns) {Configure:104(30595577ns) {ProcessConfigured:135(30149093ns) {IncomingHandshake:261(30137681ns) {DropIncomingHandshake:442(91ns) PrepareNewSessionHandshake:477(305ns) {SwitchToState:179(108ns)} GenerateSessionVirtualId:988(377ns) IssueIncomingHandshakeReply:193(30133141ns) {SwitchToState:179(1192ns)}}}}}
2025-12-24T02:11:50.570806Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241452443585082:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49172
2025-12-24T02:11:50.571060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241449562771327:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.571582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241449562771257:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.571791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585082:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.574771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241452443585007:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51856070ns) {Configure:104(51855513ns) {ProcessConfigured:135(51581648ns) {StartInitialHandshake:157(51581250ns) {DropHandshakes:470(269ns) {DropIncomingHandshake:442(24ns) DropOutgoingHandshake:457(17ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(1041ns) {SwitchToState:179(448ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.574829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241451467775312:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.574865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241451467775310:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.574973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241451467775234:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.576800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653895:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.574116Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771333:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.574428Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241449562771294:2083] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(54759682ns) {Configure:104(54759122ns) {ProcessConfigured:135(50430751ns) {StartInitialHandshake:157(50429987ns) {DropHandshakes:470(259ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(607ns) {SwitchToState:179(299ns)}}}}}
2025-12-24T02:11:50.574529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241449562771294:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.576983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241450494977104:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.576869Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241449728653895:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.577308Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653897:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.577320Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241449728653897:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.577043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585079:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.577054Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241452443585079:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.577746Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585076:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.577761Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241452443585076:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.575236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012275:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.575362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241450008012204:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22698930ns) {Configure:104(22696769ns) {ProcessConfigured:135(22317300ns) {StartInitialHandshake:157(22316886ns) {DropHandshakes:470(703ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(103ns)} GenerateSessionVirtualId:988(389ns) PrepareNewSessionHandshake:477(666ns) {SwitchToState:179(254ns)}}}}}
2025-12-24T02:11:50.575462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450008012204:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.576999Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012268:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.577006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241450008012268:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.579140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241449728653816:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(27902182ns) {HandleHandshakeStatus:329(27901290ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(378ns) SwitchToState:179(281ns) ProcessPendingSessionEvents:495(9108550ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 9093076ns max# 9034977ns}}}
2025-12-24T02:11:50.579500Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241451467775258:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(56435184ns) {Configure:104(52736317ns) {ProcessConfigured:135(52442796ns) {StartInitialHandshake:157(52441944ns) {DropHandshakes:470(304ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(424ns) PrepareNewSessionHandshake:477(11732ns) {SwitchToState:179(443ns)}}}}}
2025-12-24T02:11:50.578266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241449562771295:2084] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.579812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241449562771256:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.579832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.579839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241449562771256:2048] [node 1] ICP22 created new session: [5:7587241449562771337:2048]
2025-12-24T02:11:50.579846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241449562771337:2048] [node 1] ICS09 handshake done sender: [5:7587241449562771295:2084] self: [5:7587241449562771325:0] peer: [1:7587241449728653880:0] socket: 482 qp: -1
2025-12-24T02:11:50.579853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241449562771337:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.579861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.580274Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241451467775311:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.580300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241451467775258:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.580688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241451467775246:2059] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.580808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241451467775235:2050] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.580826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.580834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241451467775235:2050] [node 3] ICP22 created new session: [6:7587241451467775318:2050]
2025-12-24T02:11:50.580840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241451467775318:2050] [node 3] ICS09 handshake done sender: [6:7587241451467775246:2059] self: [6:7587241451467775245:0] peer: [3:7587241450494977111:0] socket: 483 qp: -1
2025-12-24T02:11:50.580846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241451467775318:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.580852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.580461Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241449562771328:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.580467Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241449562771328:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.581272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241449562771328:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.581356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241449562771290:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.581371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.581377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241449562771290:2079] [node 6] ICP22 created new session: [5:7587241449562771340:2079]
2025-12-24T02:11:50.581383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241449562771340:2079] [node 6] ICS09 handshake done sender: [5:7587241449562771328:2112] self: [5:7587241449562771304:0] peer: [6:7587241451467775316:0] socket: 511 qp: -1
2025-12-24T02:11:50.581389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241449562771340:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:50.581407Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.579556Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012266:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.579603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450008012280:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:50.580056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012275:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.587488Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241452443585086:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49172
2025-12-24T02:11:50.589816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585077:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.590505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241452443585006:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.590544Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241452443585006:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18559891ns) {IncomingHandshake:261(18559502ns) {DropIncomingHandshake:442(51ns) PrepareNewSessionHandshake:477(558ns) {SwitchToState:179(216ns)} GenerateSessionVirtualId:988(277ns) IssueIncomingHandshakeReply:193(18553325ns) {SwitchToState:179(210ns)}}}
2025-12-24T02:11:50.586911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241449728653900:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52600
2025-12-24T02:11:50.586933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241449728653901:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52600
2025-12-24T02:11:50.586943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241449728653902:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52600
2025-12-24T02:11:50.589743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653900:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.589090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241451467775233:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.590921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653886:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.590934Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241449728653886:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.595256Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241450494977093:2103] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.597588Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.597820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241449728653895:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.597920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449728653815:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.597946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.597955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449728653815:2048] [node 2] ICP22 created new session: [1:7587241449728653905:2048]
2025-12-24T02:11:50.597962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241449728653905:2048] [node 2] ICS09 handshake done sender: [1:7587241449728653895:2113] self: [1:7587241449728653890:0] peer: [2:7587241452443585084:0] socket: 513 qp: -1
2025-12-24T02:11:50.597969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241449728653905:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:50.597980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.598751Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241449562771326:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.598546Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241450008012268:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.598874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241449562771294:2083] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.598888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.598895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241449562771294:2083] [node 4] ICP22 created new session: [5:7587241449562771343:2083]
2025-12-24T02:11:50.598900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241449562771343:2083] [node 4] ICS09 handshake done sender: [5:7587241449562771326:2110] self: [5:7587241449562771335:0] peer: [4:7587241450008012267:0] socket: 496 qp: -1
2025-12-24T02:11:50.598905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241449562771343:2083] [node 4] ICS10 traffic start
2025-12-24T02:11:50.598923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.598844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241450494977037:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.598867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.598875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241450494977037:2048] [node 1] ICP22 created new session: [3:7587241450494977117:2048]
2025-12-24T02:11:50.598880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241450494977117:2048] [node 1] ICS09 handshake done sender: [3:7587241450494977093:2103] self: [3:7587241450494977106:0] peer: [1:7587241449728653877:0] socket: 487 qp: -1
2025-12-24T02:11:50.598886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241450494977117:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.598908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.599828Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241452443585006:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30925) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241452443585082:2114] held: no
2025-12-24T02:11:50.607170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241452443585079:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.607275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241452443585069:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.607296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.607306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241452443585069:2108] [node 6] ICP22 created new session: [2:7587241452443585089:2108]
2025-12-24T02:11:50.607313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241452443585089:2108] [node 6] ICS09 handshake done sender: [2:7587241452443585079:2113] self: [2:7587241452443585074:0] peer: [6:7587241451467775314:0] socket: 502 qp: -1
2025-12-24T02:11:50.607320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241452443585089:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:50.607348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.610436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241450008012265:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.610476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.610483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450008012265:2106] [node 5] ICP22 created new session: [4:7587241450008012286:2106]
2025-12-24T02:11:50.610491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241450008012286:2106] [node 5] ICS09 handshake done sender: [4:7587241450008012268:2109] self: [4:7587241450008012267:0] peer: [5:7587241449562771335:0] socket: 492 qp: -1
2025-12-24T02:11:50.610498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241450008012286:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:50.610532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.612201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241449728653897:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.612337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449728653889:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.612353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.612362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449728653889:2112] [node 6] ICP22 created new session: [1:7587241449728653908:2112]
2025-12-24T02:11:50.612368Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241449728653908:2112] [node 6] ICS09 handshake done sender: [1:7587241449728653897:2114] self: [1:7587241449728653896:0] peer: [6:7587241451467775313:0] socket: 514 qp: -1
2025-12-24T02:11:50.612376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241449728653908:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:50.612384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.610797Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977116:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.618585Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771345:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.620817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241452443585076:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.620955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241449562771327:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.619399Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012285:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.619818Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241450008012280:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(40138646ns) {Configure:104(40138224ns) {ProcessConfigured:135(39714835ns) {StartInitialHandshake:157(39696500ns) {DropHandshakes:470(618ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(483ns) PrepareNewSessionHandshake:477(1083ns) {SwitchToState:179(236ns)}} IncomingHandshake:261(8062ns) {DropIncomingHandshake:442(44ns) PrepareNewSessionHandshake:477(190ns) {SwitchToState:179(143ns)} GenerateSessionVirtualId:988(308ns) IssueIncomingHandshakeReply:193(573ns)}}}}
2025-12-24T02:11:50.620188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241450008012205:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(36878618ns) {Configure:104(36877270ns) {ProcessConfigured:135(36541795ns) {StartInitialHandshake:157(36541457ns) {DropHandshakes:470(349ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(1007ns) {SwitchToState:179(316ns)}}}}}
2025-12-24T02:11:50.622847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241450494977108:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:50.622959Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241450494977037:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(24001049ns) {HandleHandshakeStatus:329(24000436ns) {DropIncomingHandshake:442(145ns) DropOutgoingHandshake:457(249ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(23880832ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 23861855ns max# 23774592ns}}}
2025-12-24T02:11:50.622992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241450494977098:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.623009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.623018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241450494977098:2107] [node 4] ICP22 created new session: [3:7587241450494977119:2107]
2025-12-24T02:11:50.623027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241450494977119:2107] [node 4] ICS09 handshake done sender: [3:7587241450494977104:2112] self: [3:7587241450494977099:0] peer: [4:7587241450008012273:0] socket: 501 qp: -1
2025-12-24T02:11:50.623035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241450494977119:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:50.622719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449728653876:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.622750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.622763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449728653876:2107] [node 5] ICP22 created new session: [1:7587241449728653910:2107]
2025-12-24T02:11:50.622772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241449728653910:2107] [node 5] ICS09 handshake done sender: [1:7587241449728653881:2109] self: [1:7587241449728653880:0] peer: [5:7587241449562771325:0] socket: 481 qp: -1
2025-12-24T02:11:50.622781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241449728653910:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:50.622808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.623056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.622720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241452443585081:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:50.622777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241452443585067:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.622788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.622794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241452443585067:2107] [node 5] ICP22 created new session: [2:7587241452443585093:2107]
2025-12-24T02:11:50.622801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241452443585093:2107] [node 5] ICS09 handshake done sender: [2:7587241452443585076:2111] self: [2:7587241452443585073:0] peer: [5:7587241449562771332:0] socket: 500 qp: -1
2025-12-24T02:11:50.622807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241452443585093:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:50.622826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.630572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653901:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.628499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241450008012272:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.631313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450008012206:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.631336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.631344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450008012206:2050] [node 3] ICP22 created new session: [4:7587241450008012293:2050]
2025-12-24T02:11:50.631351Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241450008012293:2050] [node 3] ICS09 handshake done sender: [4:7587241450008012272:2112] self: [4:7587241450008012273:0] peer: [3:7587241450494977099:0] socket: 507 qp: -1
2025-12-24T02:11:50.631359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241450008012293:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.631369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.634812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241449562771257:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.634833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.634841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241449562771257:2049] [node 2] ICP22 created new session: [5:7587241449562771346:2049]
2025-12-24T02:11:50.634847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241449562771346:2049] [node 2] ICS09 handshake done sender: [5:7587241449562771327:2111] self: [5:7587241449562771332:0] peer: [2:7587241452443585073:0] socket: 509 qp: -1
2025-12-24T02:11:50.634853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241449562771346:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.634862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.635414Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241449562771256:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(55560312ns) {HandleHandshakeStatus:329(55559720ns) {DropIncomingHandshake:442(321ns) DropOutgoingHandshake:457(215ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(44916940ns) {ForwardSessionEventToSession:612(44904988ns) ForwardSessionEventToSession:612(1403ns) ForwardSessionEventToSession:612(1236ns) ForwardSessionEventToSession:612(671ns) ForwardSessionEventToSession:612(654ns) ForwardSessionEventToSession:612(1461ns) ForwardSessionEventToSession:612(954ns) ForwardSessionEventToSession:612(820ns) ForwardSessionEventToSession:612(603ns) ForwardSessionEventToSession:612(873ns)}}}
2025-12-24T02:11:50.636361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241450494977094:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.636674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241451467775235:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(55744483ns) {HandleHandshakeStatus:329(55744083ns) {DropIncomingHandshake:442(46ns) DropOutgoingHandshake:457(283ns) SwitchToState:179(62ns) ProcessPendingSessionEvents:495(45436390ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 45414548ns max# 45334214ns}}}
2025-12-24T02:11:50.637093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241450494977110:2114] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.637104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.637110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241450494977110:2114] [node 6] ICP22 created new session: [3:7587241450494977121:2114]
2025-12-24T02:11:50.637116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241450494977121:2114] [node 6] ICS09 handshake done sender: [3:7587241450494977094:2104] self: [3:7587241450494977111:0] peer: [6:7587241451467775245:0] socket: 488 qp: -1
2025-12-24T02:11:50.637121Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241450494977121:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:50.637141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.638084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241449728653886:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.638985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449728653884:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.638999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.639007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449728653884:2110] [node 4] ICP22 created new session: [1:7587241449728653914:2110]
2025-12-24T02:11:50.639012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241449728653914:2110] [node 4] ICS09 handshake done sender: [1:7587241449728653886:2111] self: [1:7587241449728653885:0] peer: [4:7587241450008012278:0] socket: 493 qp: -1
2025-12-24T02:11:50.639018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241449728653914:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:50.639025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.640514Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585086:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.642195Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241452443585095:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49172
2025-12-24T02:11:50.643093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241452443585082:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.642791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241451467775234:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241451467775310:2114] held: no
2025-12-24T02:11:50.643155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241452443585006:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.643167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.643173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241452443585006:2048] [node 1] ICP22 created new session: [2:7587241452443585099:2048]
2025-12-24T02:11:50.643177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241452443585099:2048] [node 1] ICS09 handshake done sender: [2:7587241452443585082:2114] self: [2:7587241452443585084:0] peer: [1:7587241449728653890:0] socket: 517 qp: -1
2025-12-24T02:11:50.643183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241452443585099:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.643188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.645654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977124:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
2025-12-24T02:11:50.647459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977124:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.647490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241450494977038:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.651338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241449728653902:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.651372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241449728653889:2112] [node 6] ICP18 handshake (actor [1:7587241449728653902:2117]) is too old
2025-12-24T02:11:50.652051Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241450008012204:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30925) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241450008012271:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.656654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241450494977126:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:579
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.656569Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012295:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.658723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012296:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.654721Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585085:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.654737Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241452443585085:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.656089Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241452443585067:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(33191522ns) {HandleHandshakeStatus:329(33190809ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(163ns) SwitchToState:179(251ns) ProcessPendingSessionEvents:495(18709327ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18685343ns max# 18598432ns}}}
2025-12-24T02:11:50.658502Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241452443585085:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.658591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241452443585007:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.658607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.658615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241452443585007:2049] [node 3] ICP22 created new session: [2:7587241452443585103:2049]
2025-12-24T02:11:50.658620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241452443585103:2049] [node 3] ICS09 handshake done sender: [2:7587241452443585085:2115] self: [2:7587241452443585072:0] peer: [3:7587241450494977125:0] socket: 540 qp: -1
2025-12-24T02:11:50.658626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241452443585103:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:50.658633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.659194Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775322:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.659260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775323:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.659371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.659726Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241450494977108:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(22095763ns) {Configure:104(22095426ns) {ProcessConfigured:135(21726774ns) {StartInitialHandshake:157(21713266ns) {DropHandshakes:470(395ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(311ns) PrepareNewSessionHandshake:477(1241ns) {SwitchToState:179(338ns)}} IncomingHandshake:261(4347ns) {DropIncomingHandshake:442(27ns) PrepareNewSessionHandshake:477(183ns) {SwitchToState:179(131ns)} GenerateSessionVirtualId:988(610ns) IssueIncomingHandshakeReply:193(338ns)}}}}
2025-12-24T02:11:50.660080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241450494977124:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.659318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585097:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.659326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241452443585097:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.659438Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241451467775311:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.661082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241450494977038:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.661098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.661106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241450494977038:2049] [node 2] ICP22 created new session: [3:7587241450494977130:2049]
2025-12-24T02:11:50.661112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241450494977130:2049] [node 2] ICS09 handshake done sender: [3:7587241450494977124:2117] self: [3:7587241450494977125:0] peer: [2:7587241452443585072:0] socket: 544 qp: -1
2025-12-24T02:11:50.661118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241450494977130:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.661124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.662282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775325:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.658980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241450008012271:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.659175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012296:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.659211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450008012205:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.660163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450008012204:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.660198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.660205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450008012204:2048] [node 1] ICP22 created new session: [4:7587241450008012298:2048]
2025-12-24T02:11:50.660210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241450008012298:2048] [node 1] ICS09 handshake done sender: [4:7587241450008012271:2111] self: [4:7587241450008012278:0] peer: [1:7587241449728653885:0] socket: 494 qp: -1
2025-12-24T02:11:50.660216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241450008012298:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.660253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:50.664072Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241450494977098:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(86802640ns) {HandleHandshakeStatus:329(86801876ns) {DropIncomingHandshake:442(509ns) DropOutgoingHandshake:457(273ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(24576821ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 24564537ns max# 24531363ns}}}
2025-12-24T02:11:50.662349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241451467775312:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:50.665660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241451467775258:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.665674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.665681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241451467775258:2066] [node 5] ICP22 created new session: [6:7587241451467775326:2066]
2025-12-24T02:11:50.665688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241451467775326:2066] [node 5] ICS09 handshake done sender: [6:7587241451467775311:2115] self: [6:7587241451467775316:0] peer: [5:7587241449562771304:0] socket: 515 qp: -1
2025-12-24T02:11:50.665695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241451467775326:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:50.665715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.666140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241451467775325:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.666147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241451467775233:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.666156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:50.666161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241451467775233:2048] [node 1] ICP22 created new session: [6:7587241451467775328:2048]
2025-12-24T02:11:50.666166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241451467775328:2048] [node 1] ICS09 handshake done sender: [6:7587241451467775312:2116] self: [6:7587241451467775313:0] peer: [1:7587241449728653896:0] socket: 516 qp: -1
2025-12-24T02:11:50.666172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241451467775328:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:50.666179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.667310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771348:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.667366Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771349:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.669070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241451467775310:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.667832Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241450008012206:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(36417048ns) {HandleHandshakeStatus:329(36416325ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(71ns) SwitchToState:179(85ns) ProcessPendingSessionEvents:495(36311543ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 36291382ns max# 36215247ns}}}
2025-12-24T02:11:50.662986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241449728653876:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(109082193ns) {HandleHandshakeStatus:329(109081232ns) {DropIncomingHandshake:442(501ns) DropOutgoingHandshake:457(678ns) SwitchToState:179(93ns) ProcessPendingSessionEvents:495(23929700ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 23912291ns max# 23863163ns}}}
2025-12-24T02:11:50.663788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241449728653884:2110] [node 4] ICP35 event processing took too much time PendingConnection:279(24710124ns) {HandleHandshakeStatus:329(24709244ns) {DropIncomingHandshake:442(4377ns) DropOutgoingHandshake:457(147ns) SwitchToState:179(151ns) ProcessPendingSessionEvents:495(24632055ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24607297ns max# 24530835ns}}}
2025-12-24T02:11:50.666954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241452443585105:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49172
2025-12-24T02:11:50.667267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241452443585105:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.674083Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241450494977110:2114] [node 6] ICP35 event processing took too much time PendingConnection:279(36879790ns) {HandleHandshakeStatus:329(36878822ns) {DropIncomingHandshake:442(129ns) DropOutgoingHandshake:457(173ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(22799351ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 22772082ns max# 22685094ns}}}
2025-12-24T02:11:50.677591Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241449562771349:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.677662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241449562771258:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.679435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241451467775234:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.679479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.679489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241451467775234:2049] [node 2] ICP22 created new session: [6:7587241451467775330:2049]
2025-12-24T02:11:50.679496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241451467775330:2049] [node 2] ICS09 handshake done sender: [6:7587241451467775310:2114] self: [6:7587241451467775314:0] peer: [2:7587241452443585074:0] socket: 503 qp: -1
2025-12-24T02:11:50.679504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241451467775330:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.679512Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:50.680016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241451467775250:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:50.682166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241450494977127:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.682179Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241450494977127:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:50.682966Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241452443585097:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.683153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241452443585081:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.683178Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.683186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241452443585081:2110] [node 4] ICP22 created new session: [2:7587241452443585108:2110]
2025-12-24T02:11:50.683193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241452443585108:2110] [node 4] ICS09 handshake done sender: [2:7587241452443585097:2118] self: [2:7587241452443585096:0] peer: [4:7587241450008012297:0] socket: 499 qp: -1
2025-12-24T02:11:50.683199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241452443585108:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:50.683207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.684161Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241450494977127:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:50.684240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241450494977108:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.684255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:50.684262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241450494977108:2111] [node 5] ICP22 created new session: [3:7587241450494977133:2111]
2025-12-24T02:11:50.684269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241450494977133:2111] [node 5] ICS09 handshake done sender: [3:7587241450494977127:2119] self: [3:7587241450494977123:0] peer: [5:7587241449562771350:0] socket: 552 qp: -1
2025-12-24T02:11:50.684274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241450494977133:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:50.684281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:50.686786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241450008012288:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:50.686804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241450008012288:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.695034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241450008012300:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:60742
2025-12-24T02:11:50.697929Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241450008012205:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5312) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241450008012296:2120] held: no
2025-12-24T02:11:50.698055Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241450008012296:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:50.698193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450008012205:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.698209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:50.698217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450008012205:2049] [node 2] ICP22 created new session: [4:7587241450008012301:2049]
2025-12-24T02:11:50.698224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241450008012301:2049] [node 2] ICS09 handshake done sender: [4:7587241450008012296:2120] self: [4:7587241450008012297:0] peer: [2:7587241452443585096:0] socket: 530 qp: -1
2025-12-24T02:11:50.698234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241450008012301:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:50.698257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.700076Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241449562771351:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11091
2025-12-24T02:11:50.701155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241449562771258:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17154) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241449562771349:2120] held: no
2025-12-24T02:11:50.701348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241449562771290:2079] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:50.701522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241449562771349:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:50.701582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241449562771258:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.701600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:50.701606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241449562771258:2050] [node 3] ICP22 created new session: [5:7587241449562771352:2050]
2025-12-24T02:11:50.701613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241449562771352:2050] [node 3] ICS09 handshake done sender: [5:7587241449562771349:2120] self: [5:7587241449562771350:0] peer: [3:7587241450494977123:0] socket: 532 qp: -1
2025-12-24T02:11:50.701620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241449562771352:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:50.701629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:50.703481Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241450494977108:2111] [node 5] ICP35 event processing took too much time PendingConnection:279(19128289ns) {HandleHandshakeStatus:329(19127583ns) {DropIncomingHandshake:442(2999ns) DropOutgoingHandshake:457(136ns) SwitchToState:179(381ns) ProcessPendingSessionEvents:495(19055281ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19031481ns max# 18945134ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:50.706964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241450008012288:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:50.707082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241450008012280:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:50.707110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:50.707118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450008012280:2115] [node 6] ICP22 created new session: [4:7587241450008012304:2115]
2025-12-24T02:11:50.707124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241450008012304:2115] [node 6] ICS09 handshake done sender: [4:7587241450008012288:2117] self: [4:7587241450008012281:0] peer: [6:7587241451467775332:0] socket: 542 qp: -1
2025-12-24T02:11:50.707130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241450008012304:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:50.707139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:50.706933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241451467775333:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55843
2025-12-24T02:11:50.708273Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241451467775250:2061] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18157) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241451467775325:2121] held: no
2025-12-24T02:11:50.708599Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241451467775325:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:50.714060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241451467775250:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:50.714087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:50.714097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241451467775250:2061] [node 4] ICP22 created new session: [6:7587241451467775334:2061]
2025-12-24T02:11:50.714105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241451467775334:2061] [node 4] ICS09 handshake done sender: [6:7587241451467775325:2121] self: [6:7587241451467775332:0] peer: [4:7587241450008012281:0] socket: 551 qp: -1
2025-12-24T02:11:50.714117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241451467775334:2061] [node 4] ICS10 traffic start
2025-12-24T02:11:50.714137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:50.742658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:50.742807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241452443585099:2048] [node 1] ICS01 socket: 517 reason# EndOfStream
2025-12-24T02:11:50.742821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241452443585006:2048] [node 1] ICP30 unregister session Session# [2:7587241452443585099:2048] VirtualId# [2:7587241452443585084:0]
2025-12-24T02:11:50.742831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241452443585099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.742869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.742901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.742940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:50.742999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241450494977117:2048] [node 1] ICS07 socket disconnect 487 reason# EndOfStream
2025-12-24T02:11:50.743003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241450494977117:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.743031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.743051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241450494977117:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:50.743739Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241450494977037:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30925) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:50.743754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241450494977117:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:50.743759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241450494977037:2048] [node 1] ICP30 unregister session Session# [3:7587241450494977117:2048] VirtualId# [3:7587241450494977106:0]
2025-12-24T02:11:50.743814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.743820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241450494977037:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30925) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.743823Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30925) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:50.750551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:50.750649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241450008012298:2048] [node 1] ICS01 socket: 494 reason# EndOfStream
2025-12-24T02:11:50.750657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241450008012204:2048] [node 1] ICP30 unregister session Session# [4:7587241450008012298:2048] VirtualId# [4:7587241450008012278:0]
2025-12-24T02:11:50.750666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241450008012298:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.750736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.750764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.750876Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:50.750960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241449562771337:2048] [node 1] ICS01 socket: 482 reason# EndOfStream
2025-12-24T02:11:50.750967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241449562771256:2048] [node 1] ICP30 unregister session Session# [5:7587241449562771337:2048] VirtualId# [5:7587241449562771325:0]
2025-12-24T02:11:50.750974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241449562771337:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.751023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.751048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.751926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:50.752027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241451467775328:2048] [node 1] ICS01 socket: 516 reason# EndOfStream
2025-12-24T02:11:50.752034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241451467775233:2048] [node 1] ICP30 unregister session Session# [6:7587241451467775328:2048] VirtualId# [6:7587241451467775313:0]
2025-12-24T02:11:50.752040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241451467775328:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.752084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:50.752110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:50.755735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241450494977037:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:50.774524Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:50.774653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241450494977130:2049] [node 2] ICS01 socket: 544 reason# EndOfStream
2025-12-24T02:11:50.774664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241450494977038:2049] [node 2] ICP30 unregister session Session# [3:7587241450494977130:2049] VirtualId# [3:7587241450494977125:0]
2025-12-24T02:11:50.774673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241450494977130:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.774746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.774770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.782968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:50.783064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241451467775330:2049] [node 2] ICS01 socket: 503 reason# EndOfStream
2025-12-24T02:11:50.783075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241451467775234:2049] [node 2] ICP30 unregister session Session# [6:7587241451467775330:2049] VirtualId# [6:7587241451467775314:0]
2025-12-24T02:11:50.783083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241451467775330:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.783131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.783156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.783035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:50.783118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241450008012301:2049] [node 2] ICS01 socket: 530 reason# EndOfStream
2025-12-24T02:11:50.783126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241450008012205:2049] [node 2] ICP30 unregister session Session# [4:7587241450008012301:2049] VirtualId# [4:7587241450008012297:0]
2025-12-24T02:11:50.783133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241450008012301:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.783184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.783209Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.783121Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:50.783201Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241449562771346:2049] [node 2] ICS01 socket: 509 reason# EndOfStream
2025-12-24T02:11:50.783208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241449562771257:2049] [node 2] ICP30 unregister session Session# [5:7587241449562771346:2049] VirtualId# [5:7587241449562771332:0]
2025-12-24T02:11:50.783213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241449562771346:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.783260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:50.783284Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:50.810973Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:50.811099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241449562771352:2050] [node 3] ICS01 socket: 532 reason# EndOfStream
2025-12-24T02:11:50.811113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241449562771258:2050] [node 3] ICP30 unregister session Session# [5:7587241449562771352:2050] VirtualId# [5:7587241449562771350:0]
2025-12-24T02:11:50.811123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241449562771352:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.811171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:50.811205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:50.811246Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:50.811318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241451467775318:2050] [node 3] ICS01 socket: 483 reason# EndOfStream
2025-12-24T02:11:50.811324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241451467775235:2050] [node 3] ICP30 unregister session Session# [6:7587241451467775318:2050] VirtualId# [6:7587241451467775245:0]
2025-12-24T02:11:50.811329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241451467775318:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.811358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:50.811390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:50.833583Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:50.833694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241451467775334:2061] [node 4] ICS01 socket: 551 reason# EndOfStream
2025-12-24T02:11:50.833704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241451467775250:2061] [node 4] ICP30 unregister session Session# [6:7587241451467775334:2061] VirtualId# [6:7587241451467775332:0]
2025-12-24T02:11:50.833712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241451467775334:2061] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:50.833763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:50.833788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:50.911115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449553895441:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:50.911123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241449553895442:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.923000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241450435862693:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.923022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241450435862694:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:50.922928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448908722026:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.922943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241448908722027:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.932906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450942105735:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.932917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450942105736:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.932923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241450942105737:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.954451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241450878883540:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.954500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241450878883541:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.954507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241450878883542:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:50.957114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241450989848158:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:50.957126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241450989848159:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:50.957132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241450989848160:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.015246Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241455237073090:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:51.015451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241455237073091:2104] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.018608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850871:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.018965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241454730830050:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:51.019671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830051:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.022028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241453848862797:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20209
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.022612Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830055:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
2025-12-24T02:11:51.022640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830056:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
2025-12-24T02:11:51.023200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241453848862805:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:51.024229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241455284815496:2082] [node 5] ICP01 ready to work
2025-12-24T02:11:51.024753Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815488:2079] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.024766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815489:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.034398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455173850893:2086] [node 4] ICP01 ready to work
2025-12-24T02:11:51.036252Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850894:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
2025-12-24T02:11:51.043309Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241450989848158:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(18242482ns) {Configure:104(18241457ns) {ProcessConfigured:135(17925191ns) {StartInitialHandshake:157(17924672ns) {DropHandshakes:470(260ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(1134ns) {SwitchToState:179(482ns)}}}}}
2025-12-24T02:11:51.043526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241455284815489:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.043732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241455284815525:2096] [node 4] ICP01 ready to work
2025-12-24T02:11:51.045782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241450989848159:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.047436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455173850911:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:51.046732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241453848862810:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:51.046937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241453848862811:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:51.047063Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862797:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.048067Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241453848862812:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20209
2025-12-24T02:11:51.054942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862812:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.062823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850894:2099] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.079028Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241450435862693:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(51679618ns) {Configure:104(51678539ns) {ProcessConfigured:135(51285979ns) {StartInitialHandshake:157(51285273ns) {DropHandshakes:470(941ns) {DropIncomingHandshake:442(82ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(440ns) PrepareNewSessionHandshake:477(2279ns) {SwitchToState:179(1457ns)}}}}}
2025-12-24T02:11:51.079323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241454730830061:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:51.079547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241454730830062:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:51.079706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830051:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.079769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830053:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.079773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241454730830053:2109] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.081697Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815535:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.082348Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241454730830053:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.082597Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241455284815489:2080] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.082676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241450989848159:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.082709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.082715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241450989848159:2049] [node 2] ICP22 created new session: [6:7587241455284815536:2049]
2025-12-24T02:11:51.082721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241455284815536:2049] [node 2] ICS09 handshake done sender: [6:7587241455284815489:2080] self: [6:7587241455284815530:0] peer: [2:7587241454730830052:0] socket: 505 qp: -1
2025-12-24T02:11:51.082728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241455284815536:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.082739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.083029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830055:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.083041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241454730830050:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.083052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.083056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241454730830050:2107] [node 6] ICP22 created new session: [2:7587241454730830073:2107]
2025-12-24T02:11:51.083060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241454730830073:2107] [node 6] ICS09 handshake done sender: [2:7587241454730830053:2109] self: [2:7587241454730830052:0] peer: [6:7587241455284815530:0] socket: 497 qp: -1
2025-12-24T02:11:51.083063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241454730830073:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:51.083070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.083801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073105:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.091630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453203689389:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:51.092355Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453203689392:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:51.092566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453203689393:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:51.092921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689385:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.092978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689386:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.093011Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689387:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.093044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689388:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.094386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850917:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.095021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830056:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.094924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241453848862820:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20209
2025-12-24T02:11:51.094980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241453848862821:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20209
2025-12-24T02:11:51.095136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689387:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.095179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689386:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.096361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241448908722026:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.095195Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850871:2080] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.095229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241455173850893:2086] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.098766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241455237073105:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.098800Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241455237073095:2107] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.098804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241455237073095:2107] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.099349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450942105735:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.099562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850919:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
2025-12-24T02:11:51.102934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862807:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.102949Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241453848862807:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.104831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241453848862807:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.104982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241453848862805:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.104995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.105003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241453848862805:2110] [node 4] ICP22 created new session: [1:7587241453848862824:2110]
2025-12-24T02:11:51.105009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241453848862824:2110] [node 4] ICS09 handshake done sender: [1:7587241453848862807:2111] self: [1:7587241453848862806:0] peer: [4:7587241455237073111:0] socket: 514 qp: -1
2025-12-24T02:11:51.105015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241453848862824:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:51.105022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.105285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241453848862823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20209
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.102834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241455237073095:2107] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.104044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241455237073090:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.104059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.104068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241455237073090:2102] [node 5] ICP22 created new session: [4:7587241455237073112:2102]
2025-12-24T02:11:51.104075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241455237073112:2102] [node 5] ICS09 handshake done sender: [4:7587241455237073095:2107] self: [4:7587241455237073094:0] peer: [5:7587241455173850918:0] socket: 490 qp: -1
2025-12-24T02:11:51.104084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241455237073112:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:51.104095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.106688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073115:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.103912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455173850871:2080] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.106778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241455173850893:2086] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.106793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.106799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455173850893:2086] [node 4] ICP22 created new session: [5:7587241455173850920:2086]
2025-12-24T02:11:51.106804Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455173850920:2086] [node 4] ICS09 handshake done sender: [5:7587241455173850871:2080] self: [5:7587241455173850918:0] peer: [4:7587241455237073094:0] socket: 492 qp: -1
2025-12-24T02:11:51.106810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455173850920:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:51.106917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241455284815488:2079] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.106817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.107155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.107179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241455284815525:2096] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.107081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241450435862693:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.108442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862820:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.114870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862799:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.114883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241453848862799:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.115834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830078:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
2025-12-24T02:11:51.115997Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241453848862799:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.116212Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241454730830056:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.116284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241450435862693:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.116304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.116312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241450435862693:2048] [node 1] ICP22 created new session: [2:7587241454730830079:2048]
2025-12-24T02:11:51.116319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241454730830079:2048] [node 1] ICS09 handshake done sender: [2:7587241454730830056:2111] self: [2:7587241454730830077:0] peer: [1:7587241453848862798:0] socket: 504 qp: -1
2025-12-24T02:11:51.116326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241454730830079:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.116335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.117480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449553895441:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.117525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.117532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449553895441:2048] [node 2] ICP22 created new session: [1:7587241453848862829:2048]
2025-12-24T02:11:51.117537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241453848862829:2048] [node 2] ICS09 handshake done sender: [1:7587241453848862799:2108] self: [1:7587241453848862798:0] peer: [2:7587241454730830077:0] socket: 501 qp: -1
2025-12-24T02:11:51.117543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241453848862829:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:51.117550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.114796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241455237073097:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.114820Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241455237073097:2108] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.119512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689385:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.119606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689388:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.120204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241448908722027:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.120230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689407:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.120608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241455237073105:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.121039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450942105735:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.121052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.121058Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450942105735:2048] [node 1] ICP22 created new session: [4:7587241455237073118:2048]
2025-12-24T02:11:51.121064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241455237073118:2048] [node 1] ICS09 handshake done sender: [4:7587241455237073105:2113] self: [4:7587241455237073111:0] peer: [1:7587241453848862806:0] socket: 515 qp: -1
2025-12-24T02:11:51.121069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241455237073118:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.121076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.127665Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830065:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.127679Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241454730830065:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.129651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689397:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.129664Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453203689397:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.127352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850923:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
2025-12-24T02:11:51.129515Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850917:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.129552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241450878883542:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.130570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850925:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.130842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815541:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.130887Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815542:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.131147Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850915:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.131154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241455173850915:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.131435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453203689397:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.131038Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241455284815541:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.131068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241455284815496:2082] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.135314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.135431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689407:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.135493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453203689392:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.135510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.135517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453203689392:2107] [node 5] ICP22 created new session: [3:7587241453203689411:2107]
2025-12-24T02:11:51.135526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453203689411:2107] [node 5] ICS09 handshake done sender: [3:7587241453203689397:2116] self: [3:7587241453203689396:0] peer: [5:7587241455173850924:0] socket: 524 qp: -1
2025-12-24T02:11:51.135535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453203689411:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:51.135545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.136170Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241455284815538:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.136201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241450989848160:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.136650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455173850915:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.136885Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241455284815541:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.136952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241455284815496:2082] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.136968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.136973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241455284815496:2082] [node 5] ICP22 created new session: [6:7587241455284815546:2082]
2025-12-24T02:11:51.136978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241455284815546:2082] [node 5] ICS09 handshake done sender: [6:7587241455284815541:2119] self: [6:7587241455284815543:0] peer: [5:7587241455173850912:0] socket: 495 qp: -1
2025-12-24T02:11:51.136985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241455284815546:2082] [node 5] ICS10 traffic start
2025-12-24T02:11:51.136993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.138103Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689395:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.138113Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453203689395:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.139904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815548:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.140099Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241455173850911:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.140119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.140126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455173850911:2103] [node 6] ICP22 created new session: [5:7587241455173850927:2103]
2025-12-24T02:11:51.140131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455173850927:2103] [node 6] ICS09 handshake done sender: [5:7587241455173850915:2116] self: [5:7587241455173850912:0] peer: [6:7587241455284815543:0] socket: 541 qp: -1
2025-12-24T02:11:51.140137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455173850927:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:51.140146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.140626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453203689395:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.144361Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862801:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.144376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241453848862801:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.144985Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862821:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.145007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862823:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.145604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241453848862805:2110] [node 4] ICP18 handshake (actor [1:7587241453848862823:2119]) is too old
2025-12-24T02:11:51.145925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453203689389:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.145948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.145957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453203689389:2109] [node 6] ICP22 created new session: [3:7587241453203689417:2109]
2025-12-24T02:11:51.145964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453203689417:2109] [node 6] ICS09 handshake done sender: [3:7587241453203689395:2115] self: [3:7587241453203689394:0] peer: [6:7587241455284815545:0] socket: 523 qp: -1
2025-12-24T02:11:51.145971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453203689417:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:51.145981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.147603Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241455237073097:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.147820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830083:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
2025-12-24T02:11:51.147968Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241454730830065:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.148092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073116:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
2025-12-24T02:11:51.148194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241455237073091:2104] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.148923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241450435862694:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.148204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.148209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241455237073091:2104] [node 6] ICP22 created new session: [4:7587241455237073121:2104]
2025-12-24T02:11:51.148941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.148946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241450435862694:2049] [node 3] ICP22 created new session: [2:7587241454730830085:2049]
2025-12-24T02:11:51.148217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241455237073121:2104] [node 6] ICS09 handshake done sender: [4:7587241455237073097:2108] self: [4:7587241455237073096:0] peer: [6:7587241455284815540:0] socket: 491 qp: -1
2025-12-24T02:11:51.148952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241454730830085:2049] [node 3] ICS09 handshake done sender: [2:7587241454730830065:2115] self: [2:7587241454730830064:0] peer: [3:7587241453203689409:0] socket: 518 qp: -1
2025-12-24T02:11:51.148238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241455237073121:2104] [node 6] ICS10 traffic start
2025-12-24T02:11:51.148957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241454730830085:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.148965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.149313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241454730830084:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9015
2025-12-24T02:11:51.148249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.148688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073120:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
2025-12-24T02:11:51.150884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241455237073116:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.152186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450942105736:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.152484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850923:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.152525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241455284815538:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.152675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455173850917:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.152882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241453848862801:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.153012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241449553895442:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.153033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.153038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241449553895442:2049] [node 3] ICP22 created new session: [1:7587241453848862834:2049]
2025-12-24T02:11:51.153044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241453848862834:2049] [node 3] ICS09 handshake done sender: [1:7587241453848862801:2109] self: [1:7587241453848862800:0] peer: [3:7587241453203689406:0] socket: 502 qp: -1
2025-12-24T02:11:51.153050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241453848862834:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.153059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.153590Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241450878883542:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8099) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241455173850917:2117] held: no
2025-12-24T02:11:51.153679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241450878883542:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.153687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.153694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241450878883542:2050] [node 3] ICP22 created new session: [5:7587241455173850930:2050]
2025-12-24T02:11:51.153699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455173850930:2050] [node 3] ICS09 handshake done sender: [5:7587241455173850917:2117] self: [5:7587241455173850924:0] peer: [3:7587241453203689396:0] socket: 525 qp: -1
2025-12-24T02:11:51.153704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455173850930:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.153710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.154217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689419:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.154351Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241455284815488:2079] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.154280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241450878883540:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.154443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850929:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
2025-12-24T02:11:51.154808Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455173850929:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.154833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241450878883541:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.155016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241450989848160:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.155030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.155036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241450989848160:2050] [node 3] ICP22 created new session: [6:7587241455284815551:2050]
2025-12-24T02:11:51.155041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241455284815551:2050] [node 3] ICS09 handshake done sender: [6:7587241455284815538:2118] self: [6:7587241455284815545:0] peer: [3:7587241453203689394:0] socket: 534 qp: -1
2025-12-24T02:11:51.155046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241455284815551:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.155054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.155204Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241450989848160:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:51.155415Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815550:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.155423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241455284815525:2096] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.155432Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.155436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241455284815525:2096] [node 4] ICP22 created new session: [6:7587241455284815553:2096]
2025-12-24T02:11:51.155441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241455284815553:2096] [node 4] ICS09 handshake done sender: [6:7587241455284815488:2079] self: [6:7587241455284815540:0] peer: [4:7587241455237073096:0] socket: 493 qp: -1
2025-12-24T02:11:51.155445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241455284815553:2096] [node 4] ICS10 traffic start
2025-12-24T02:11:51.155450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.156654Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073125:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
2025-12-24T02:11:51.158934Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862815:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.158947Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241453848862815:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.159318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830067:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.159341Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241454730830067:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.159965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830083:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.160003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830069:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.160006Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241454730830069:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.160701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241454730830084:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.160813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241450435862694:2049] [node 3] ICP18 handshake (actor [2:7587241454730830083:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.161304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850934:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.163660Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455173850935:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33382
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.163985Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241454730830069:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.164224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455173850929:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.164302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241450878883541:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.164322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.164336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241450878883541:2049] [node 2] ICP22 created new session: [5:7587241455173850936:2049]
2025-12-24T02:11:51.164347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455173850936:2049] [node 2] ICS09 handshake done sender: [5:7587241455173850929:2121] self: [5:7587241455173850933:0] peer: [2:7587241454730830068:0] socket: 558 qp: -1
2025-12-24T02:11:51.164355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455173850936:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.164363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.165531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241454730830067:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.163793Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073126:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
2025-12-24T02:11:51.165733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241455237073116:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.165801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450942105736:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.165813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.165820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450942105736:2049] [node 2] ICP22 created new session: [4:7587241455237073127:2049]
2025-12-24T02:11:51.165826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241455237073127:2049] [node 2] ICS09 handshake done sender: [4:7587241455237073116:2117] self: [4:7587241455237073124:0] peer: [2:7587241454730830066:0] socket: 537 qp: -1
2025-12-24T02:11:51.165833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241455237073127:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.165840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.166925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241454730830062:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.166950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.166962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241454730830062:2113] [node 5] ICP22 created new session: [2:7587241454730830090:2113]
2025-12-24T02:11:51.166968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241454730830090:2113] [node 5] ICS09 handshake done sender: [2:7587241454730830069:2117] self: [2:7587241454730830068:0] peer: [5:7587241455173850933:0] socket: 552 qp: -1
2025-12-24T02:11:51.166975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241454730830090:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:51.166983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.167950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241453848862815:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.168068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241453848862810:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.168083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.168089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241453848862810:2112] [node 5] ICP22 created new session: [1:7587241453848862837:2112]
2025-12-24T02:11:51.168094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241453848862837:2112] [node 5] ICS09 handshake done sender: [1:7587241453848862815:2115] self: [1:7587241453848862814:0] peer: [5:7587241455173850932:0] socket: 532 qp: -1
2025-12-24T02:11:51.168100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241453848862837:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:51.168107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.172155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241455237073091:2104] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:51.173231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241455237073125:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.173266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241450942105737:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.173413Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241448908722027:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14115) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241453203689388:2113] held: no
2025-12-24T02:11:51.173512Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241448908722026:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241453203689386:2111] held: no
2025-12-24T02:11:51.183445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241454730830061:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.183479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.183491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241454730830061:2112] [node 4] ICP22 created new session: [2:7587241454730830092:2112]
2025-12-24T02:11:51.183499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241454730830092:2112] [node 4] ICS09 handshake done sender: [2:7587241454730830067:2116] self: [2:7587241454730830066:0] peer: [4:7587241455237073124:0] socket: 535 qp: -1
2025-12-24T02:11:51.183508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241454730830092:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:51.183519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.187364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453203689401:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.187376Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453203689401:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.187987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453203689388:2113] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.188943Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453203689416:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41759
2025-12-24T02:11:51.189091Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241450878883540:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241455173850923:2119] held: no
2025-12-24T02:11:51.189225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455173850923:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.189590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241448908722027:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.189606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.189614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448908722027:2049] [node 2] ICP22 created new session: [3:7587241453203689422:2049]
2025-12-24T02:11:51.189620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453203689422:2049] [node 2] ICS09 handshake done sender: [3:7587241453203689388:2113] self: [3:7587241453203689409:0] peer: [2:7587241454730830064:0] socket: 521 qp: -1
2025-12-24T02:11:51.189627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453203689422:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.189635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.189898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453203689386:2111] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.190154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241455237073130:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7707
2025-12-24T02:11:51.190340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241455284815550:2123] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.190383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241450989848158:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.190705Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241453848862817:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.190718Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241453848862817:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.190283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241450878883540:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.190297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.190302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241450878883540:2048] [node 1] ICP22 created new session: [5:7587241455173850938:2048]
2025-12-24T02:11:51.190307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455173850938:2048] [node 1] ICS09 handshake done sender: [5:7587241455173850923:2119] self: [5:7587241455173850932:0] peer: [1:7587241453848862814:0] socket: 542 qp: -1
2025-12-24T02:11:51.190312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455173850938:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.190319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.191596Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241453848862817:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.191679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241453848862811:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.191693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.191698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241453848862811:2113] [node 6] ICP22 created new session: [1:7587241453848862840:2113]
2025-12-24T02:11:51.191702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241453848862840:2113] [node 6] ICS09 handshake done sender: [1:7587241453848862817:2116] self: [1:7587241453848862816:0] peer: [6:7587241455284815555:0] socket: 510 qp: -1
2025-12-24T02:11:51.191707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241453848862840:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:51.191713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.192286Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453203689401:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.192519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241455237073125:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.192582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241450942105737:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.192595Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.192600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241450942105737:2050] [node 3] ICP22 created new session: [4:7587241455237073131:2050]
2025-12-24T02:11:51.192605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241455237073131:2050] [node 3] ICS09 handshake done sender: [4:7587241455237073125:2119] self: [4:7587241455237073129:0] peer: [3:7587241453203689400:0] socket: 509 qp: -1
2025-12-24T02:11:51.192611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241455237073131:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.192618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.193805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241448908722026:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.193818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.193823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241448908722026:2048] [node 1] ICP22 created new session: [3:7587241453203689424:2048]
2025-12-24T02:11:51.193829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453203689424:2048] [node 1] ICS09 handshake done sender: [3:7587241453203689386:2111] self: [3:7587241453203689406:0] peer: [1:7587241453848862800:0] socket: 511 qp: -1
2025-12-24T02:11:51.193833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453203689424:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.193840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.194041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453203689393:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.194050Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.194054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453203689393:2108] [node 4] ICP22 created new session: [3:7587241453203689426:2108]
2025-12-24T02:11:51.194059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453203689426:2108] [node 4] ICS09 handshake done sender: [3:7587241453203689401:2118] self: [3:7587241453203689400:0] peer: [4:7587241455237073129:0] socket: 531 qp: -1
2025-12-24T02:11:51.194062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453203689426:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:51.194068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.196139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241450989848158:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:61774) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241455284815550:2123] held: no
2025-12-24T02:11:51.196278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241455284815556:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27396
2025-12-24T02:11:51.196497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241455284815550:2123] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.196574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241450989848158:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.196587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.196593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241450989848158:2048] [node 1] ICP22 created new session: [6:7587241455284815557:2048]
2025-12-24T02:11:51.196599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241455284815557:2048] [node 1] ICS09 handshake done sender: [6:7587241455284815550:2123] self: [6:7587241455284815555:0] peer: [1:7587241453848862816:0] socket: 557 qp: -1
2025-12-24T02:11:51.196604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241455284815557:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.196611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.212508Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:51.212616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241454730830079:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-12-24T02:11:51.212626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241450435862693:2048] [node 1] ICP30 unregister session Session# [2:7587241454730830079:2048] VirtualId# [2:7587241454730830077:0]
2025-12-24T02:11:51.212634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241454730830079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.212673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.212696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.213287Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:51.213358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241453203689424:2048] [node 1] ICS01 socket: 511 reason# EndOfStream
2025-12-24T02:11:51.213364Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241448908722026:2048] [node 1] ICP30 unregister session Session# [3:7587241453203689424:2048] VirtualId# [3:7587241453203689406:0]
2025-12-24T02:11:51.213369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241453203689424:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.213395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.213413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.218050Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:51.218135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241455237073118:2048] [node 1] ICS01 socket: 515 reason# EndOfStream
2025-12-24T02:11:51.218143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241450942105735:2048] [node 1] ICP30 unregister session Session# [4:7587241455237073118:2048] VirtualId# [4:7587241455237073111:0]
2025-12-24T02:11:51.218149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241455237073118:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.218194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.218218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.218773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:51.218837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455173850938:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-12-24T02:11:51.218843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241450878883540:2048] [node 1] ICP30 unregister session Session# [5:7587241455173850938:2048] VirtualId# [5:7587241455173850932:0]
2025-12-24T02:11:51.218848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455173850938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.218885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.218908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.218942Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:51.219002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241455284815557:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:51.219007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241450989848158:2048] [node 1] ICP30 unregister session Session# [6:7587241455284815557:2048] VirtualId# [6:7587241455284815555:0]
2025-12-24T02:11:51.219010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241455284815557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.219037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.219056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.224429Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:51.224517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241453203689422:2049] [node 2] ICS01 socket: 521 reason# EndOfStream
2025-12-24T02:11:51.224524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241448908722027:2049] [node 2] ICP30 unregister session Session# [3:7587241453203689422:2049] VirtualId# [3:7587241453203689409:0]
2025-12-24T02:11:51.224529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241453203689422:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.224563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.224580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.229697Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:51.229780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241455284815536:2049] [node 2] ICS07 socket disconnect 505 reason# EndOfStream
2025-12-24T02:11:51.229788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241455284815536:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.229829Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.229848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241455284815536:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:51.230165Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:51.230248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455173850936:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-12-24T02:11:51.230254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241450878883541:2049] [node 2] ICP30 unregister session Session# [5:7587241455173850936:2049] VirtualId# [5:7587241455173850933:0]
2025-12-24T02:11:51.230259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455173850936:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.230289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.230309Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.230611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241450989848159:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14115) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:51.230626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241455284815536:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:51.230630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241450989848159:2049] [node 2] ICP30 unregister session Session# [6:7587241455284815536:2049] VirtualId# [6:7587241455284815530:0]
2025-12-24T02:11:51.230681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.230687Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241450989848159:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14115) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.230690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14115) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.231797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:51.231888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241455237073127:2049] [node 2] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:51.231895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241450942105736:2049] [node 2] ICP30 unregister session Session# [4:7587241455237073127:2049] VirtualId# [4:7587241455237073124:0]
2025-12-24T02:11:51.231900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241455237073127:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.231947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.231971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.242804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241450989848159:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:51.250365Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:51.250566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455173850930:2050] [node 3] ICS01 socket: 525 reason# EndOfStream
2025-12-24T02:11:51.250579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241450878883542:2050] [node 3] ICP30 unregister session Session# [5:7587241455173850930:2050] VirtualId# [5:7587241455173850924:0]
2025-12-24T02:11:51.250587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455173850930:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.250647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:51.250675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:51.250730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:51.250797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241455284815551:2050] [node 3] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:51.250802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241450989848160:2050] [node 3] ICP30 unregister session Session# [6:7587241455284815551:2050] VirtualId# [6:7587241455284815545:0]
2025-12-24T02:11:51.250808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241455284815551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.250836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:51.250859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:51.262781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:51.262889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241455284815553:2096] [node 4] ICS01 socket: 493 reason# EndOfStream
2025-12-24T02:11:51.262900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241455284815525:2096] [node 4] ICP30 unregister session Session# [6:7587241455284815553:2096] VirtualId# [6:7587241455284815540:0]
2025-12-24T02:11:51.262909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241455284815553:2096] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.262962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:51.262990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:51.318838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241453449748559:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.318854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241453449748560:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:51.323008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456812775043:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:51.323024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456812775044:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:51.319122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241456570373683:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.319131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241456570373684:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.333032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456709863902:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.333044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456709863903:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.333049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456709863904:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.353102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241456639898151:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.353118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241456639898152:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.353125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241456639898153:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.362414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241456266024002:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.362426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241456266024003:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.362431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241456266024004:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.401241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024028:2073] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.402486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456709863935:2079] [node 5] ICP01 ready to work
2025-12-24T02:11:51.399566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241456639898169:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:51.403235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456812775104:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:51.411199Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241456570373744:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:51.411350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241453449748623:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:51.411519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241453449748624:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:51.413264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456812775110:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.401976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241456266024031:2075] [node 5] ICP01 ready to work
2025-12-24T02:11:51.402605Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241456266024042:2081] [node 4] ICP01 ready to work
2025-12-24T02:11:51.416310Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024064:2104] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.416325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024065:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.416335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024066:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.414837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456709863959:2082] [node 6] ICP01 ready to work
2025-12-24T02:11:51.416044Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241456639898212:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:51.417186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898219:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.413791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456812775113:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:51.414515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456812775119:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39229
2025-12-24T02:11:51.415065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241453449748636:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35199
2025-12-24T02:11:51.418600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241456266024028:2073] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.426068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241453449748637:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35199
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.426886Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241456639898151:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19359360ns) {Configure:104(19358594ns) {ProcessConfigured:135(18991159ns) {StartInitialHandshake:157(18990398ns) {DropHandshakes:470(337ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(1814ns) {SwitchToState:179(1035ns)}}}}}
2025-12-24T02:11:51.428201Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898228:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.430819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241453449748622:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:51.434839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748637:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.438802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241456570373684:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26710983ns) {Configure:104(26710504ns) {ProcessConfigured:135(26495304ns) {StartInitialHandshake:157(26494486ns) {DropHandshakes:470(219ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(184ns) PrepareNewSessionHandshake:477(2215ns) {SwitchToState:179(1536ns)}}}}}
2025-12-24T02:11:51.439155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241456570373755:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:51.439307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241456570373756:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:51.439469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373754:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.440409Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373752:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.440461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373753:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.441587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863975:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.441614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863976:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.441625Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863977:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.442152Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241456266024003:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(39494783ns) {Configure:104(39494086ns) {ProcessConfigured:135(39191951ns) {StartInitialHandshake:157(39191740ns) {DropHandshakes:470(244ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(25ns)} GenerateSessionVirtualId:988(815ns) PrepareNewSessionHandshake:477(887ns) {SwitchToState:179(275ns)}}}}}
2025-12-24T02:11:51.442185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241456266024031:2075] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.443207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241453449748640:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35199
2025-12-24T02:11:51.442804Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241456570373683:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23612513ns) {Configure:104(23612099ns) {ProcessConfigured:135(23225791ns) {StartInitialHandshake:157(23225479ns) {DropHandshakes:470(506ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(672ns) {SwitchToState:179(488ns)}}}}}
2025-12-24T02:11:51.446753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241456266024064:2104] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.446792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241456266024003:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.447092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373752:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.447172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241456570373684:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.449877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898171:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.449906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241456639898171:2065] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.450723Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898231:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.450797Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863976:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.451316Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241456812775104:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(48002407ns) {Configure:104(48001319ns) {ProcessConfigured:135(47589746ns) {StartInitialHandshake:157(47588931ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(1722ns) {SwitchToState:179(947ns)}}}}}
2025-12-24T02:11:51.451383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775119:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.451618Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456812775122:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39229
2025-12-24T02:11:51.458824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748626:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.458853Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241453449748626:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.474051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898231:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.474112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241456639898153:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.475200Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241456639898171:2065] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.475317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241456639898169:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.475335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.475345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241456639898169:2064] [node 6] ICP22 created new session: [5:7587241456639898234:2064]
2025-12-24T02:11:51.475353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241456639898234:2064] [node 6] ICS09 handshake done sender: [5:7587241456639898171:2065] self: [5:7587241456639898170:0] peer: [6:7587241456266024082:0] socket: 492 qp: -1
2025-12-24T02:11:51.475363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241456639898234:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:51.475378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.478095Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024086:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.481518Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863977:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.478714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241456266024028:2073] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.478768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241456266024031:2075] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.478785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.478793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241456266024031:2075] [node 5] ICP22 created new session: [6:7587241456266024087:2075]
2025-12-24T02:11:51.478803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241456266024087:2075] [node 5] ICS09 handshake done sender: [6:7587241456266024028:2073] self: [6:7587241456266024082:0] peer: [5:7587241456639898170:0] socket: 494 qp: -1
2025-12-24T02:11:51.478812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241456266024087:2075] [node 5] ICS10 traffic start
2025-12-24T02:11:51.478832Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.479203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373759:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.479219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241456570373759:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.481654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456709863904:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.478803Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241453449748622:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(47739899ns) {Configure:104(47738645ns) {ProcessConfigured:135(47327187ns) {StartInitialHandshake:157(47325852ns) {DropHandshakes:470(986ns) {DropIncomingHandshake:442(179ns) DropOutgoingHandshake:457(195ns)} GenerateSessionVirtualId:988(313ns) PrepareNewSessionHandshake:477(2216ns) {SwitchToState:179(1015ns)}}}}}
2025-12-24T02:11:51.479136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748630:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.479150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241453449748630:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.481740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241453449748626:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.481837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241453449748623:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.481850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.481858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241453449748623:2108] [node 6] ICP22 created new session: [2:7587241453449748643:2108]
2025-12-24T02:11:51.481864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241453449748643:2108] [node 6] ICS09 handshake done sender: [2:7587241453449748626:2110] self: [2:7587241453449748625:0] peer: [6:7587241456266024083:0] socket: 495 qp: -1
2025-12-24T02:11:51.481871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241453449748643:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:51.481891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.482035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241456570373759:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.482321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373746:2108] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.482337Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241456570373746:2108] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.482822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898219:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.483047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241456570373755:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.483081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.483092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241456570373755:2109] [node 5] ICP22 created new session: [3:7587241456570373767:2109]
2025-12-24T02:11:51.483103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241456570373767:2109] [node 5] ICS09 handshake done sender: [3:7587241456570373759:2115] self: [3:7587241456570373758:0] peer: [5:7587241456639898232:0] socket: 528 qp: -1
2025-12-24T02:11:51.483118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241456570373767:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:51.483132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.482379Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748640:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.488052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241456639898169:2064] [node 6] ICP18 handshake (actor [5:7587241456639898219:2107]) is too old
2025-12-24T02:11:51.487348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863985:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.488576Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775108:2109] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.488589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456812775108:2109] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.489474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456812775108:2109] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.488446Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748636:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.488482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241453449748559:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.489398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241453449748648:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35199
2025-12-24T02:11:51.489657Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241453449748636:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.489777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241453449748559:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.489793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.489801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241453449748559:2048] [node 1] ICP22 created new session: [2:7587241453449748649:2048]
2025-12-24T02:11:51.489808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241453449748649:2048] [node 1] ICS09 handshake done sender: [2:7587241453449748636:2114] self: [2:7587241453449748647:0] peer: [1:7587241456812775107:0] socket: 502 qp: -1
2025-12-24T02:11:51.489815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241453449748649:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.489824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.491056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241456570373746:2108] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.491163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863975:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.491314Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024090:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.491337Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024089:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.493023Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775122:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.493194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456812775043:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.493215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.493220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456812775043:2048] [node 2] ICP22 created new session: [1:7587241456812775127:2048]
2025-12-24T02:11:51.493227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456812775127:2048] [node 2] ICS09 handshake done sender: [1:7587241456812775108:2109] self: [1:7587241456812775107:0] peer: [2:7587241453449748647:0] socket: 498 qp: -1
2025-12-24T02:11:51.493233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456812775127:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:51.493240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.491497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373754:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.494448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456709863977:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.492756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373771:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.494646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373753:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.494676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241456570373683:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.494856Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898228:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.495021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241456639898212:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.495105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898237:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.495262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373771:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.495292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241456570373755:2109] [node 5] ICP18 handshake (actor [3:7587241456570373771:2118]) is too old
2025-12-24T02:11:51.495420Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241453449748630:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.495271Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775112:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.495280Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456812775112:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.498860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241453449748560:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.498887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.498893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241453449748560:2049] [node 3] ICP22 created new session: [2:7587241453449748652:2049]
2025-12-24T02:11:51.498901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241453449748652:2049] [node 3] ICS09 handshake done sender: [2:7587241453449748630:2112] self: [2:7587241453449748629:0] peer: [3:7587241456570373765:0] socket: 497 qp: -1
2025-12-24T02:11:51.498908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241453449748652:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.498928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.498866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373774:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.499290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241456570373744:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.499304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.499310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241456570373744:2107] [node 4] ICP22 created new session: [3:7587241456570373775:2107]
2025-12-24T02:11:51.499317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241456570373775:2107] [node 4] ICS09 handshake done sender: [3:7587241456570373746:2108] self: [3:7587241456570373745:0] peer: [4:7587241456709863984:0] socket: 526 qp: -1
2025-12-24T02:11:51.499324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241456570373775:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:51.499331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.499650Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456812775112:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.498477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898238:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.498534Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898239:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.498797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241456639898231:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.499616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241456639898153:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.499629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.499637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241456639898153:2050] [node 3] ICP22 created new session: [5:7587241456639898241:2050]
2025-12-24T02:11:51.499642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241456639898241:2050] [node 3] ICS09 handshake done sender: [5:7587241456639898231:2117] self: [5:7587241456639898232:0] peer: [3:7587241456570373758:0] socket: 532 qp: -1
2025-12-24T02:11:51.499648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241456639898241:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.499654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.500693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898238:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.500848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241456639898152:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.502237Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748632:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.502249Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241453449748632:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.502953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241453449748632:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.503045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241453449748624:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.503062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.503067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241453449748624:2109] [node 5] ICP22 created new session: [2:7587241453449748656:2109]
2025-12-24T02:11:51.503073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241453449748656:2109] [node 5] ICS09 handshake done sender: [2:7587241453449748632:2113] self: [2:7587241453449748631:0] peer: [5:7587241456639898244:0] socket: 540 qp: -1
2025-12-24T02:11:51.503077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241453449748656:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:51.503095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.502982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456812775044:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.503000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.503006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456812775044:2049] [node 3] ICP22 created new session: [1:7587241456812775132:2049]
2025-12-24T02:11:51.503011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456812775132:2049] [node 3] ICS09 handshake done sender: [1:7587241456812775112:2111] self: [1:7587241456812775111:0] peer: [3:7587241456570373773:0] socket: 500 qp: -1
2025-12-24T02:11:51.503016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456812775132:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.503023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.503297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456812775131:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39229
2025-12-24T02:11:51.502851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456709863904:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.502879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.502887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456709863904:2050] [node 3] ICP22 created new session: [4:7587241456709863987:2050]
2025-12-24T02:11:51.502894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456709863987:2050] [node 3] ICS09 handshake done sender: [4:7587241456709863977:2114] self: [4:7587241456709863984:0] peer: [3:7587241456570373745:0] socket: 530 qp: -1
2025-12-24T02:11:51.502902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456709863987:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.502911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.502836Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898245:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.503711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241456639898239:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.503859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241456639898238:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.503962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241456639898151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.503973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241456639898152:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.503985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.503991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241456639898152:2049] [node 2] ICP22 created new session: [5:7587241456639898247:2049]
2025-12-24T02:11:51.503996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241456639898247:2049] [node 2] ICS09 handshake done sender: [5:7587241456639898238:2119] self: [5:7587241456639898244:0] peer: [2:7587241453449748631:0] socket: 550 qp: -1
2025-12-24T02:11:51.504002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241456639898247:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.504009Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.504713Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775131:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.504754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775118:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.504758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456812775118:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.505849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863966:2104] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.505863Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241456709863966:2104] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.509724Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456812775118:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.509836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456812775113:2110] [node 5] ICP20 outgoing handshake succeeded
No RDMA context found for GID: 2025-12-24T02:11:51.509856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
::ffff:127.1.0.02025-12-24T02:11:51.509862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456812775113:2110] [node 5] ICP22 created new session: [1:7587241456812775137:2110]
2025-12-24T02:11:51.509868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456812775137:2110] [node 5] ICS09 handshake done sender: [1:7587241456812775118:2113] self: [1:7587241456812775117:0] peer: [5:7587241456639898246:0] socket: 548 qp: -1
2025-12-24T02:11:51.509874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456812775137:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:51.509883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.510168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898249:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.510271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863989:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.510353Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709863990:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.510762Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241456266024064:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.514862Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456709863966:2104] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.515232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456812775139:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39229
2025-12-24T02:11:51.516633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863990:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.516648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241456709863935:2079] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.516662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.516667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456709863935:2079] [node 5] ICP22 created new session: [4:7587241456709863994:2079]
2025-12-24T02:11:51.516672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456709863994:2079] [node 5] ICS09 handshake done sender: [4:7587241456709863966:2104] self: [4:7587241456709863936:0] peer: [5:7587241456639898240:0] socket: 516 qp: -1
2025-12-24T02:11:51.516676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456709863994:2079] [node 5] ICS10 traffic start
2025-12-24T02:11:51.516682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.518325Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456709863902:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.515494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241456266024003:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.515524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.515535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241456266024003:2049] [node 2] ICP22 created new session: [6:7587241456266024091:2049]
2025-12-24T02:11:51.515542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241456266024091:2049] [node 2] ICS09 handshake done sender: [6:7587241456266024064:2104] self: [6:7587241456266024083:0] peer: [2:7587241453449748625:0] socket: 506 qp: -1
2025-12-24T02:11:51.515550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241456266024091:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.515560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.519101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241456266024066:2106] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.519188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241456266024065:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.519214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241456266024042:2081] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.519275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241456266024002:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.516196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241456570373752:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.519135Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373777:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
2025-12-24T02:11:51.519406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863963:2103] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.519417Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241456709863963:2103] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.522172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775123:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.522184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456812775123:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.522792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775115:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.522797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456812775115:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.523714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456812775123:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.523902Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456812775115:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.525951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024096:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.526137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241456266024065:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.524524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241456570373684:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241456570373752:2112] held: no
2025-12-24T02:11:51.524626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241456570373684:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.524639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.524646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241456570373684:2049] [node 2] ICP22 created new session: [3:7587241456570373779:2049]
2025-12-24T02:11:51.524653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241456570373779:2049] [node 2] ICS09 handshake done sender: [3:7587241456570373752:2112] self: [3:7587241456570373765:0] peer: [2:7587241453449748629:0] socket: 523 qp: -1
2025-12-24T02:11:51.524661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241456570373779:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.524669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.526219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241456266024002:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.526234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.526240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241456266024002:2048] [node 1] ICP22 created new session: [6:7587241456266024097:2048]
2025-12-24T02:11:51.526246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241456266024097:2048] [node 1] ICS09 handshake done sender: [6:7587241456266024065:2105] self: [6:7587241456266024095:0] peer: [1:7587241456812775114:0] socket: 507 qp: -1
2025-12-24T02:11:51.526253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241456266024097:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.526261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.527998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456709863989:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.528116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456709863903:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.528207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456709863963:2103] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.528419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241456709863959:2082] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.528438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.531312Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748641:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.531325Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241453449748641:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.532074Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241453449748641:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.532156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241453449748622:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.532173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.532191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241453449748622:2107] [node 4] ICP22 created new session: [2:7587241453449748659:2107]
2025-12-24T02:11:51.532198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241453449748659:2107] [node 4] ICS09 handshake done sender: [2:7587241453449748641:2117] self: [2:7587241453449748638:0] peer: [4:7587241456709863998:0] socket: 496 qp: -1
2025-12-24T02:11:51.532206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241453449748659:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:51.532214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.528447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456709863959:2082] [node 6] ICP22 created new session: [4:7587241456709863999:2082]
2025-12-24T02:11:51.528462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456709863999:2082] [node 6] ICS09 handshake done sender: [4:7587241456709863963:2103] self: [4:7587241456709863962:0] peer: [6:7587241456266024094:0] socket: 505 qp: -1
2025-12-24T02:11:51.528469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456709863999:2082] [node 6] ICS10 traffic start
2025-12-24T02:11:51.528479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.530979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241456639898151:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15769) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241456639898239:2120] held: no
2025-12-24T02:11:51.534873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241453449748661:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35199
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.535200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456812775139:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.535222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456812775110:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.535239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.535245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456812775110:2108] [node 6] ICP22 created new session: [1:7587241456812775142:2108]
2025-12-24T02:11:51.535251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456812775142:2108] [node 6] ICS09 handshake done sender: [1:7587241456812775115:2112] self: [1:7587241456812775114:0] peer: [6:7587241456266024095:0] socket: 501 qp: -1
2025-12-24T02:11:51.535258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456812775142:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:51.535268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.536030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709864001:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.536215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456709864002:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:9813
2025-12-24T02:11:51.536326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456709863989:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.536469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456709863990:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.536516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456709863903:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.536527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.536536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456709863903:2049] [node 2] ICP22 created new session: [4:7587241456709864003:2049]
2025-12-24T02:11:51.536541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456709864003:2049] [node 2] ICS09 handshake done sender: [4:7587241456709863989:2119] self: [4:7587241456709863998:0] peer: [2:7587241453449748638:0] socket: 556 qp: -1
2025-12-24T02:11:51.536546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456709864003:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.536553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.538024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456709863902:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.538036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.538042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456709863902:2048] [node 1] ICP22 created new session: [4:7587241456709864005:2048]
2025-12-24T02:11:51.538047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456709864005:2048] [node 1] ICS09 handshake done sender: [4:7587241456709863990:2120] self: [4:7587241456709863996:0] peer: [1:7587241456812775105:0] socket: 504 qp: -1
2025-12-24T02:11:51.538053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456709864005:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.538061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.538914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241453449748661:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.539282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241453449748622:2107] [node 4] ICP18 handshake (actor [2:7587241453449748661:2119]) is too old
2025-12-24T02:11:51.539986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241456570373753:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.539817Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456812775104:2107] [node 4] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.539835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.539843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456812775104:2107] [node 4] ICP22 created new session: [1:7587241456812775144:2107]
2025-12-24T02:11:51.539849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456812775144:2107] [node 4] ICS09 handshake done sender: [1:7587241456812775123:2116] self: [1:7587241456812775105:0] peer: [4:7587241456709863996:0] socket: 557 qp: -1
2025-12-24T02:11:51.539854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456812775144:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:51.539862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.540040Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241456812775104:2107] [node 4] ICP18 handshake (actor [1:7587241456812775139:2118]) is too old
2025-12-24T02:11:51.540163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241456570373683:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.540176Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.540184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241456570373683:2048] [node 1] ICP22 created new session: [3:7587241456570373781:2048]
2025-12-24T02:11:51.540190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241456570373781:2048] [node 1] ICS09 handshake done sender: [3:7587241456570373753:2113] self: [3:7587241456570373773:0] peer: [1:7587241456812775111:0] socket: 524 qp: -1
2025-12-24T02:11:51.540196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241456570373781:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.540203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.540627Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241456570373783:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51264
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.541010Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456812775146:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39229
2025-12-24T02:11:51.548813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241456266024090:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.548853Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241456266024042:2081] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241456266024066:2106] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.549931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241456266024004:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.549966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024099:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.550315Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241456266024066:2106] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.550663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241456639898212:2067] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21798) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241456639898228:2114] held: no
2025-12-24T02:11:51.550807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241456639898239:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.550857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241456266024042:2081] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.550875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.550881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241456266024042:2081] [node 4] ICP22 created new session: [6:7587241456266024101:2081]
2025-12-24T02:11:51.550888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241456266024101:2081] [node 4] ICS09 handshake done sender: [6:7587241456266024066:2106] self: [6:7587241456266024094:0] peer: [4:7587241456709863962:0] socket: 508 qp: -1
2025-12-24T02:11:51.550897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241456266024101:2081] [node 4] ICS10 traffic start
2025-12-24T02:11:51.550905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.552200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241456266024103:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52533
2025-12-24T02:11:51.553006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241456266024090:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.553067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241456266024004:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.553080Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.553085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241456266024004:2050] [node 3] ICP22 created new session: [6:7587241456266024104:2050]
2025-12-24T02:11:51.553090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241456266024104:2050] [node 3] ICS09 handshake done sender: [6:7587241456266024090:2121] self: [6:7587241456266024100:0] peer: [3:7587241456570373760:0] socket: 547 qp: -1
2025-12-24T02:11:51.553095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241456266024104:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.553103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.550973Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373761:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.550987Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241456570373761:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.552309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241456570373761:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.552414Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241456570373783:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.552425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241456570373756:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.552435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.552442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241456570373756:2110] [node 6] ICP22 created new session: [3:7587241456570373785:2110]
2025-12-24T02:11:51.552447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241456570373785:2110] [node 6] ICS09 handshake done sender: [3:7587241456570373761:2116] self: [3:7587241456570373760:0] peer: [6:7587241456266024100:0] socket: 543 qp: -1
2025-12-24T02:11:51.552452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241456570373785:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:51.552460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.552713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241456570373756:2110] [node 6] ICP18 handshake (actor [3:7587241456570373783:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.555011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241456639898151:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.555032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.555038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241456639898151:2048] [node 1] ICP22 created new session: [5:7587241456639898250:2048]
2025-12-24T02:11:51.555044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241456639898250:2048] [node 1] ICS09 handshake done sender: [5:7587241456639898239:2120] self: [5:7587241456639898246:0] peer: [1:7587241456812775117:0] socket: 551 qp: -1
2025-12-24T02:11:51.555050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241456639898250:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.555058Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.555616Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241456639898252:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46665
2025-12-24T02:11:51.558919Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241456639898228:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.559006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241456639898212:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.559017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.559027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241456639898212:2067] [node 4] ICP22 created new session: [5:7587241456639898253:2067]
2025-12-24T02:11:51.559034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241456639898253:2067] [node 4] ICS09 handshake done sender: [5:7587241456639898228:2114] self: [5:7587241456639898240:0] peer: [4:7587241456709863936:0] socket: 521 qp: -1
2025-12-24T02:11:51.559042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241456639898253:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:51.559051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.578045Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:51.578159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241453449748649:2048] [node 1] ICS01 socket: 502 reason# EndOfStream
2025-12-24T02:11:51.578171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241453449748559:2048] [node 1] ICP30 unregister session Session# [2:7587241453449748649:2048] VirtualId# [2:7587241453449748647:0]
2025-12-24T02:11:51.578179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241453449748649:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.578216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.578239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.578675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:51.578767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241456570373781:2048] [node 1] ICS01 socket: 524 reason# EndOfStream
2025-12-24T02:11:51.578773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241456570373683:2048] [node 1] ICP30 unregister session Session# [3:7587241456570373781:2048] VirtualId# [3:7587241456570373773:0]
2025-12-24T02:11:51.578778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241456570373781:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.578800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.578816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.582021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:51.582110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241456639898250:2048] [node 1] ICS01 socket: 551 reason# EndOfStream
2025-12-24T02:11:51.582119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241456639898151:2048] [node 1] ICP30 unregister session Session# [5:7587241456639898250:2048] VirtualId# [5:7587241456639898246:0]
2025-12-24T02:11:51.582125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241456639898250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.582179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.582206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.582688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:51.582812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241456709864005:2048] [node 1] ICS01 socket: 504 reason# EndOfStream
2025-12-24T02:11:51.582818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241456709863902:2048] [node 1] ICP30 unregister session Session# [4:7587241456709864005:2048] VirtualId# [4:7587241456709863996:0]
2025-12-24T02:11:51.582823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241456709864005:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.582857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.582879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.586372Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:51.586465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241456266024097:2048] [node 1] ICS01 socket: 507 reason# EndOfStream
2025-12-24T02:11:51.586475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241456266024002:2048] [node 1] ICP30 unregister session Session# [6:7587241456266024097:2048] VirtualId# [6:7587241456266024095:0]
2025-12-24T02:11:51.586481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241456266024097:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.586529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.586552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.595522Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:51.595619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241456570373779:2049] [node 2] ICS01 socket: 523 reason# EndOfStream
2025-12-24T02:11:51.595628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241456570373684:2049] [node 2] ICP30 unregister session Session# [3:7587241456570373779:2049] VirtualId# [3:7587241456570373765:0]
2025-12-24T02:11:51.595635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241456570373779:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.595668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.595690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.611993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:51.612128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241456639898247:2049] [node 2] ICS01 socket: 550 reason# EndOfStream
2025-12-24T02:11:51.612141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241456639898152:2049] [node 2] ICP30 unregister session Session# [5:7587241456639898247:2049] VirtualId# [5:7587241456639898244:0]
2025-12-24T02:11:51.612151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241456639898247:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.612212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.612244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.612364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:51.612437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241456709864003:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:51.612443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241456709863903:2049] [node 2] ICP30 unregister session Session# [4:7587241456709864003:2049] VirtualId# [4:7587241456709863998:0]
2025-12-24T02:11:51.612448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241456709864003:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.612477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.612499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.614378Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:51.614468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241456266024091:2049] [node 2] ICS01 socket: 506 reason# EndOfStream
2025-12-24T02:11:51.614476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241456266024003:2049] [node 2] ICP30 unregister session Session# [6:7587241456266024091:2049] VirtualId# [6:7587241456266024083:0]
2025-12-24T02:11:51.614482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241456266024091:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.614528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.614552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.626939Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:51.627049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241456266024104:2050] [node 3] ICS01 socket: 547 reason# EndOfStream
2025-12-24T02:11:51.627061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241456266024004:2050] [node 3] ICP30 unregister session Session# [6:7587241456266024104:2050] VirtualId# [6:7587241456266024100:0]
2025-12-24T02:11:51.627071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241456266024104:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.627121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:51.627146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:51.627315Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:51.627419Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241456639898241:2050] [node 3] ICS01 socket: 532 reason# EndOfStream
2025-12-24T02:11:51.627425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241456639898153:2050] [node 3] ICP30 unregister session Session# [5:7587241456639898241:2050] VirtualId# [5:7587241456639898232:0]
2025-12-24T02:11:51.627430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241456639898241:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.627458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:51.627480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:51.659346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:51.659480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241456266024101:2081] [node 4] ICS01 socket: 508 reason# EndOfStream
2025-12-24T02:11:51.659492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241456266024042:2081] [node 4] ICP30 unregister session Session# [6:7587241456266024101:2081] VirtualId# [6:7587241456266024094:0]
2025-12-24T02:11:51.659502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241456266024101:2081] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.659569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:51.659598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:51.670854Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:51.670967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241456266024087:2075] [node 5] ICS01 socket: 494 reason# EndOfStream
2025-12-24T02:11:51.670978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241456266024031:2075] [node 5] ICP30 unregister session Session# [6:7587241456266024087:2075] VirtualId# [6:7587241456266024082:0]
2025-12-24T02:11:51.670989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241456266024087:2075] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.671031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:51.671057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:51.750979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456368750365:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:51.750989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456368750366:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:51.751111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241456696710340:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.751124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241456696710341:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:51.751136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456263051163:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.751147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456263051164:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.751152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456263051165:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.752915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455862427557:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.752926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455862427558:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.752934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455862427559:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.756977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241454562615537:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.757001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241454562615538:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.757007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241454562615539:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:51.751295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453124296488:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:51.751307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453124296489:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:51.783382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241454562615595:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:51.784031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241454562615599:2104] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.784982Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051218:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
2025-12-24T02:11:51.784820Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241456696710400:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55621
2025-12-24T02:11:51.785071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456263051219:2101] [node 5] ICP01 ready to work
2025-12-24T02:11:51.786786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455862427610:2098] [node 6] ICP01 ready to work
2025-12-24T02:11:51.807093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456368750432:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:51.807276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456368750433:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.808676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241454562615539:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(21838094ns) {Configure:104(21836862ns) {ProcessConfigured:135(21464920ns) {StartInitialHandshake:157(21464653ns) {DropHandshakes:470(417ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(78ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(1015ns) {SwitchToState:179(784ns)}}}}}
2025-12-24T02:11:51.811358Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241455862427610:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24497678ns) {Configure:104(24497126ns) {ProcessConfigured:135(24064619ns) {StartInitialHandshake:157(24064133ns) {DropHandshakes:470(445ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(1551ns) {SwitchToState:179(232ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.812363Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241456696710340:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24641063ns) {Configure:104(24640501ns) {ProcessConfigured:135(24240437ns) {StartInitialHandshake:157(24240282ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(40ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(295ns) PrepareNewSessionHandshake:477(900ns) {SwitchToState:179(47ns)}}}}}
2025-12-24T02:11:51.812535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241456696710408:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:51.812669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241456696710409:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:51.813572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241456696710410:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55621
2025-12-24T02:11:51.813729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615613:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.813756Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615614:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.808966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456368750442:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36093
2025-12-24T02:11:51.808983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456368750443:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36093
2025-12-24T02:11:51.809006Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456368750444:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36093
2025-12-24T02:11:51.814244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241456368750431:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:51.818322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453124296559:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:51.818760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241456263051237:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:51.819448Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241455862427558:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(32131555ns) {Configure:104(13660161ns) {ProcessConfigured:135(13244576ns) {StartInitialHandshake:157(13244350ns) {DropHandshakes:470(564ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(527ns) PrepareNewSessionHandshake:477(811ns) {SwitchToState:179(359ns)}}}}}
2025-12-24T02:11:51.822790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241456696710406:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:51.822891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710400:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.823800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427628:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.823833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427629:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.823860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427630:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.823950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427631:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.825129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453124296557:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:51.826367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427630:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.824933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241456368750447:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36093
2025-12-24T02:11:51.830864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241453124296558:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:51.833893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241455862427634:2115] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.833159Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241456696710341:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(44987268ns) {Configure:104(44984868ns) {ProcessConfigured:135(44709407ns) {StartInitialHandshake:157(44709259ns) {DropHandshakes:470(305ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(153ns) PrepareNewSessionHandshake:477(1273ns) {SwitchToState:179(589ns)}}}}}
2025-12-24T02:11:51.833751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241456696710340:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.834553Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427635:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.838141Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750437:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.838166Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456368750437:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.838215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750426:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.838222Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456368750426:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.843161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241455862427557:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.843232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241455862427557:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16780602ns) {Configure:104(16779851ns) {ProcessConfigured:135(16327960ns) {IncomingHandshake:261(16311716ns) {DropIncomingHandshake:442(189ns) PrepareNewSessionHandshake:477(777ns) {SwitchToState:179(482ns)} GenerateSessionVirtualId:988(493ns) IssueIncomingHandshakeReply:193(16304250ns) {SwitchToState:179(999ns)}}}}}
2025-12-24T02:11:51.848292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750444:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.852488Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051218:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.860138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241454562615614:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.860197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241454562615538:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.859876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296568:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.859906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296567:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.860733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427631:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.860859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427628:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.861011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750447:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.861044Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750442:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.861138Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456368750437:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.861334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296567:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.861367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241453124296488:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.861497Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750435:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.861509Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456368750435:2111] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.862147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.862370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241454562615613:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.862494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615618:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.862514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241454562615595:2102] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.862756Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456368750426:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.862796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456368750435:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.860403Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710414:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.860420Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241456696710414:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.863285Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241454562615614:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.863194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453124296567:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.862880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456368750366:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.862898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456368750432:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.862904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.862909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.862915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456368750432:2109] [node 5] ICP22 created new session: [1:7587241456368750456:2109]
2025-12-24T02:11:51.862916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456368750366:2049] [node 3] ICP22 created new session: [1:7587241456368750455:2049]
2025-12-24T02:11:51.862922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456368750456:2109] [node 5] ICS09 handshake done sender: [1:7587241456368750437:2112] self: [1:7587241456368750436:0] peer: [5:7587241455862427632:0] socket: 514 qp: -1
2025-12-24T02:11:51.862925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456368750455:2049] [node 3] ICS09 handshake done sender: [1:7587241456368750435:2111] self: [1:7587241456368750434:0] peer: [3:7587241453124296569:0] socket: 513 qp: -1
2025-12-24T02:11:51.862932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456368750456:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:51.862932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456368750455:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.862940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.862941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.863516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241453124296488:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.863533Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.863539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453124296488:2048] [node 1] ICP22 created new session: [3:7587241453124296571:2048]
2025-12-24T02:11:51.863546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453124296571:2048] [node 1] ICS09 handshake done sender: [3:7587241453124296567:2115] self: [3:7587241453124296569:0] peer: [1:7587241456368750434:0] socket: 528 qp: -1
2025-12-24T02:11:51.863552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453124296571:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.863585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.863187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241456696710414:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.866721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241456696710409:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.866732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.866739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241456696710409:2110] [node 6] ICP22 created new session: [2:7587241456696710423:2110]
2025-12-24T02:11:51.866745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241456696710423:2110] [node 6] ICS09 handshake done sender: [2:7587241456696710414:2114] self: [2:7587241456696710413:0] peer: [6:7587241454562615617:0] socket: 521 qp: -1
2025-12-24T02:11:51.866750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241456696710423:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:51.866769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.867247Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427629:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.867282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427619:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.867286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241455862427619:2106] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.867714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241455862427558:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.867859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241455862427634:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.868157Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241456696710425:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55621
2025-12-24T02:11:51.868203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241456696710426:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55621
2025-12-24T02:11:51.868228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241456696710427:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55621
2025-12-24T02:11:51.868499Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710412:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.868507Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241456696710412:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.869026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241456696710400:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.870359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241456696710340:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.870390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.870403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241456696710340:2048] [node 1] ICP22 created new session: [2:7587241456696710430:2048]
2025-12-24T02:11:51.870413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241456696710430:2048] [node 1] ICS09 handshake done sender: [2:7587241456696710400:2107] self: [2:7587241456696710421:0] peer: [1:7587241456368750425:0] socket: 502 qp: -1
2025-12-24T02:11:51.870423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241456696710430:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.870449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.875840Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455862427619:2106] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.876613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241455862427610:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.876632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.876641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455862427610:2098] [node 6] ICP22 created new session: [5:7587241455862427646:2098]
2025-12-24T02:11:51.876649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455862427646:2098] [node 6] ICS09 handshake done sender: [5:7587241455862427619:2106] self: [5:7587241455862427612:0] peer: [6:7587241454562615619:0] socket: 517 qp: -1
2025-12-24T02:11:51.876655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455862427646:2098] [node 6] ICS10 traffic start
2025-12-24T02:11:51.876663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.876981Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427644:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.876994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427645:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.874872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051221:2103] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.874890Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241456263051221:2103] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.879080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427643:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.879183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456263051221:2103] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.879424Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241456368750432:2109] [node 5] ICP35 event processing took too much time PendingConnection:279(16401463ns) {HandleHandshakeStatus:329(16400862ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(208ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(16302806ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16281854ns max# 16190942ns}}}
2025-12-24T02:11:51.881165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456368750365:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.881186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.881193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456368750365:2048] [node 2] ICP22 created new session: [1:7587241456368750459:2048]
2025-12-24T02:11:51.881200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456368750459:2048] [node 2] ICS09 handshake done sender: [1:7587241456368750426:2107] self: [1:7587241456368750425:0] peer: [2:7587241456696710421:0] socket: 498 qp: -1
2025-12-24T02:11:51.881207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456368750459:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:51.881215Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.882429Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241456368750366:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(19454357ns) {HandleHandshakeStatus:329(19453430ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(452ns) SwitchToState:179(245ns) ProcessPendingSessionEvents:495(19112993ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19094893ns max# 19018101ns}}}
2025-12-24T02:11:51.881755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241454562615538:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.881778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.881784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241454562615538:2049] [node 2] ICP22 created new session: [6:7587241454562615622:2049]
2025-12-24T02:11:51.881789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241454562615622:2049] [node 2] ICS09 handshake done sender: [6:7587241454562615614:2115] self: [6:7587241454562615617:0] peer: [2:7587241456696710413:0] socket: 524 qp: -1
2025-12-24T02:11:51.881794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241454562615622:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.881801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.882132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615620:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.882167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615621:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.883050Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051244:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.883120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051245:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
2025-12-24T02:11:51.884854Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241454562615620:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.884885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241454562615595:2102] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14363) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241454562615613:2114] held: no
2025-12-24T02:11:51.885142Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241454562615613:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.886945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241456263051219:2101] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.886966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.886974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456263051219:2101] [node 5] ICP22 created new session: [4:7587241456263051246:2101]
2025-12-24T02:11:51.886981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456263051246:2101] [node 5] ICS09 handshake done sender: [4:7587241456263051221:2103] self: [4:7587241456263051220:0] peer: [5:7587241455862427642:0] socket: 505 qp: -1
2025-12-24T02:11:51.886989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456263051246:2101] [node 5] ICS10 traffic start
2025-12-24T02:11:51.887003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.887143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241454562615595:2102] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.887157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.887165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241454562615595:2102] [node 5] ICP22 created new session: [6:7587241454562615625:2102]
2025-12-24T02:11:51.887170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241454562615625:2102] [node 5] ICS09 handshake done sender: [6:7587241454562615613:2114] self: [6:7587241454562615619:0] peer: [5:7587241455862427612:0] socket: 523 qp: -1
2025-12-24T02:11:51.887176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241454562615625:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:51.887183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.887411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241454562615537:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.888866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615628:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.892735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615629:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.890889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710425:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.890928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710426:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.890991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241456696710412:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.893500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241456696710408:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.893516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.893523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241456696710408:2109] [node 5] ICP22 created new session: [2:7587241456696710432:2109]
2025-12-24T02:11:51.893529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241456696710432:2109] [node 5] ICS09 handshake done sender: [2:7587241456696710412:2113] self: [2:7587241456696710411:0] peer: [5:7587241455862427640:0] socket: 522 qp: -1
2025-12-24T02:11:51.893535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241456696710432:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:51.893542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.893810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241456696710408:2109] [node 5] ICP18 handshake (actor [2:7587241456696710426:2118]) is too old
2025-12-24T02:11:51.891271Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241453124296488:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(27646530ns) {HandleHandshakeStatus:329(27645910ns) {DropIncomingHandshake:442(338ns) DropOutgoingHandshake:457(2385ns) SwitchToState:179(113ns) ProcessPendingSessionEvents:495(8361500ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8332820ns max# 8237675ns}}}
2025-12-24T02:11:51.893032Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296574:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.893155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296573:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.894619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296574:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.894651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296573:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.894933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241454562615628:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.894975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241454562615599:2104] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.896122Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241456696710409:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(29296878ns) {HandleHandshakeStatus:329(29296462ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(115ns) SwitchToState:179(231ns) ProcessPendingSessionEvents:495(13135665ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13106502ns max# 13022182ns}}}
2025-12-24T02:11:51.896999Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455862427629:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.897098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241453124296489:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.898522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296568:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.897244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750439:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.897258Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456368750439:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.898970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456368750439:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.899991Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615631:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.900042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615632:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.900333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241454562615620:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.898961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051239:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.898979Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241456263051239:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.899075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241456696710340:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(28593204ns) {HandleHandshakeStatus:329(28592692ns) {DropIncomingHandshake:442(456ns) DropOutgoingHandshake:457(2269ns) SwitchToState:179(215ns) ProcessPendingSessionEvents:495(16104569ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16077386ns max# 15994377ns}}}
2025-12-24T02:11:51.899135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710410:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.901455Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456263051239:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.901585Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051245:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.901599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241456263051237:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.901613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.901618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456263051237:2109] [node 6] ICP22 created new session: [4:7587241456263051250:2109]
2025-12-24T02:11:51.901622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456263051250:2109] [node 6] ICS09 handshake done sender: [4:7587241456263051239:2114] self: [4:7587241456263051238:0] peer: [6:7587241454562615630:0] socket: 527 qp: -1
2025-12-24T02:11:51.901626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456263051250:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:51.901655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.900566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241454562615537:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.900587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.900596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241454562615537:2048] [node 1] ICP22 created new session: [6:7587241454562615633:2048]
2025-12-24T02:11:51.900604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241454562615633:2048] [node 1] ICS09 handshake done sender: [6:7587241454562615620:2118] self: [6:7587241454562615627:0] peer: [1:7587241456368750438:0] socket: 547 qp: -1
2025-12-24T02:11:51.900611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241454562615633:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.900620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.901982Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455862427630:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.901920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241454562615628:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.902041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241454562615599:2104] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.902049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.902054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241454562615599:2104] [node 4] ICP22 created new session: [6:7587241454562615635:2104]
2025-12-24T02:11:51.902058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241454562615635:2104] [node 4] ICS09 handshake done sender: [6:7587241454562615628:2120] self: [6:7587241454562615630:0] peer: [4:7587241456263051238:0] socket: 561 qp: -1
2025-12-24T02:11:51.902062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241454562615635:2104] [node 4] ICS10 traffic start
2025-12-24T02:11:51.902068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.902243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051252:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
2025-12-24T02:11:51.902308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456263051163:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.902498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456368750433:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.902519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.902523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456368750433:2110] [node 6] ICP22 created new session: [1:7587241456368750463:2110]
2025-12-24T02:11:51.902528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456368750463:2110] [node 6] ICS09 handshake done sender: [1:7587241456368750439:2113] self: [1:7587241456368750438:0] peer: [6:7587241454562615627:0] socket: 546 qp: -1
2025-12-24T02:11:51.902532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456368750463:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:51.902538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.902506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710418:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.902518Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241456696710418:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.902815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241455862427635:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.902846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241455862427634:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.902858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.902865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455862427634:2115] [node 4] ICP22 created new session: [5:7587241455862427649:2115]
2025-12-24T02:11:51.902870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455862427649:2115] [node 4] ICS09 handshake done sender: [5:7587241455862427629:2112] self: [5:7587241455862427642:0] peer: [4:7587241456263051220:0] socket: 532 qp: -1
2025-12-24T02:11:51.902877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455862427649:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:51.902886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.902979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455862427631:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:51.903276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051252:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.904533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456263051164:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.904847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241456696710420:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.904856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241456696710420:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.906313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241456696710420:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.907217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241456696710418:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.907634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241456696710341:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.907659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.907667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241456696710341:2049] [node 3] ICP22 created new session: [2:7587241456696710439:2049]
2025-12-24T02:11:51.907675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241456696710439:2049] [node 3] ICS09 handshake done sender: [2:7587241456696710418:2115] self: [2:7587241456696710402:0] peer: [3:7587241453124296577:0] socket: 551 qp: -1
2025-12-24T02:11:51.907683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241456696710439:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:51.907691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.907972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241456696710406:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.907982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.907987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241456696710406:2108] [node 4] ICP22 created new session: [2:7587241456696710441:2108]
2025-12-24T02:11:51.907992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241456696710441:2108] [node 4] ICS09 handshake done sender: [2:7587241456696710420:2116] self: [2:7587241456696710419:0] peer: [4:7587241456263051255:0] socket: 563 qp: -1
2025-12-24T02:11:51.907996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241456696710441:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:51.908002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.908466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051244:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.908504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241456263051219:2101] [node 5] ICP18 handshake (actor [4:7587241456263051244:2116]) is too old
2025-12-24T02:11:51.909597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241455862427557:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.909616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.909624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455862427557:2048] [node 1] ICP22 created new session: [5:7587241455862427651:2048]
2025-12-24T02:11:51.909631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455862427651:2048] [node 1] ICS09 handshake done sender: [5:7587241455862427630:2113] self: [5:7587241455862427632:0] peer: [1:7587241456368750436:0] socket: 533 qp: -1
2025-12-24T02:11:51.909637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455862427651:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.909645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.910056Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241455862427558:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241455862427631:2114] held: no
2025-12-24T02:11:51.910162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241455862427558:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.910170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.910176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455862427558:2049] [node 2] ICP22 created new session: [5:7587241455862427653:2049]
2025-12-24T02:11:51.910181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455862427653:2049] [node 2] ICS09 handshake done sender: [5:7587241455862427631:2114] self: [5:7587241455862427640:0] peer: [2:7587241456696710411:0] socket: 534 qp: -1
2025-12-24T02:11:51.910187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455862427653:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.910193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.910010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241456368750446:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.910027Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241456368750446:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.910676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241454562615629:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.910729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241454562615539:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.910683Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241456263051164:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241456263051252:2118] held: no
2025-12-24T02:11:51.910726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296579:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.910780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241453124296489:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17881) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241453124296573:2118] held: no
2025-12-24T02:11:51.910903Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296563:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.910906Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453124296563:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.913633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241456368750446:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.913758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241456368750431:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.913777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.913784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241456368750431:2108] [node 4] ICP22 created new session: [1:7587241456368750466:2108]
2025-12-24T02:11:51.913793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241456368750466:2108] [node 4] ICS09 handshake done sender: [1:7587241456368750446:2117] self: [1:7587241456368750445:0] peer: [4:7587241456263051254:0] socket: 557 qp: -1
2025-12-24T02:11:51.913798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241456368750466:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:51.913805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.915689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241453124296580:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:49953
2025-12-24T02:11:51.915702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241455862427559:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:51.916063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453124296563:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:51.916375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296561:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.916388Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453124296561:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.916895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453124296557:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.916910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:51.916918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453124296557:2111] [node 6] ICP22 created new session: [3:7587241453124296583:2111]
2025-12-24T02:11:51.916924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453124296583:2111] [node 6] ICS09 handshake done sender: [3:7587241453124296563:2113] self: [3:7587241453124296562:0] peer: [6:7587241454562615637:0] socket: 520 qp: -1
2025-12-24T02:11:51.916930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453124296583:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:51.916952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:51.917288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453124296573:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.917262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241456263051163:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241456263051245:2117] held: no
2025-12-24T02:11:51.917521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051256:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.917865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296579:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.917880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241453124296489:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.917891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.917897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453124296489:2049] [node 2] ICP22 created new session: [3:7587241453124296586:2049]
2025-12-24T02:11:51.917904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453124296586:2049] [node 2] ICS09 handshake done sender: [3:7587241453124296573:2118] self: [3:7587241453124296577:0] peer: [2:7587241456696710402:0] socket: 558 qp: -1
2025-12-24T02:11:51.917910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453124296586:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.917916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.918412Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241454562615638:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:37976
2025-12-24T02:11:51.918505Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241454562615629:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.918600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241454562615539:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.918618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.918623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241454562615539:2050] [node 3] ICP22 created new session: [6:7587241454562615639:2050]
2025-12-24T02:11:51.918629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241454562615639:2050] [node 3] ICS09 handshake done sender: [6:7587241454562615629:2121] self: [6:7587241454562615637:0] peer: [3:7587241453124296562:0] socket: 562 qp: -1
2025-12-24T02:11:51.918635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241454562615639:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.918641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.919501Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456263051252:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.919588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456263051164:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.919602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:51.919609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456263051164:2049] [node 2] ICP22 created new session: [4:7587241456263051259:2049]
2025-12-24T02:11:51.919615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456263051259:2049] [node 2] ICS09 handshake done sender: [4:7587241456263051252:2118] self: [4:7587241456263051255:0] peer: [2:7587241456696710419:0] socket: 564 qp: -1
2025-12-24T02:11:51.919621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456263051259:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:51.919627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:51.919930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051258:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
2025-12-24T02:11:51.920205Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453124296561:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:51.920467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453124296559:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.920481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:51.920487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453124296559:2110] [node 5] ICP22 created new session: [3:7587241453124296589:2110]
2025-12-24T02:11:51.920492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453124296589:2110] [node 5] ICS09 handshake done sender: [3:7587241453124296561:2112] self: [3:7587241453124296560:0] peer: [5:7587241455862427655:0] socket: 536 qp: -1
2025-12-24T02:11:51.920497Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453124296589:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:51.920504Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:51.927433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051257:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.932213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241455862427656:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6968
2025-12-24T02:11:51.932078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456263051245:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:51.932548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241455862427635:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.932658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456263051163:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.932679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:51.932689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456263051163:2048] [node 1] ICP22 created new session: [4:7587241456263051261:2048]
2025-12-24T02:11:51.932697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456263051261:2048] [node 1] ICS09 handshake done sender: [4:7587241456263051245:2117] self: [4:7587241456263051254:0] peer: [1:7587241456368750445:0] socket: 560 qp: -1
2025-12-24T02:11:51.932706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456263051261:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:51.932717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:51.933301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241455862427559:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.933323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.933330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241455862427559:2050] [node 3] ICP22 created new session: [5:7587241455862427657:2050]
2025-12-24T02:11:51.933336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241455862427657:2050] [node 3] ICS09 handshake done sender: [5:7587241455862427635:2116] self: [5:7587241455862427655:0] peer: [3:7587241453124296560:0] socket: 537 qp: -1
2025-12-24T02:11:51.933342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241455862427657:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.933348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.935385Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241456263051258:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.935434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241456263051165:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:51.936971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241456263051264:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:1912
2025-12-24T02:11:51.937181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241456263051258:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:51.938093Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241456263051165:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8643) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241456263051258:2121] held: no
2025-12-24T02:11:51.935601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241453124296565:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:51.935618Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241453124296565:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:51.936330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241453124296565:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:51.936405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241453124296558:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:51.936423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:51.938472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241456263051165:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:51.938487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:51.938495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241456263051165:2050] [node 3] ICP22 created new session: [4:7587241456263051265:2050]
2025-12-24T02:11:51.938502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241456263051265:2050] [node 3] ICS09 handshake done sender: [4:7587241456263051258:2121] self: [4:7587241456263051263:0] peer: [3:7587241453124296564:0] socket: 510 qp: -1
2025-12-24T02:11:51.938507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241456263051265:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:51.938518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:51.936431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241453124296558:2109] [node 4] ICP22 created new session: [3:7587241453124296592:2109]
2025-12-24T02:11:51.936438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241453124296592:2109] [node 4] ICS09 handshake done sender: [3:7587241453124296565:2114] self: [3:7587241453124296564:0] peer: [4:7587241456263051263:0] socket: 519 qp: -1
2025-12-24T02:11:51.936444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241453124296592:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:51.936452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:51.960113Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:51.960226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241456696710430:2048] [node 1] ICS07 socket disconnect 502 reason# EndOfStream
2025-12-24T02:11:51.960236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241456696710430:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.960297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.960318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241456696710430:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:51.963022Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:51.963107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241453124296571:2048] [node 1] ICS07 socket disconnect 528 reason# EndOfStream
2025-12-24T02:11:51.963115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241453124296571:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.963169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.963199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241453124296571:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:51.963639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241456696710340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:51.963657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241456696710430:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:51.964100Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241456696710340:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:51.964114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241456696710430:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:51.964121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241456696710340:2048] [node 1] ICP30 unregister session Session# [2:7587241456696710430:2048] VirtualId# [2:7587241456696710421:0]
2025-12-24T02:11:51.964188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.964196Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241456696710340:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.964201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.967214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241453124296488:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:51.967231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241453124296571:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:51.967236Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241453124296488:2048] [node 1] ICP30 unregister session Session# [3:7587241453124296571:2048] VirtualId# [3:7587241453124296569:0]
2025-12-24T02:11:51.967290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.967295Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241453124296488:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.967298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64908) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:51.970746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [2:7587241456696710340:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:51.973772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:51.973885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241454562615633:2048] [node 1] ICS01 socket: 547 reason# EndOfStream
2025-12-24T02:11:51.973895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241454562615537:2048] [node 1] ICP30 unregister session Session# [6:7587241454562615633:2048] VirtualId# [6:7587241454562615627:0]
2025-12-24T02:11:51.973904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241454562615633:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.973950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.973976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.975122Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:51.975196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455862427651:2048] [node 1] ICS01 socket: 533 reason# EndOfStream
2025-12-24T02:11:51.975202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241455862427557:2048] [node 1] ICP30 unregister session Session# [5:7587241455862427651:2048] VirtualId# [5:7587241455862427632:0]
2025-12-24T02:11:51.975208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455862427651:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.975236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.975258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.974897Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:51.974989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241456263051261:2048] [node 1] ICS01 socket: 560 reason# EndOfStream
2025-12-24T02:11:51.974997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241456263051163:2048] [node 1] ICP30 unregister session Session# [4:7587241456263051261:2048] VirtualId# [4:7587241456263051254:0]
2025-12-24T02:11:51.975004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241456263051261:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.975048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:51.975078Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:51.978022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241453124296488:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:51.981847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:51.981954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241453124296586:2049] [node 2] ICS01 socket: 558 reason# EndOfStream
2025-12-24T02:11:51.981963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241453124296489:2049] [node 2] ICP30 unregister session Session# [3:7587241453124296586:2049] VirtualId# [3:7587241453124296577:0]
2025-12-24T02:11:51.981971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241453124296586:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.982023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.982048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.986551Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:51.986645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241454562615622:2049] [node 2] ICS01 socket: 524 reason# EndOfStream
2025-12-24T02:11:51.986655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241454562615538:2049] [node 2] ICP30 unregister session Session# [6:7587241454562615622:2049] VirtualId# [6:7587241454562615617:0]
2025-12-24T02:11:51.986663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241454562615622:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.986709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.986731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.986993Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:51.987064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455862427653:2049] [node 2] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:51.987069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241455862427558:2049] [node 2] ICP30 unregister session Session# [5:7587241455862427653:2049] VirtualId# [5:7587241455862427640:0]
2025-12-24T02:11:51.987073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455862427653:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.987102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.987124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:51.986778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:51.986869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241456263051259:2049] [node 2] ICS01 socket: 564 reason# EndOfStream
2025-12-24T02:11:51.986877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241456263051164:2049] [node 2] ICP30 unregister session Session# [4:7587241456263051259:2049] VirtualId# [4:7587241456263051255:0]
2025-12-24T02:11:51.986883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241456263051259:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:51.986924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:51.986950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.006800Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:52.006913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241454562615639:2050] [node 3] ICS01 socket: 562 reason# EndOfStream
2025-12-24T02:11:52.006926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241454562615539:2050] [node 3] ICP30 unregister session Session# [6:7587241454562615639:2050] VirtualId# [6:7587241454562615637:0]
2025-12-24T02:11:52.006935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241454562615639:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.006982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.007008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.014770Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:52.014875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241455862427657:2050] [node 3] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:52.014885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241455862427559:2050] [node 3] ICP30 unregister session Session# [5:7587241455862427657:2050] VirtualId# [5:7587241455862427655:0]
2025-12-24T02:11:52.014893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241455862427657:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.014951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.014982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.030811Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:52.030923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241454562615635:2104] [node 4] ICS01 socket: 561 reason# EndOfStream
2025-12-24T02:11:52.030934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241454562615599:2104] [node 4] ICP30 unregister session Session# [6:7587241454562615635:2104] VirtualId# [6:7587241454562615630:0]
2025-12-24T02:11:52.030942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241454562615635:2104] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.031004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:52.031032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:52.070946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241457939772775:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:52.070956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241457939772776:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.078178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241460352970509:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.078235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241460352970510:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.078244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241460352970511:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.085970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241459804017835:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.088352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241459804017836:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.090864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241458596124522:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.098579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241458596124523:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.098118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241460075283305:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.098131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241460075283306:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.098138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241460075283307:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.098963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241459960509966:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.098977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241459960509967:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.098983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241459960509968:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.141671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241459804017896:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.143114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241460075283350:2083] [node 4] ICP01 ready to work
2025-12-24T02:11:52.143245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241460075283353:2091] [node 6] ICP01 ready to work
2025-12-24T02:11:52.143294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283342:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.145120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241457939772838:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.154787Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241457939772842:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24349
2025-12-24T02:11:52.154976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124582:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.154995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124583:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.162723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241459960509986:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:52.166722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241459960509987:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:52.167499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241459804017901:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:52.167748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241459804017904:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:52.155423Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241457939772844:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:52.162772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241457939772841:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.181715Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510015:2089] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.179138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241459804017835:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32407850ns) {Configure:104(32407085ns) {ProcessConfigured:135(32007785ns) {StartInitialHandshake:157(32007552ns) {DropHandshakes:470(404ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(240ns) PrepareNewSessionHandshake:477(689ns) {SwitchToState:179(194ns)}}}}}
2025-12-24T02:11:52.175764Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241458596124588:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:52.182511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241458596124596:2113] [node 4] ICP01 ready to work
2025-12-24T02:11:52.184622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283342:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.187165Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241457939772838:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(41977459ns) {Configure:104(41976821ns) {ProcessConfigured:135(41673085ns) {StartInitialHandshake:157(41672718ns) {DropHandshakes:470(409ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(1326ns) {SwitchToState:179(660ns)}}}}}
2025-12-24T02:11:52.187465Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241457939772850:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24349
2025-12-24T02:11:52.187868Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241457939772841:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25017222ns) {Configure:104(25016326ns) {ProcessConfigured:135(24630037ns) {StartInitialHandshake:157(24629716ns) {DropHandshakes:470(673ns) {DropIncomingHandshake:442(188ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(758ns) PrepareNewSessionHandshake:477(499ns) {SwitchToState:179(85ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.187534Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241459960509967:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(41499656ns) {Configure:104(41497658ns) {ProcessConfigured:135(41080304ns) {StartInitialHandshake:157(41079547ns) {DropHandshakes:470(479ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(209ns) PrepareNewSessionHandshake:477(892ns) {SwitchToState:179(288ns)}}}}}
2025-12-24T02:11:52.190144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510043:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.190221Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241459960509986:2060] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27322777ns) {Configure:104(27322040ns) {ProcessConfigured:135(26947772ns) {StartInitialHandshake:157(26947157ns) {DropHandshakes:470(649ns) {DropIncomingHandshake:442(93ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(262ns) PrepareNewSessionHandshake:477(420ns) {SwitchToState:179(230ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.190356Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772842:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.190426Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241457939772776:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34949328ns) {Configure:104(34948625ns) {ProcessConfigured:135(34560266ns) {StartInitialHandshake:157(34559744ns) {DropHandshakes:470(386ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(791ns) {SwitchToState:179(557ns)}}}}}
2025-12-24T02:11:52.190965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241460352970576:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:52.191041Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970574:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.191053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970575:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.191788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772852:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.191794Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241457939772852:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.192162Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241459804017904:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24325811ns) {Configure:104(24325370ns) {ProcessConfigured:135(23933486ns) {StartInitialHandshake:157(23933120ns) {DropHandshakes:470(340ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(219ns) PrepareNewSessionHandshake:477(935ns) {SwitchToState:179(232ns)}}}}}
2025-12-24T02:11:52.191678Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241459960510043:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.191711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241459960509966:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.194896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017898:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.194909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241459804017898:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.198653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970575:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.200771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241459804017916:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36916
2025-12-24T02:11:52.200878Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241459804017914:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36916
2025-12-24T02:11:52.200933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241459804017915:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36916
2025-12-24T02:11:52.202955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017914:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.202997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241459804017835:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.203050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124582:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.203307Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017915:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.203143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772835:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.203154Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241457939772835:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.203781Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241457939772835:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.203852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241457939772775:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.203866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.203874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241457939772775:2048] [node 2] ICP22 created new session: [1:7587241457939772858:2048]
2025-12-24T02:11:52.203882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241457939772858:2048] [node 2] ICS09 handshake done sender: [1:7587241457939772835:2106] self: [1:7587241457939772834:0] peer: [2:7587241459804017917:0] socket: 507 qp: -1
2025-12-24T02:11:52.203898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241457939772858:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:52.203917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.207176Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017916:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.207282Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241459804017919:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36916
2025-12-24T02:11:52.208110Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241459804017914:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.208311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241459804017835:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.208329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.208336Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241459804017835:2048] [node 1] ICP22 created new session: [2:7587241459804017921:2048]
2025-12-24T02:11:52.208342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241459804017921:2048] [node 1] ICS09 handshake done sender: [2:7587241459804017914:2115] self: [2:7587241459804017917:0] peer: [1:7587241457939772834:0] socket: 509 qp: -1
2025-12-24T02:11:52.208348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241459804017921:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.208366Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.207473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283383:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.207521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283384:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.208253Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283385:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.209581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510046:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.207663Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241460352970576:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16046291ns) {Configure:104(16046022ns) {ProcessConfigured:135(15662169ns) {StartInitialHandshake:157(15661057ns) {DropHandshakes:470(154ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(201ns) PrepareNewSessionHandshake:477(949ns) {SwitchToState:179(192ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.210060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124583:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.210211Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124601:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.210297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283383:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.210323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241460075283307:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.211628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241460075283306:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.211674Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241460075283306:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(26977871ns) {Configure:104(26977524ns) {ProcessConfigured:135(26604427ns) {IncomingHandshake:261(26594249ns) {DropIncomingHandshake:442(70ns) PrepareNewSessionHandshake:477(272ns) {SwitchToState:179(127ns)} GenerateSessionVirtualId:988(269ns) IssueIncomingHandshakeReply:193(26590628ns) {SwitchToState:179(748ns)}}}}}
2025-12-24T02:11:52.213700Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241459960510046:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.213741Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241459960509967:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.214766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241460352970571:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:52.214791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241457939772852:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.215654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241458596124599:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:52.216200Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124600:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.216290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124590:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.216296Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241458596124590:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.217092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241458596124590:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.217175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241458596124588:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.217193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.217198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241458596124588:2110] [node 5] ICP22 created new session: [3:7587241458596124606:2110]
2025-12-24T02:11:52.217204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241458596124606:2110] [node 5] ICS09 handshake done sender: [3:7587241458596124590:2111] self: [3:7587241458596124589:0] peer: [5:7587241460075283386:0] socket: 523 qp: -1
2025-12-24T02:11:52.217210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241458596124606:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:52.217218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.217582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124605:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.218104Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283384:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.218732Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241459960509987:2064] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51854867ns) {Configure:104(51854287ns) {ProcessConfigured:135(51507919ns) {StartInitialHandshake:157(51507300ns) {DropHandshakes:470(988ns) {DropIncomingHandshake:442(39ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(377ns) PrepareNewSessionHandshake:477(829ns) {SwitchToState:179(422ns)}}}}}
2025-12-24T02:11:52.218893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772850:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.218912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241457939772841:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.218926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.218932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241457939772841:2108] [node 6] ICP22 created new session: [1:7587241457939772862:2108]
2025-12-24T02:11:52.218937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241457939772862:2108] [node 6] ICS09 handshake done sender: [1:7587241457939772852:2114] self: [1:7587241457939772848:0] peer: [6:7587241459960510045:0] socket: 533 qp: -1
2025-12-24T02:11:52.218942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241457939772862:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:52.218949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.219126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241460352970509:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20388157ns) {Configure:104(20387630ns) {ProcessConfigured:135(20066415ns) {StartInitialHandshake:157(20066221ns) {DropHandshakes:470(680ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(350ns) PrepareNewSessionHandshake:477(618ns) {SwitchToState:179(243ns)}}}}}
2025-12-24T02:11:52.219970Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241460075283307:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241460075283383:2115] held: no
2025-12-24T02:11:52.220096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241460075283305:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.218900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241459804017898:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.220223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241459804017896:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.220231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.220237Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241459804017896:2107] [node 5] ICP22 created new session: [2:7587241459804017925:2107]
2025-12-24T02:11:52.220244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241459804017925:2107] [node 5] ICS09 handshake done sender: [2:7587241459804017898:2108] self: [2:7587241459804017897:0] peer: [5:7587241460075283379:0] socket: 504 qp: -1
2025-12-24T02:11:52.220249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241459804017925:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:52.220266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.222824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124600:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.222852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241458596124523:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.222795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772849:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.222804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241457939772849:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.223754Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241457939772775:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(19835717ns) {HandleHandshakeStatus:329(19835022ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(252ns) SwitchToState:179(59ns) ProcessPendingSessionEvents:495(19731502ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 19719463ns max# 19678357ns}}}
2025-12-24T02:11:52.225423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241457939772865:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24349
2025-12-24T02:11:52.226250Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241457939772849:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.225978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017913:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.225991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241459804017913:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.226873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017909:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.226880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241459804017909:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.227056Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241457939772844:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(39486721ns) {Configure:104(39485856ns) {ProcessConfigured:135(39046359ns) {StartInitialHandshake:157(39045424ns) {DropHandshakes:470(441ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(956ns) {SwitchToState:179(435ns)}}}}}
2025-12-24T02:11:52.227084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241457939772838:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.227095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.227105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241457939772838:2107] [node 5] ICP22 created new session: [1:7587241457939772868:2107]
2025-12-24T02:11:52.227112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241457939772868:2107] [node 5] ICS09 handshake done sender: [1:7587241457939772849:2112] self: [1:7587241457939772839:0] peer: [5:7587241460075283387:0] socket: 531 qp: -1
2025-12-24T02:11:52.227120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241457939772868:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:52.227155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.227746Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241459960509966:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241459960510043:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.231166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241459960510015:2089] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.231206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241459960509987:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.231266Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510050:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.231389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241459960510043:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.231576Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241459960509966:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.231592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.231598Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241459960509966:2048] [node 1] ICP22 created new session: [6:7587241459960510051:2048]
2025-12-24T02:11:52.231604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241459960510051:2048] [node 1] ICS09 handshake done sender: [6:7587241459960510043:2114] self: [6:7587241459960510045:0] peer: [1:7587241457939772848:0] socket: 534 qp: -1
2025-12-24T02:11:52.231611Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241459960510051:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.231628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.231937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510053:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.231878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241458596124596:2113] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(49305911ns) {Configure:104(49304847ns) {ProcessConfigured:135(48899418ns) {StartInitialHandshake:157(48899024ns) {DropHandshakes:470(534ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(626ns) {SwitchToState:179(428ns)}}}}}
2025-12-24T02:11:52.233354Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124601:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.234774Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241460352970510:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51560113ns) {Configure:104(51558408ns) {ProcessConfigured:135(51180555ns) {StartInitialHandshake:157(51180191ns) {DropHandshakes:470(578ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(119ns)} GenerateSessionVirtualId:988(220ns) PrepareNewSessionHandshake:477(628ns) {SwitchToState:179(397ns)}}}}}
2025-12-24T02:11:52.234862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241460352970510:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.235483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283368:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.235503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241460075283368:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.235861Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241459960510053:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.235908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241459960509968:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.239218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283388:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.239403Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283389:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.240269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510055:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.240611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124605:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.240649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241458596124522:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.241905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283390:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.242048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241460075283342:2084] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.240140Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241457939772841:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(21109035ns) {HandleHandshakeStatus:329(21108580ns) {DropIncomingHandshake:442(3555ns) DropOutgoingHandshake:457(107ns) SwitchToState:179(167ns) ProcessPendingSessionEvents:495(13927037ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 13908543ns max# 13831936ns}}}
2025-12-24T02:11:52.240434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772865:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.241522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241457939772870:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24349
2025-12-24T02:11:52.243179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241460075283383:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.244870Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241459804017913:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.245012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124615:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.244297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241460075283306:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.244310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.244315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241460075283306:2049] [node 2] ICP22 created new session: [5:7587241460075283392:2049]
2025-12-24T02:11:52.244320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241460075283392:2049] [node 2] ICS09 handshake done sender: [5:7587241460075283342:2084] self: [5:7587241460075283379:0] peer: [2:7587241459804017897:0] socket: 506 qp: -1
2025-12-24T02:11:52.244326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241460075283392:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.244332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.244734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283385:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.244783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241460075283384:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.245152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241458596124605:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.244998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241459960510046:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.245442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241459804017904:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.245462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.245468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241459804017904:2110] [node 6] ICP22 created new session: [2:7587241459804017928:2110]
2025-12-24T02:11:52.245472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241459804017928:2110] [node 6] ICS09 handshake done sender: [2:7587241459804017913:2114] self: [2:7587241459804017905:0] peer: [6:7587241459960510047:0] socket: 544 qp: -1
2025-12-24T02:11:52.245476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241459804017928:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:52.245482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.246077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241459960509967:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.246095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.246101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241459960509967:2049] [node 2] ICP22 created new session: [6:7587241459960510056:2049]
2025-12-24T02:11:52.246106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241459960510056:2049] [node 2] ICS09 handshake done sender: [6:7587241459960510046:2116] self: [6:7587241459960510047:0] peer: [2:7587241459804017905:0] socket: 545 qp: -1
2025-12-24T02:11:52.246112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241459960510056:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.246118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.246554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241459804017909:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.246658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241460075283350:2083] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.246876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241460075283368:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.245893Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970574:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.245972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970590:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.246874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241458596124522:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.246889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.246894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241458596124522:2048] [node 1] ICP22 created new session: [3:7587241458596124616:2048]
2025-12-24T02:11:52.246901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241458596124616:2048] [node 1] ICS09 handshake done sender: [3:7587241458596124605:2117] self: [3:7587241458596124614:0] peer: [1:7587241457939772845:0] socket: 552 qp: -1
2025-12-24T02:11:52.246906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241458596124616:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.246923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.243246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772854:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.243257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241457939772854:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.243789Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241457939772854:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.243874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241457939772776:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.243906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.243913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241457939772776:2049] [node 3] ICP22 created new session: [1:7587241457939772873:2049]
2025-12-24T02:11:52.243920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241457939772873:2049] [node 3] ICS09 handshake done sender: [1:7587241457939772854:2115] self: [1:7587241457939772845:0] peer: [3:7587241458596124614:0] socket: 549 qp: -1
2025-12-24T02:11:52.243927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241457939772873:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:52.243934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.247083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241460075283307:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.247096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.247103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241460075283307:2050] [node 3] ICP22 created new session: [5:7587241460075283396:2050]
2025-12-24T02:11:52.247110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241460075283396:2050] [node 3] ICS09 handshake done sender: [5:7587241460075283383:2115] self: [5:7587241460075283386:0] peer: [3:7587241458596124589:0] socket: 529 qp: -1
2025-12-24T02:11:52.247117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241460075283396:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.247134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.247216Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017907:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.247227Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241459804017907:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.247570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241459804017836:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.247580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.247586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241459804017836:2049] [node 3] ICP22 created new session: [2:7587241459804017931:2049]
2025-12-24T02:11:52.247590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241459804017931:2049] [node 3] ICS09 handshake done sender: [2:7587241459804017909:2112] self: [2:7587241459804017908:0] peer: [3:7587241458596124608:0] socket: 526 qp: -1
2025-12-24T02:11:52.247595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241459804017931:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:52.247601Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.247152Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124609:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.247163Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241458596124609:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.251490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772870:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.251656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970590:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.251696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241460352970509:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.252034Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241457939772838:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(24846842ns) {HandleHandshakeStatus:329(24846141ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(154ns) SwitchToState:179(85ns) ProcessPendingSessionEvents:495(13968339ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 13949282ns max# 13857491ns}}}
2025-12-24T02:11:52.252070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241457939772838:2107] [node 5] ICP18 handshake (actor [1:7587241457939772870:2118]) is too old
2025-12-24T02:11:52.253443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241458596124618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:10818
2025-12-24T02:11:52.254630Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241457939772856:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.254637Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241457939772856:2116] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.255102Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970593:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.256503Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970593:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.258931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970581:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.258940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241460352970581:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.261853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241460075283305:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.261875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.261882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241460075283305:2048] [node 1] ICP22 created new session: [5:7587241460075283398:2048]
2025-12-24T02:11:52.261890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241460075283398:2048] [node 1] ICS09 handshake done sender: [5:7587241460075283384:2116] self: [5:7587241460075283387:0] peer: [1:7587241457939772839:0] socket: 540 qp: -1
2025-12-24T02:11:52.261897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241460075283398:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.261905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.263656Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241458596124522:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16653555ns) {HandleHandshakeStatus:329(16653111ns) {DropIncomingHandshake:442(146ns) DropOutgoingHandshake:457(1237ns) SwitchToState:179(71ns) ProcessPendingSessionEvents:495(12776745ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 12758758ns max# 12681737ns}}}
2025-12-24T02:11:52.263733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241458596124609:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.264292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241458596124599:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.264313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.264320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241458596124599:2114] [node 6] ICP22 created new session: [3:7587241458596124620:2114]
2025-12-24T02:11:52.264326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241458596124620:2114] [node 6] ICS09 handshake done sender: [3:7587241458596124609:2118] self: [3:7587241458596124603:0] peer: [6:7587241459960510054:0] socket: 558 qp: -1
2025-12-24T02:11:52.264332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241458596124620:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:52.264339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.265400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241458596124523:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13456) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241458596124600:2115] held: no
2025-12-24T02:11:52.265493Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241458596124522:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:52.265733Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241458596124600:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.264746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241459804017907:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.264828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241459804017901:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.264836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.264843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241459804017901:2109] [node 4] ICP22 created new session: [2:7587241459804017934:2109]
2025-12-24T02:11:52.264850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241459804017934:2109] [node 4] ICS09 handshake done sender: [2:7587241459804017907:2111] self: [2:7587241459804017906:0] peer: [4:7587241460352970589:0] socket: 525 qp: -1
2025-12-24T02:11:52.264856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241459804017934:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:52.264862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.266295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241458596124523:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.266308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.266313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241458596124523:2049] [node 2] ICP22 created new session: [3:7587241458596124623:2049]
2025-12-24T02:11:52.266318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241458596124623:2049] [node 2] ICS09 handshake done sender: [3:7587241458596124600:2115] self: [3:7587241458596124608:0] peer: [2:7587241459804017908:0] socket: 528 qp: -1
2025-12-24T02:11:52.266323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241458596124623:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.266328Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.267791Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241459960509968:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241459960510053:2119] held: no
2025-12-24T02:11:52.267935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510061:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.270957Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241460075283305:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:52.271049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283400:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.270912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241459960510053:2119] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.274898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510060:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.278007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241460075283353:2091] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.278036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.278048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241460075283353:2091] [node 6] ICP22 created new session: [5:7587241460075283401:2091]
2025-12-24T02:11:52.278057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241460075283401:2091] [node 6] ICS09 handshake done sender: [5:7587241460075283368:2104] self: [5:7587241460075283359:0] peer: [6:7587241459960510049:0] socket: 513 qp: -1
2025-12-24T02:11:52.278065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241460075283401:2091] [node 6] ICS10 traffic start
2025-12-24T02:11:52.278113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.278398Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241460075283353:2091] [node 6] ICP35 event processing took too much time PendingConnection:279(19526293ns) {HandleHandshakeStatus:329(19525443ns) {DropIncomingHandshake:442(913ns) DropOutgoingHandshake:457(235ns) SwitchToState:179(238ns) ProcessPendingSessionEvents:495(131188ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 107320ns max# 25566ns}}}
2025-12-24T02:11:52.278795Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241460075283307:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(31682910ns) {HandleHandshakeStatus:329(31682379ns) {DropIncomingHandshake:442(133ns) DropOutgoingHandshake:457(70ns) SwitchToState:179(96ns) ProcessPendingSessionEvents:495(19978696ns) {ForwardSessionEventToSession:612(19963749ns) ForwardSessionEventToSession:612(1685ns) ForwardSessionEventToSession:612(1269ns) ForwardSessionEventToSession:612(657ns) ForwardSessionEventToSession:612(741ns) ForwardSessionEventToSession:612(1552ns) ForwardSessionEventToSession:612(888ns) ForwardSessionEventToSession:612(989ns) ForwardSessionEventToSession:612(700ns) ForwardSessionEventToSession:612(1021ns)}}}
2025-12-24T02:11:52.279154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241457939772856:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.279297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241457939772844:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.279322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.279330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241457939772844:2110] [node 4] ICP22 created new session: [1:7587241457939772876:2110]
2025-12-24T02:11:52.279338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241457939772876:2110] [node 4] ICS09 handshake done sender: [1:7587241457939772856:2116] self: [1:7587241457939772846:0] peer: [4:7587241460352970592:0] socket: 557 qp: -1
2025-12-24T02:11:52.279344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241457939772876:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:52.279353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.279876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241460352970581:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.279998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241460352970576:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.280013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.280019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241460352970576:2109] [node 5] ICP22 created new session: [4:7587241460352970596:2109]
2025-12-24T02:11:52.280025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241460352970596:2109] [node 5] ICS09 handshake done sender: [4:7587241460352970581:2113] self: [4:7587241460352970579:0] peer: [5:7587241460075283395:0] socket: 542 qp: -1
2025-12-24T02:11:52.280030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241460352970596:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:52.280037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.279692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241459960509968:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.279710Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.279717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241459960509968:2050] [node 3] ICP22 created new session: [6:7587241459960510062:2050]
2025-12-24T02:11:52.279723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241459960510062:2050] [node 3] ICS09 handshake done sender: [6:7587241459960510053:2119] self: [6:7587241459960510054:0] peer: [3:7587241458596124603:0] socket: 559 qp: -1
2025-12-24T02:11:52.279730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241459960510062:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.279739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.281294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241459960510015:2089] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.282519Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970599:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.282546Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970600:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.282592Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970601:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.282880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510064:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.282793Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241460075283350:2083] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13719) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241460075283385:2117] held: no
2025-12-24T02:11:52.282981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241460075283400:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.283004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [5:7587241460075283353:2091] [node 6] ICP18 handshake (actor [5:7587241460075283400:2121]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.283273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241460075283403:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33558
2025-12-24T02:11:52.282817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970600:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.283042Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241460352970509:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241460352970590:2117] held: no
2025-12-24T02:11:52.283202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241460352970511:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.283386Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241460075283385:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.283375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241458596124612:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.283387Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241458596124612:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.283577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241459960509987:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5763) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241459960510015:2089] held: no
2025-12-24T02:11:52.283830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241460352970590:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.283918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241459960510064:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.283946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241459960509986:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.284079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241459960509987:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.284088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.284093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241459960509987:2064] [node 5] ICP22 created new session: [6:7587241459960510066:2064]
2025-12-24T02:11:52.284097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241459960510066:2064] [node 5] ICS09 handshake done sender: [6:7587241459960510015:2089] self: [6:7587241459960510049:0] peer: [5:7587241460075283359:0] socket: 516 qp: -1
2025-12-24T02:11:52.284101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241459960510066:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:52.284110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.283911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241460075283350:2083] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.283925Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.283931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241460075283350:2083] [node 4] ICP22 created new session: [5:7587241460075283404:2083]
2025-12-24T02:11:52.283937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241460075283404:2083] [node 4] ICS09 handshake done sender: [5:7587241460075283385:2117] self: [5:7587241460075283395:0] peer: [4:7587241460352970579:0] socket: 543 qp: -1
2025-12-24T02:11:52.283944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241460075283404:2083] [node 4] ICS10 traffic start
2025-12-24T02:11:52.283951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.284750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241460352970509:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.284763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.284768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241460352970509:2048] [node 1] ICP22 created new session: [4:7587241460352970604:2048]
2025-12-24T02:11:52.284773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241460352970604:2048] [node 1] ICS09 handshake done sender: [4:7587241460352970590:2117] self: [4:7587241460352970592:0] peer: [1:7587241457939772846:0] socket: 515 qp: -1
2025-12-24T02:11:52.284778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241460352970604:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.284783Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.285897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241460352970586:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.285909Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241460352970586:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.288690Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241458596124612:2119] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.288876Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241460352970575:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.289320Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241460352970606:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38709
2025-12-24T02:11:52.289745Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241458596124596:2113] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.289768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.289773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241458596124596:2113] [node 4] ICP22 created new session: [3:7587241458596124626:2113]
2025-12-24T02:11:52.289778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241458596124626:2113] [node 4] ICS09 handshake done sender: [3:7587241458596124612:2119] self: [3:7587241458596124597:0] peer: [4:7587241460352970602:0] socket: 517 qp: -1
2025-12-24T02:11:52.289784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241458596124626:2113] [node 4] ICS10 traffic start
2025-12-24T02:11:52.289792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.291149Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241459804017936:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36916
2025-12-24T02:11:52.292337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241459804017936:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.292085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241460352970510:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.292105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.292115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241460352970510:2049] [node 2] ICP22 created new session: [4:7587241460352970608:2049]
2025-12-24T02:11:52.292122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241460352970608:2049] [node 2] ICS09 handshake done sender: [4:7587241460352970575:2111] self: [4:7587241460352970589:0] peer: [2:7587241459804017906:0] socket: 535 qp: -1
2025-12-24T02:11:52.292130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241460352970608:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.292136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.295443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241459960510068:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30279
2025-12-24T02:11:52.295577Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241460352970586:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.296086Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241459960510064:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.296238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241459960509986:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.296253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.296259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241459960509986:2060] [node 4] ICP22 created new session: [6:7587241459960510069:2060]
2025-12-24T02:11:52.296266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241459960510069:2060] [node 4] ICS09 handshake done sender: [6:7587241459960510064:2123] self: [6:7587241459960510065:0] peer: [4:7587241460352970585:0] socket: 556 qp: -1
2025-12-24T02:11:52.296272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241459960510069:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:52.296288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.298766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241459804017901:2109] [node 4] ICP18 handshake (actor [2:7587241459804017936:2119]) is too old
2025-12-24T02:11:52.302734Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241460352970511:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241460352970600:2120] held: no
2025-12-24T02:11:52.302856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241460352970600:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.302861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241460352970571:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.302882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.302890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241460352970571:2108] [node 6] ICP22 created new session: [4:7587241460352970610:2108]
2025-12-24T02:11:52.302897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241460352970610:2108] [node 6] ICS09 handshake done sender: [4:7587241460352970586:2115] self: [4:7587241460352970585:0] peer: [6:7587241459960510065:0] socket: 572 qp: -1
2025-12-24T02:11:52.302904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241460352970610:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:52.302913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.307510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241460352970511:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.307536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.307545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241460352970511:2050] [node 3] ICP22 created new session: [4:7587241460352970612:2050]
2025-12-24T02:11:52.307552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241460352970612:2050] [node 3] ICS09 handshake done sender: [4:7587241460352970600:2120] self: [4:7587241460352970602:0] peer: [3:7587241458596124597:0] socket: 532 qp: -1
2025-12-24T02:11:52.307560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241460352970612:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.307569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.319860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:52.319971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241459804017921:2048] [node 1] ICS01 socket: 509 reason# EndOfStream
2025-12-24T02:11:52.319982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241459804017835:2048] [node 1] ICP30 unregister session Session# [2:7587241459804017921:2048] VirtualId# [2:7587241459804017917:0]
2025-12-24T02:11:52.319989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241459804017921:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.320030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.320053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.320489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:52.320563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241458596124616:2048] [node 1] ICS07 socket disconnect 552 reason# EndOfStream
2025-12-24T02:11:52.320568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241458596124616:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.320596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.320614Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241458596124616:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.322928Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241458596124522:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:52.322939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241458596124616:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.323267Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241458596124522:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.323278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241458596124616:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.323285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241458596124522:2048] [node 1] ICP30 unregister session Session# [3:7587241458596124616:2048] VirtualId# [3:7587241458596124614:0]
2025-12-24T02:11:52.323340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.323347Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241458596124522:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.323350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7519) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.334377Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:52.334491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241460352970604:2048] [node 1] ICS01 socket: 515 reason# EndOfStream
2025-12-24T02:11:52.334510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241460352970509:2048] [node 1] ICP30 unregister session Session# [4:7587241460352970604:2048] VirtualId# [4:7587241460352970592:0]
2025-12-24T02:11:52.334519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241460352970604:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.334582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.334615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.338115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241458596124522:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:52.334658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:52.334818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241459960510051:2048] [node 1] ICS01 socket: 534 reason# EndOfStream
2025-12-24T02:11:52.334824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241459960509966:2048] [node 1] ICP30 unregister session Session# [6:7587241459960510051:2048] VirtualId# [6:7587241459960510045:0]
2025-12-24T02:11:52.334829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241459960510051:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.334861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.334881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.338738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:52.338819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241460075283398:2048] [node 1] ICS01 socket: 540 reason# EndOfStream
2025-12-24T02:11:52.338826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241460075283305:2048] [node 1] ICP30 unregister session Session# [5:7587241460075283398:2048] VirtualId# [5:7587241460075283387:0]
2025-12-24T02:11:52.338832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241460075283398:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.338880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.338903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.350451Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:52.350550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241458596124623:2049] [node 2] ICS01 socket: 528 reason# EndOfStream
2025-12-24T02:11:52.350561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241458596124523:2049] [node 2] ICP30 unregister session Session# [3:7587241458596124623:2049] VirtualId# [3:7587241458596124608:0]
2025-12-24T02:11:52.350569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241458596124623:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.350602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.350620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.358913Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:52.359032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241460352970608:2049] [node 2] ICS07 socket disconnect 535 reason# EndOfStream
2025-12-24T02:11:52.359042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241460352970608:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.359106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.359129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241460352970608:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:52.360096Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241460352970510:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13456) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.360109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241460352970608:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.360114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241460352970510:2049] [node 2] ICP30 unregister session Session# [4:7587241460352970608:2049] VirtualId# [4:7587241460352970589:0]
2025-12-24T02:11:52.360165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.360172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241460352970510:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13456) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.360177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13456) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.359390Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:52.359473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241459960510056:2049] [node 2] ICS01 socket: 545 reason# EndOfStream
2025-12-24T02:11:52.359481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241459960509967:2049] [node 2] ICP30 unregister session Session# [6:7587241459960510056:2049] VirtualId# [6:7587241459960510047:0]
2025-12-24T02:11:52.359486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241459960510056:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.359517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.359545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.366755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:52.366857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241460075283392:2049] [node 2] ICS01 socket: 506 reason# EndOfStream
2025-12-24T02:11:52.366868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241460075283306:2049] [node 2] ICP30 unregister session Session# [5:7587241460075283392:2049] VirtualId# [5:7587241460075283379:0]
2025-12-24T02:11:52.366875Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241460075283392:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.366924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.366950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.370865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241460352970510:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:52.379030Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:52.379122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241459960510062:2050] [node 3] ICS01 socket: 559 reason# EndOfStream
2025-12-24T02:11:52.379132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241459960509968:2050] [node 3] ICP30 unregister session Session# [6:7587241459960510062:2050] VirtualId# [6:7587241459960510054:0]
2025-12-24T02:11:52.379139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241459960510062:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.379193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.379219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.379265Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:52.379333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241460352970612:2050] [node 3] ICS01 socket: 532 reason# EndOfStream
2025-12-24T02:11:52.379339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241460352970511:2050] [node 3] ICP30 unregister session Session# [4:7587241460352970612:2050] VirtualId# [4:7587241460352970602:0]
2025-12-24T02:11:52.379343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241460352970612:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.379373Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.379393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.378991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:52.379095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241460075283396:2050] [node 3] ICS07 socket disconnect 529 reason# EndOfStream
2025-12-24T02:11:52.379102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241460075283396:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.379145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.379175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241460075283396:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:52.379603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241460075283307:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.379612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241460075283396:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.379616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241460075283307:2050] [node 3] ICP30 unregister session Session# [5:7587241460075283396:2050] VirtualId# [5:7587241460075283386:0]
2025-12-24T02:11:52.379636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.379643Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241460075283307:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.379648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16938) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.390925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241460075283307:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:52.410858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:52.410999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241459960510069:2060] [node 4] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:52.411011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241459960509986:2060] [node 4] ICP30 unregister session Session# [6:7587241459960510069:2060] VirtualId# [6:7587241459960510065:0]
2025-12-24T02:11:52.411022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241459960510069:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.411092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:52.411120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:52.478802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241458251924724:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.478823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241458251924725:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.479033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241458718612902:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:52.479048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241458718612903:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.478942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241457689544013:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.478953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241457689544014:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.498946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460059373399:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.498968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460059373400:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.478298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457714258011:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.478308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457714258012:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.478313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457714258013:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.490966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241457984132038:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.490979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241457984132039:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.506839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460059373398:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.490985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241457984132040:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.523358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457714258061:2096] [node 6] ICP01 ready to work
2025-12-24T02:11:52.523224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460059373426:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:52.523358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460059373427:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:52.523167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241457984132065:2066] [node 6] ICP01 ready to work
2025-12-24T02:11:52.523270Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241457984132067:2070] [node 4] ICP01 ready to work
2025-12-24T02:11:52.526413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241457689544070:2099] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.529375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132081:2081] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.533303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457714258072:2102] [node 5] ICP01 ready to work
2025-12-24T02:11:52.536213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241458251924789:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:52.535367Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241458718612964:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43307
2025-12-24T02:11:52.542920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241458251924796:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.543712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132117:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.546348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258089:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.546445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258090:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.546613Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258089:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.547065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241458718612975:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43307
2025-12-24T02:11:52.547088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241458718612976:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43307
2025-12-24T02:11:52.554924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612975:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.559233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132117:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.559296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241457984132039:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.559451Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612976:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.562731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241458718612973:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:52.562731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241458718612971:2107] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.565758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258092:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.562756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241458718612974:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:52.567051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132081:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.565986Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241457689544070:2099] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(39467635ns) {Configure:104(39467104ns) {ProcessConfigured:135(38769655ns) {StartInitialHandshake:157(38769207ns) {DropHandshakes:470(395ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(1228ns) {SwitchToState:179(394ns)}}}}}
2025-12-24T02:11:52.566493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241457689544072:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:52.567008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612964:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.566409Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924798:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.566430Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241458251924798:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.566975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241458251924787:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:52.566800Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241460059373399:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(29442134ns) {Configure:104(29441360ns) {ProcessConfigured:135(29208041ns) {StartInitialHandshake:157(29207433ns) {DropHandshakes:470(233ns) {DropIncomingHandshake:442(23ns) DropOutgoingHandshake:457(25ns)} GenerateSessionVirtualId:988(215ns) PrepareNewSessionHandshake:477(2692ns) {SwitchToState:179(1070ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.574673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544086:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.574772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544087:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.577489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241458718612985:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43307
2025-12-24T02:11:52.575013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241458251924725:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40170625ns) {Configure:104(40170140ns) {ProcessConfigured:135(39789522ns) {StartInitialHandshake:157(39789228ns) {DropHandshakes:470(373ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(239ns) PrepareNewSessionHandshake:477(932ns) {SwitchToState:179(517ns)}}}}}
2025-12-24T02:11:52.578482Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241457689544014:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16886114ns) {Configure:104(16885186ns) {ProcessConfigured:135(16547550ns) {StartInitialHandshake:157(16547106ns) {DropHandshakes:470(962ns) {DropIncomingHandshake:442(211ns) DropOutgoingHandshake:457(116ns)} GenerateSessionVirtualId:988(330ns) PrepareNewSessionHandshake:477(955ns) {SwitchToState:179(333ns)}}}}}
2025-12-24T02:11:52.578819Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544088:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.578827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241457689544076:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:52.578918Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544087:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.578926Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544086:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.583588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544088:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.583618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241457689544013:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.582878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241458718612971:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(20031730ns) {Configure:104(20030742ns) {ProcessConfigured:135(19663031ns) {StartInitialHandshake:157(19662704ns) {DropHandshakes:470(856ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(248ns)} GenerateSessionVirtualId:988(231ns) PrepareNewSessionHandshake:477(1285ns) {SwitchToState:179(718ns)}}}}}
2025-12-24T02:11:52.587261Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241458251924798:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.587358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241458251924796:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.587374Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.587384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241458251924796:2111] [node 5] ICP22 created new session: [2:7587241458251924803:2111]
2025-12-24T02:11:52.587391Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241458251924803:2111] [node 5] ICS09 handshake done sender: [2:7587241458251924798:2112] self: [2:7587241458251924797:0] peer: [5:7587241457984132119:0] socket: 531 qp: -1
2025-12-24T02:11:52.587400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241458251924803:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:52.587412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.590342Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241458251924787:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23261232ns) {Configure:104(23260556ns) {ProcessConfigured:135(22889993ns) {StartInitialHandshake:157(22889783ns) {DropHandshakes:470(176ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(942ns) {SwitchToState:179(223ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.590033Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241457689544072:2102] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23448526ns) {Configure:104(23447898ns) {ProcessConfigured:135(23025322ns) {StartInitialHandshake:157(23024639ns) {DropHandshakes:470(592ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(246ns) PrepareNewSessionHandshake:477(1783ns) {SwitchToState:179(330ns)}}}}}
2025-12-24T02:11:52.591194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612966:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.591214Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241458718612966:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.594928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258092:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.595000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457714258011:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.595844Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612980:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.595854Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241458718612980:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.595212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241458251924807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:11:52.595263Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241458251924808:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:11:52.598843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241458718612980:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.598838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924807:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.598890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241458251924724:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.599183Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612985:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.599200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241458718612973:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.599220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.599229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241458718612973:2108] [node 4] ICP22 created new session: [1:7587241458718612989:2108]
2025-12-24T02:11:52.599236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241458718612989:2108] [node 4] ICS09 handshake done sender: [1:7587241458718612980:2115] self: [1:7587241458718612979:0] peer: [4:7587241457714258093:0] socket: 538 qp: -1
2025-12-24T02:11:52.599243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241458718612989:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:52.599264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.594720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544096:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.595331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241457689544076:2106] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16360572ns) {Configure:104(16359452ns) {ProcessConfigured:135(15936358ns) {StartInitialHandshake:157(15935794ns) {DropHandshakes:470(1581ns) {DropIncomingHandshake:442(188ns) DropOutgoingHandshake:457(209ns)} GenerateSessionVirtualId:988(344ns) PrepareNewSessionHandshake:477(717ns) {SwitchToState:179(302ns)}}}}}
2025-12-24T02:11:52.595368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544096:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.595389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241457689544014:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.600507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241457714258011:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241457714258092:2117] held: no
2025-12-24T02:11:52.602928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924801:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.602939Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241458251924801:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.603129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612969:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.603141Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241458718612969:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.606491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924808:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.607182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373477:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.607239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373478:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.607245Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373476:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.607285Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373479:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.608053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241458718612966:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.608174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241458718612903:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.608201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.608209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241458718612903:2049] [node 3] ICP22 created new session: [1:7587241458718612994:2049]
2025-12-24T02:11:52.608215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241458718612994:2049] [node 3] ICS09 handshake done sender: [1:7587241458718612966:2110] self: [1:7587241458718612965:0] peer: [3:7587241457689544093:0] socket: 524 qp: -1
2025-12-24T02:11:52.608223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241458718612994:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:52.608242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.610821Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258090:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.611171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241457689544013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241457689544087:2113] held: no
2025-12-24T02:11:52.611296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544100:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.610976Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241460059373478:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.611076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460059373399:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.611404Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241460059373479:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.611423Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241460059373476:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.611456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460059373400:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.611467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460059373427:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.611533Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241460059373477:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.611547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460059373426:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.614234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924791:2109] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.614247Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241458251924791:2109] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.616086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544082:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.616100Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241457689544082:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.616651Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241457689544082:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.616733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241457689544070:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.616749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.616757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241457689544070:2099] [node 6] ICP22 created new session: [3:7587241457689544103:2099]
2025-12-24T02:11:52.616764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241457689544103:2099] [node 6] ICS09 handshake done sender: [3:7587241457689544082:2110] self: [3:7587241457689544071:0] peer: [6:7587241460059373482:0] socket: 540 qp: -1
2025-12-24T02:11:52.616771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241457689544103:2099] [node 6] ICS10 traffic start
2025-12-24T02:11:52.616791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.617276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241458251924801:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.617405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241458251924725:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.617427Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.617434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241458251924725:2049] [node 3] ICP22 created new session: [2:7587241458251924814:2049]
2025-12-24T02:11:52.617441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241458251924814:2049] [node 3] ICS09 handshake done sender: [2:7587241458251924801:2113] self: [2:7587241458251924788:0] peer: [3:7587241457689544098:0] socket: 549 qp: -1
2025-12-24T02:11:52.617446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241458251924814:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:52.617454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.618019Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241458251924791:2109] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.618198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241458251924789:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.618210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.618217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241458251924789:2106] [node 6] ICP22 created new session: [2:7587241458251924816:2106]
2025-12-24T02:11:52.618222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241458251924816:2106] [node 6] ICS09 handshake done sender: [2:7587241458251924791:2109] self: [2:7587241458251924790:0] peer: [6:7587241460059373481:0] socket: 526 qp: -1
2025-12-24T02:11:52.618227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241458251924816:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:52.618235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.619138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132071:2076] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.619155Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241457984132071:2076] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.619031Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373480:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.620137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241460059373480:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.620186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460059373398:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.622843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258063:2097] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.622861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241457714258063:2097] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.622735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241458251924811:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:11:52.624894Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241457689544087:2113] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.630890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924811:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.631037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132121:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.631171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241458718612969:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.630031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241457689544014:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15287) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241457689544096:2117] held: no
2025-12-24T02:11:52.630173Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241457689544106:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18165
2025-12-24T02:11:52.634111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241457689544013:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.634141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.634152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241457689544013:2048] [node 1] ICP22 created new session: [3:7587241457689544107:2048]
2025-12-24T02:11:52.634160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241457689544107:2048] [node 1] ICS09 handshake done sender: [3:7587241457689544087:2113] self: [3:7587241457689544093:0] peer: [1:7587241458718612965:0] socket: 542 qp: -1
2025-12-24T02:11:52.634170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241457689544107:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.634179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.634571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241458251924819:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.631354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258096:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.634512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241458718612902:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.634541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.634549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241458718612902:2048] [node 2] ICP22 created new session: [1:7587241458718612998:2048]
2025-12-24T02:11:52.634556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241458718612998:2048] [node 2] ICS09 handshake done sender: [1:7587241458718612969:2111] self: [1:7587241458718612968:0] peer: [2:7587241458251924809:0] socket: 525 qp: -1
2025-12-24T02:11:52.634563Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241458718612998:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:52.634571Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.634863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612986:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.634874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241458718612986:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.634956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457714258092:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.634793Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241458251924807:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.638765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457714258011:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.638786Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.638794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457714258011:2048] [node 1] ICP22 created new session: [4:7587241457714258099:2048]
2025-12-24T02:11:52.638802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241457714258099:2048] [node 1] ICS09 handshake done sender: [4:7587241457714258092:2117] self: [4:7587241457714258093:0] peer: [1:7587241458718612979:0] socket: 539 qp: -1
2025-12-24T02:11:52.638811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241457714258099:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.638837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.644231Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132122:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.643074Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373487:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.644807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373486:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.643865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258097:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.644009Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258098:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.648524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241457689544096:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.646859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241460059373479:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.646992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460059373400:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.647006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.647014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460059373400:2050] [node 3] ICP22 created new session: [6:7587241460059373488:2050]
2025-12-24T02:11:52.647022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241460059373488:2050] [node 3] ICS09 handshake done sender: [6:7587241460059373479:2118] self: [6:7587241460059373482:0] peer: [3:7587241457689544071:0] socket: 558 qp: -1
2025-12-24T02:11:52.647032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241460059373488:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.647053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.646790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132123:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.646807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241457984132071:2076] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.646864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132122:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.647043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241457984132065:2066] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.647057Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.647064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241457984132065:2066] [node 6] ICP22 created new session: [5:7587241457984132126:2066]
2025-12-24T02:11:52.647070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241457984132126:2066] [node 6] ICS09 handshake done sender: [5:7587241457984132071:2076] self: [5:7587241457984132070:0] peer: [6:7587241460059373483:0] socket: 511 qp: -1
2025-12-24T02:11:52.647079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241457984132126:2066] [node 6] ICS10 traffic start
2025-12-24T02:11:52.647097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.651017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258097:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.651056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258098:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.654833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241457689544013:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(20669275ns) {HandleHandshakeStatus:329(20667801ns) {DropIncomingHandshake:442(245ns) DropOutgoingHandshake:457(219ns) SwitchToState:179(407ns) ProcessPendingSessionEvents:495(20555775ns) {unordered ForwardSessionEventToSession:612 num# 20 sum# 20548752ns max# 20529819ns}}}
2025-12-24T02:11:52.657492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241457689544014:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.657516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.657527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241457689544014:2049] [node 2] ICP22 created new session: [3:7587241457689544109:2049]
2025-12-24T02:11:52.657535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241457689544109:2049] [node 2] ICS09 handshake done sender: [3:7587241457689544096:2117] self: [3:7587241457689544098:0] peer: [2:7587241458251924788:0] socket: 550 qp: -1
2025-12-24T02:11:52.657544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241457689544109:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.657555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.663676Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241458718612902:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(29056597ns) {HandleHandshakeStatus:329(29055886ns) {DropIncomingHandshake:442(5795ns) DropOutgoingHandshake:457(279ns) SwitchToState:179(241ns) ProcessPendingSessionEvents:495(28951368ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 28931738ns max# 28857093ns}}}
2025-12-24T02:11:52.672465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457714258063:2097] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.672601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241458718612986:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:52.672678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241458718612971:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.672698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.672707Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241458718612971:2107] [node 6] ICP22 created new session: [1:7587241458718613001:2107]
2025-12-24T02:11:52.672714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241458718613001:2107] [node 6] ICS09 handshake done sender: [1:7587241458718612986:2118] self: [1:7587241458718612978:0] peer: [6:7587241460059373485:0] socket: 560 qp: -1
2025-12-24T02:11:52.672719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241458718613001:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:52.672727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.674723Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718612984:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.674742Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241458718612984:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.674729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241458251924724:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.674764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.674781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241458251924724:2048] [node 1] ICP22 created new session: [2:7587241458251924821:2048]
2025-12-24T02:11:52.674790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241458251924821:2048] [node 1] ICS09 handshake done sender: [2:7587241458251924807:2115] self: [2:7587241458251924809:0] peer: [1:7587241458718612968:0] socket: 527 qp: -1
2025-12-24T02:11:52.674806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241458251924821:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.674841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.674927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241458251924806:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.674937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241458251924806:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.674978Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544094:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.674993Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241457689544094:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.675248Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241457984132117:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.675749Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241460059373478:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.675857Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373490:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.675027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241457714258061:2096] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.675051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:52.675061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457714258061:2096] [node 6] ICP22 created new session: [4:7587241457714258104:2096]
2025-12-24T02:11:52.675068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241457714258104:2096] [node 6] ICS09 handshake done sender: [4:7587241457714258063:2097] self: [4:7587241457714258062:0] peer: [6:7587241460059373484:0] socket: 513 qp: -1
2025-12-24T02:11:52.675075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241457714258104:2096] [node 6] ICS10 traffic start
2025-12-24T02:11:52.675084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:52.682805Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132123:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.682835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241457984132039:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.682854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.682862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241457984132039:2049] [node 2] ICP22 created new session: [5:7587241457984132128:2049]
2025-12-24T02:11:52.682871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241457984132128:2049] [node 2] ICS09 handshake done sender: [5:7587241457984132117:2116] self: [5:7587241457984132119:0] peer: [2:7587241458251924797:0] socket: 532 qp: -1
2025-12-24T02:11:52.682879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241457984132128:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.682888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.683426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241457984132040:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.686855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373491:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.687231Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241460059373400:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(40096138ns) {HandleHandshakeStatus:329(40095335ns) {DropIncomingHandshake:442(311ns) DropOutgoingHandshake:457(1660ns) SwitchToState:179(66ns) ProcessPendingSessionEvents:495(14859041ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14839580ns max# 14759470ns}}}
2025-12-24T02:11:52.687261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241460059373492:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18512
2025-12-24T02:11:52.687203Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241457689544097:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.687216Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241457689544097:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.694719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241457984132038:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.694767Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241457984132038:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(47845328ns) {IncomingHandshake:261(47844235ns) {DropIncomingHandshake:442(76ns) PrepareNewSessionHandshake:477(820ns) {SwitchToState:179(308ns)} GenerateSessionVirtualId:988(546ns) IssueIncomingHandshakeReply:193(47837043ns) {SwitchToState:179(859ns)}}}
2025-12-24T02:11:52.694783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457714258013:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.694824Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241457714258013:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(43713776ns) {IncomingHandshake:261(43713349ns) {DropIncomingHandshake:442(50ns) PrepareNewSessionHandshake:477(882ns) {SwitchToState:179(387ns)} GenerateSessionVirtualId:988(280ns) IssueIncomingHandshakeReply:193(43706902ns) {SwitchToState:179(593ns)}}}
2025-12-24T02:11:52.694870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457714258012:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.694891Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241457714258012:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(43763311ns) {IncomingHandshake:261(43763228ns) {DropIncomingHandshake:442(29ns) PrepareNewSessionHandshake:477(366ns) {SwitchToState:179(168ns)} GenerateSessionVirtualId:988(141ns) IssueIncomingHandshakeReply:193(43760040ns) {SwitchToState:179(619ns)}}}
2025-12-24T02:11:52.698666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460059373399:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.698690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.699433Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241458251924823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46907
2025-12-24T02:11:52.702044Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241458718612984:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.702144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241458718612974:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.702156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.702160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241458718612974:2112] [node 5] ICP22 created new session: [1:7587241458718613004:2112]
2025-12-24T02:11:52.702165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241458718613004:2112] [node 5] ICS09 handshake done sender: [1:7587241458718612984:2116] self: [1:7587241458718612983:0] peer: [5:7587241457984132125:0] socket: 523 qp: -1
2025-12-24T02:11:52.702171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241458718613004:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:52.702177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.698968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241457984132065:2066] [node 6] ICP35 event processing took too much time PendingConnection:279(51807523ns) {HandleHandshakeStatus:329(51806971ns) {DropIncomingHandshake:442(3079ns) DropOutgoingHandshake:457(182ns) SwitchToState:179(141ns) ProcessPendingSessionEvents:495(24023889ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24002987ns max# 23919649ns}}}
2025-12-24T02:11:52.699312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241460059373476:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.699552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460059373399:2049] [node 2] ICP22 created new session: [6:7587241460059373494:2049]
2025-12-24T02:11:52.699560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241460059373494:2049] [node 2] ICS09 handshake done sender: [6:7587241460059373478:2117] self: [6:7587241460059373481:0] peer: [2:7587241458251924790:0] socket: 557 qp: -1
2025-12-24T02:11:52.699568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241460059373494:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.699613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.699941Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241460059373398:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241460059373480:2119] held: no
2025-12-24T02:11:52.700148Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241460059373426:2068] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241460059373477:2116] held: no
2025-12-24T02:11:52.701190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241460059373480:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.701277Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241460059373427:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12563) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241460059373476:2115] held: no
2025-12-24T02:11:52.701354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460059373427:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.701362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.701369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460059373427:2070] [node 5] ICP22 created new session: [6:7587241460059373496:2070]
2025-12-24T02:11:52.701373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241460059373496:2070] [node 5] ICS09 handshake done sender: [6:7587241460059373476:2115] self: [6:7587241460059373483:0] peer: [5:7587241457984132070:0] socket: 536 qp: -1
2025-12-24T02:11:52.701380Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241460059373496:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:52.701384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.701534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460059373398:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.701541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.701545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460059373398:2048] [node 1] ICP22 created new session: [6:7587241460059373498:2048]
2025-12-24T02:11:52.701548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241460059373498:2048] [node 1] ICS09 handshake done sender: [6:7587241460059373480:2119] self: [6:7587241460059373485:0] peer: [1:7587241458718612978:0] socket: 546 qp: -1
2025-12-24T02:11:52.701551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241460059373498:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.701556Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.701623Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241460059373477:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.702643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460059373426:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.702651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.702664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460059373426:2068] [node 4] ICP22 created new session: [6:7587241460059373500:2068]
2025-12-24T02:11:52.702667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241460059373500:2068] [node 4] ICS09 handshake done sender: [6:7587241460059373477:2116] self: [6:7587241460059373484:0] peer: [4:7587241457714258062:0] socket: 556 qp: -1
2025-12-24T02:11:52.702672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241460059373500:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:52.702677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.704307Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241457714258061:2096] [node 6] ICP35 event processing took too much time PendingConnection:279(29181984ns) {HandleHandshakeStatus:329(29181246ns) {DropIncomingHandshake:442(5138ns) DropOutgoingHandshake:457(105ns) SwitchToState:179(135ns) ProcessPendingSessionEvents:495(29075446ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 29049038ns max# 28928578ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.705241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241458718613006:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43307
2025-12-24T02:11:52.705958Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241458718613006:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.705989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241458718612974:2112] [node 5] ICP18 handshake (actor [1:7587241458718613006:2119]) is too old
2025-12-24T02:11:52.706946Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241457689544094:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.706901Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241457984132038:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241457984132122:2119] held: no
2025-12-24T02:11:52.707045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241457689544072:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.707062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.707068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241457689544072:2102] [node 4] ICP22 created new session: [3:7587241457689544113:2102]
2025-12-24T02:11:52.707073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241457689544113:2102] [node 4] ICS09 handshake done sender: [3:7587241457689544094:2116] self: [3:7587241457689544084:0] peer: [4:7587241457714258101:0] socket: 564 qp: -1
2025-12-24T02:11:52.707079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241457689544113:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:52.707086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.707196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241457689544097:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.707189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241458251924806:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.707274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241457689544076:2106] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.707291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.707297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241457689544076:2106] [node 5] ICP22 created new session: [3:7587241457689544115:2106]
2025-12-24T02:11:52.707303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241457689544115:2106] [node 5] ICS09 handshake done sender: [3:7587241457689544097:2118] self: [3:7587241457689544092:0] peer: [5:7587241457984132130:0] socket: 566 qp: -1
2025-12-24T02:11:52.707308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241457689544115:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:52.707316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:52.707514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241458251924787:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.707532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.707538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241458251924787:2108] [node 4] ICP22 created new session: [2:7587241458251924825:2108]
2025-12-24T02:11:52.707544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241458251924825:2108] [node 4] ICS09 handshake done sender: [2:7587241458251924806:2114] self: [2:7587241458251924800:0] peer: [4:7587241457714258102:0] socket: 567 qp: -1
2025-12-24T02:11:52.707549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241458251924825:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:52.707554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.709766Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241458718612971:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(36989677ns) {HandleHandshakeStatus:329(36988578ns) {DropIncomingHandshake:442(4547ns) DropOutgoingHandshake:457(428ns) SwitchToState:179(353ns) ProcessPendingSessionEvents:495(18026205ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18004346ns max# 17925168ns}}}
2025-12-24T02:11:52.711866Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241458251924724:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(76865688ns) {HandleHandshakeStatus:329(76865206ns) {DropIncomingHandshake:442(1206ns) DropOutgoingHandshake:457(3281ns) SwitchToState:179(270ns) ProcessPendingSessionEvents:495(20087254ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 20068872ns max# 19998664ns}}}
2025-12-24T02:11:52.711918Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241458251924724:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.715182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241457984132040:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62790) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241457984132123:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.715970Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132132:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.716014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132133:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.716059Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132134:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.716197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241457984132123:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.717211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241457984132040:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.717228Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.717234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241457984132040:2050] [node 3] ICP22 created new session: [5:7587241457984132135:2050]
2025-12-24T02:11:52.717240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241457984132135:2050] [node 3] ICS09 handshake done sender: [5:7587241457984132123:2120] self: [5:7587241457984132130:0] peer: [3:7587241457689544092:0] socket: 571 qp: -1
2025-12-24T02:11:52.717247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241457984132135:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.717269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.717792Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241457984132122:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:52.718062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241457984132038:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.718075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:52.718080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241457984132038:2048] [node 1] ICP22 created new session: [5:7587241457984132137:2048]
2025-12-24T02:11:52.718085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241457984132137:2048] [node 1] ICS09 handshake done sender: [5:7587241457984132122:2119] self: [5:7587241457984132125:0] peer: [1:7587241458718612983:0] socket: 570 qp: -1
2025-12-24T02:11:52.718090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241457984132137:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:52.718099Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:52.714581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258107:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.714749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241457714258108:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28408
2025-12-24T02:11:52.718914Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241457714258013:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62790) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241457714258097:2119] held: no
2025-12-24T02:11:52.719028Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241457714258012:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15287) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241457714258098:2120] held: no
2025-12-24T02:11:52.719574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457714258097:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:52.722933Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241457984132132:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.723015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241457984132067:2070] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:52.719783Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457714258098:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:52.719831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457714258013:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.719842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:52.719849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457714258013:2050] [node 3] ICP22 created new session: [4:7587241457714258109:2050]
2025-12-24T02:11:52.719854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241457714258109:2050] [node 3] ICS09 handshake done sender: [4:7587241457714258097:2119] self: [4:7587241457714258101:0] peer: [3:7587241457689544084:0] socket: 572 qp: -1
2025-12-24T02:11:52.719860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241457714258109:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:52.719867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:52.720069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457714258012:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.720077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:52.720080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457714258012:2049] [node 2] ICP22 created new session: [4:7587241457714258111:2049]
2025-12-24T02:11:52.720084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241457714258111:2049] [node 2] ICS09 handshake done sender: [4:7587241457714258098:2120] self: [4:7587241457714258102:0] peer: [2:7587241458251924800:0] socket: 573 qp: -1
2025-12-24T02:11:52.720090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241457714258111:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:52.720094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:52.726863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457714258088:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:52.726881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241457714258088:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:52.728028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457714258088:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:52.728124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241457714258072:2102] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:52.728148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:52.728157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457714258072:2102] [node 5] ICP22 created new session: [4:7587241457714258114:2102]
2025-12-24T02:11:52.728164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241457714258114:2102] [node 5] ICS09 handshake done sender: [4:7587241457714258088:2114] self: [4:7587241457714258073:0] peer: [5:7587241457984132139:0] socket: 519 qp: -1
2025-12-24T02:11:52.728172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241457714258114:2102] [node 5] ICS10 traffic start
2025-12-24T02:11:52.728181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.730500Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241457984132067:2070] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63598) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241457984132132:2121] held: no
2025-12-24T02:11:52.730714Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241457984132140:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4913
2025-12-24T02:11:52.731113Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241457984132132:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:52.731221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241457984132067:2070] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:52.731235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:52.731240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241457984132067:2070] [node 4] ICP22 created new session: [5:7587241457984132141:2070]
2025-12-24T02:11:52.731247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241457984132141:2070] [node 4] ICS09 handshake done sender: [5:7587241457984132132:2121] self: [5:7587241457984132139:0] peer: [4:7587241457714258073:0] socket: 530 qp: -1
2025-12-24T02:11:52.731254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241457984132141:2070] [node 4] ICS10 traffic start
2025-12-24T02:11:52.731264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:52.746158Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:52.746261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241458251924821:2048] [node 1] ICS07 socket disconnect 527 reason# EndOfStream
2025-12-24T02:11:52.746269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241458251924821:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.746312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.746329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241458251924821:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.747038Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:52.747107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241457689544107:2048] [node 1] ICS07 socket disconnect 542 reason# EndOfStream
2025-12-24T02:11:52.747112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241457689544107:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.747143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.747160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241457689544107:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.750904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241457689544013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:52.750918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241457689544107:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.751281Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241457689544013:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.751292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241457689544107:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.751298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241457689544013:2048] [node 1] ICP30 unregister session Session# [3:7587241457689544107:2048] VirtualId# [3:7587241457689544093:0]
2025-12-24T02:11:52.751329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.751337Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241457689544013:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.751340Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.752328Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241458251924724:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:52.752341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241458251924821:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:52.752704Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241458251924724:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.752715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241458251924821:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.752721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241458251924724:2048] [node 1] ICP30 unregister session Session# [2:7587241458251924821:2048] VirtualId# [2:7587241458251924809:0]
2025-12-24T02:11:52.752773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.752778Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241458251924724:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.752781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11177) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.758603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241457689544013:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:52.762777Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:52.762899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241457984132137:2048] [node 1] ICS01 socket: 570 reason# EndOfStream
2025-12-24T02:11:52.762910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241457984132038:2048] [node 1] ICP30 unregister session Session# [5:7587241457984132137:2048] VirtualId# [5:7587241457984132125:0]
2025-12-24T02:11:52.762919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241457984132137:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.762970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.763003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.762981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:52.763069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241457714258099:2048] [node 1] ICS01 socket: 539 reason# EndOfStream
2025-12-24T02:11:52.763077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241457714258011:2048] [node 1] ICP30 unregister session Session# [4:7587241457714258099:2048] VirtualId# [4:7587241457714258093:0]
2025-12-24T02:11:52.763085Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241457714258099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.763130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.763156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.763249Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:52.763319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241460059373498:2048] [node 1] ICS01 socket: 546 reason# EndOfStream
2025-12-24T02:11:52.763324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460059373398:2048] [node 1] ICP30 unregister session Session# [6:7587241460059373498:2048] VirtualId# [6:7587241460059373485:0]
2025-12-24T02:11:52.763330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241460059373498:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.763362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:52.763387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:52.773602Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:52.773710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241457689544109:2049] [node 2] ICS01 socket: 550 reason# EndOfStream
2025-12-24T02:11:52.773720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241457689544014:2049] [node 2] ICP30 unregister session Session# [3:7587241457689544109:2049] VirtualId# [3:7587241457689544098:0]
2025-12-24T02:11:52.773727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241457689544109:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.773768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.773791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.786651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:52.786773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241457714258111:2049] [node 2] ICS01 socket: 573 reason# EndOfStream
2025-12-24T02:11:52.786785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241457714258012:2049] [node 2] ICP30 unregister session Session# [4:7587241457714258111:2049] VirtualId# [4:7587241457714258102:0]
2025-12-24T02:11:52.786793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241457714258111:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.786858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.786883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.787144Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:52.787205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241460059373494:2049] [node 2] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:52.787209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460059373399:2049] [node 2] ICP30 unregister session Session# [6:7587241460059373494:2049] VirtualId# [6:7587241460059373481:0]
2025-12-24T02:11:52.787214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241460059373494:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.787240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.787261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.786930Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:52.786994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241457984132128:2049] [node 2] ICS01 socket: 532 reason# EndOfStream
2025-12-24T02:11:52.786999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241457984132039:2049] [node 2] ICP30 unregister session Session# [5:7587241457984132128:2049] VirtualId# [5:7587241457984132119:0]
2025-12-24T02:11:52.787004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241457984132128:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.787032Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:52.787055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:52.810994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:52.811100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241460059373488:2050] [node 3] ICS07 socket disconnect 558 reason# EndOfStream
2025-12-24T02:11:52.811107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241460059373488:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.811174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.811197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241460059373488:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:52.811455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:52.811523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241457984132135:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-12-24T02:11:52.811535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241457984132040:2050] [node 3] ICP30 unregister session Session# [5:7587241457984132135:2050] VirtualId# [5:7587241457984132130:0]
2025-12-24T02:11:52.811540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241457984132135:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.811570Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:52.811594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.817232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241460059373400:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62790) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:52.817251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241460059373488:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:52.817258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460059373400:2050] [node 3] ICP30 unregister session Session# [6:7587241460059373488:2050] VirtualId# [6:7587241460059373482:0]
2025-12-24T02:11:52.817324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:52.817333Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241460059373400:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:62790) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.817336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62790) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:52.829555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241460059373400:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:52.842842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:52.842954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241460059373500:2068] [node 4] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:52.842963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460059373426:2068] [node 4] ICP30 unregister session Session# [6:7587241460059373500:2068] VirtualId# [6:7587241460059373484:0]
2025-12-24T02:11:52.842975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241460059373500:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:52.843033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:52.843062Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:52.887066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241459188889695:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:52.887080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241459188889696:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.894834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241460854335188:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.894844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241460854335189:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.895020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241460019691711:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.895036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241460019691712:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:52.906847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457940734046:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.906859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457940734047:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.906864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457940734048:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.917654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460155881465:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.917663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460155881466:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.917668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460155881467:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.918791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241458328348941:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:52.918809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241458328348942:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:52.918815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241458328348943:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:52.974153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241460854335247:2105] [node 6] ICP01 ready to work
2025-12-24T02:11:52.974901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241457940734100:2099] [node 6] ICP01 ready to work
2025-12-24T02:11:52.974953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241459188889757:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:52.975059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241459188889758:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:52.978010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241460155881484:2060] [node 5] ICP01 ready to work
2025-12-24T02:11:52.978930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241458328348992:2090] [node 6] ICP01 ready to work
2025-12-24T02:11:52.979037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241458328348993:2092] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:52.987388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241459188889764:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:52.988330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241459188889765:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.007009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241465149302552:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:53.007118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241465149302553:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:53.008327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302551:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.015739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241460019691711:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(40085156ns) {Configure:104(40083918ns) {ProcessConfigured:135(39687614ns) {StartInitialHandshake:157(39686898ns) {DropHandshakes:470(262ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(2161ns) {SwitchToState:179(1257ns)}}}}}
2025-12-24T02:11:53.016071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241464314659072:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:53.016187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241464314659073:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:53.016285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241464314659074:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:53.017584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241464314659075:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45623
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.021282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241462235701417:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:53.022854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701415:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
2025-12-24T02:11:53.022900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701416:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.018935Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241460155881484:2060] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(40835198ns) {Configure:104(40834615ns) {ProcessConfigured:135(40476277ns) {StartInitialHandshake:157(40476064ns) {DropHandshakes:470(377ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(405ns) PrepareNewSessionHandshake:477(1224ns) {SwitchToState:179(170ns)}}}}}
2025-12-24T02:11:53.019792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464450848832:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:53.020713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848833:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.023387Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848834:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.028537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302564:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
2025-12-24T02:11:53.028762Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241458328348992:2090] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(49651966ns) {Configure:104(49651444ns) {ProcessConfigured:135(49044521ns) {StartInitialHandshake:157(49043732ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(72ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(1490ns) {SwitchToState:179(584ns)}}}}}
2025-12-24T02:11:53.030911Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848835:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.031002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464450848834:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.031029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464450848833:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.032230Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241464314659087:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45623
2025-12-24T02:11:53.032334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241462235701416:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.032759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457940734047:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.032970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659079:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.032977Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241464314659079:2113] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.033564Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701421:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
2025-12-24T02:11:53.033658Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241464314659079:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.033848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241462235701416:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.033901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457940734047:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.033917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.033923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457940734047:2049] [node 2] ICP22 created new session: [4:7587241462235701422:2049]
2025-12-24T02:11:53.033929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241462235701422:2049] [node 2] ICS09 handshake done sender: [4:7587241462235701416:2114] self: [4:7587241462235701420:0] peer: [2:7587241464314659078:0] socket: 543 qp: -1
2025-12-24T02:11:53.033936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241462235701422:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.033946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.034131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241464314659072:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.034142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.034146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241464314659072:2107] [node 4] ICP22 created new session: [2:7587241464314659089:2107]
2025-12-24T02:11:53.034151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241464314659089:2107] [node 4] ICS09 handshake done sender: [2:7587241464314659079:2113] self: [2:7587241464314659078:0] peer: [4:7587241462235701420:0] socket: 534 qp: -1
2025-12-24T02:11:53.034156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241464314659089:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:53.034161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.035687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241465149302551:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.035773Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241465149302564:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.035814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241460854335189:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.036138Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659075:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.044301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464450848832:2069] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.044331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460155881467:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.046813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241460854335251:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.046828Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241460854335251:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.047734Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241460854335251:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.047566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659077:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.047580Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241464314659077:2112] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.051196Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316310:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.051265Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316311:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.051296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316312:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.054867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241457940734105:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.054881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241457940734105:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.055805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241457940734105:2104] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.067871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241460854335247:2105] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.067895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.067916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241460854335247:2105] [node 6] ICP22 created new session: [3:7587241465149302568:2105]
2025-12-24T02:11:53.067925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241465149302568:2105] [node 6] ICS09 handshake done sender: [3:7587241460854335251:2108] self: [3:7587241460854335248:0] peer: [6:7587241464450848843:0] socket: 516 qp: -1
2025-12-24T02:11:53.067933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241465149302568:2105] [node 6] ICS10 traffic start
2025-12-24T02:11:53.067943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.069905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302571:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
2025-12-24T02:11:53.067771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241464314659093:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45623
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.071099Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241465149302564:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.071589Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463483857071:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-12-24T02:11:53.071646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463483857072:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-12-24T02:11:53.071685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463483857073:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:17169
2025-12-24T02:11:53.071881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241460854335189:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.071917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.071926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241460854335189:2049] [node 2] ICP22 created new session: [3:7587241465149302572:2049]
2025-12-24T02:11:53.071932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241465149302572:2049] [node 2] ICS09 handshake done sender: [3:7587241465149302564:2116] self: [3:7587241465149302566:0] peer: [2:7587241464314659076:0] socket: 545 qp: -1
2025-12-24T02:11:53.071939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241465149302572:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.071950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.070852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241464314659077:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.073841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241457940734100:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.073861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.073870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457940734100:2099] [node 6] ICP22 created new session: [4:7587241462235701426:2099]
2025-12-24T02:11:53.073878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241462235701426:2099] [node 6] ICS09 handshake done sender: [4:7587241457940734105:2104] self: [4:7587241457940734101:0] peer: [6:7587241464450848842:0] socket: 518 qp: -1
2025-12-24T02:11:53.073885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241462235701426:2099] [node 6] ICS10 traffic start
2025-12-24T02:11:53.073893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.074215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463483857071:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.074941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241460019691712:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.074966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.074975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241460019691712:2049] [node 3] ICP22 created new session: [2:7587241464314659095:2049]
2025-12-24T02:11:53.074982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241464314659095:2049] [node 3] ICS09 handshake done sender: [2:7587241464314659077:2112] self: [2:7587241464314659076:0] peer: [3:7587241465149302566:0] socket: 533 qp: -1
2025-12-24T02:11:53.074987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241464314659095:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.074994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.078804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463483857072:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.078851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464450848835:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.080647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302575:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.078740Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316312:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.078829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316310:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.078968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848846:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.079144Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848847:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.079192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464450848834:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.079907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464450848832:2069] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.079921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.079930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464450848832:2069] [node 4] ICP22 created new session: [6:7587241464450848848:2069]
2025-12-24T02:11:53.079936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464450848848:2069] [node 4] ICS09 handshake done sender: [6:7587241464450848834:2114] self: [6:7587241464450848842:0] peer: [4:7587241457940734101:0] socket: 537 qp: -1
2025-12-24T02:11:53.079958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464450848848:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:53.079966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.080443Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848844:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.080882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848845:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.080954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460155881465:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.080976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464450848833:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.082240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460155881467:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.082253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.082260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460155881467:2050] [node 3] ICP22 created new session: [6:7587241464450848852:2050]
2025-12-24T02:11:53.082266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464450848852:2050] [node 3] ICS09 handshake done sender: [6:7587241464450848833:2113] self: [6:7587241464450848843:0] peer: [3:7587241460854335248:0] socket: 536 qp: -1
2025-12-24T02:11:53.082273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464450848852:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.082281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.082984Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463483857073:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.083155Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659087:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.090900Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241457940734100:2099] [node 6] ICP35 event processing took too much time PendingConnection:279(16973413ns) {HandleHandshakeStatus:329(16972719ns) {DropIncomingHandshake:442(216ns) DropOutgoingHandshake:457(152ns) SwitchToState:179(81ns) ProcessPendingSessionEvents:495(16874064ns) {unordered ForwardSessionEventToSession:612 num# 60 sum# 16851304ns max# 16797590ns}}}
2025-12-24T02:11:53.091283Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241462235701415:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.091318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889767:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.091331Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241459188889767:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.095505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316311:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.095660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241458328348942:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.095677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241458328348941:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.095801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316318:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.096836Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889765:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.096884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889763:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.096888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241459188889763:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.098202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241459188889767:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.098317Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241459188889763:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.098382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241459188889764:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.098402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.098409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241459188889764:2109] [node 6] ICP22 created new session: [1:7587241463483857080:2109]
2025-12-24T02:11:53.098415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463483857080:2109] [node 6] ICS09 handshake done sender: [1:7587241459188889767:2113] self: [1:7587241459188889766:0] peer: [6:7587241464450848851:0] socket: 525 qp: -1
2025-12-24T02:11:53.098422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463483857080:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:53.098430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.098663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241459188889758:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.098669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.098673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241459188889758:2108] [node 5] ICP22 created new session: [1:7587241463483857082:2108]
2025-12-24T02:11:53.098677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463483857082:2108] [node 5] ICS09 handshake done sender: [1:7587241459188889763:2111] self: [1:7587241459188889762:0] peer: [5:7587241462623316320:0] socket: 524 qp: -1
2025-12-24T02:11:53.098681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463483857082:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:53.098685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.099097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316318:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.099132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241458328348943:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.099207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.099188Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464450848845:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.099454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460155881484:2060] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.099583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241465149302559:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.099595Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241465149302559:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.100016Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241460854335247:2105] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.100167Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302576:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
2025-12-24T02:11:53.100413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241462623316310:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.100488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241458328348941:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.100503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.100509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241458328348941:2048] [node 1] ICP22 created new session: [5:7587241462623316325:2048]
2025-12-24T02:11:53.100515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241462623316325:2048] [node 1] ICS09 handshake done sender: [5:7587241462623316310:2114] self: [5:7587241462623316320:0] peer: [1:7587241459188889762:0] socket: 550 qp: -1
2025-12-24T02:11:53.100521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241462623316325:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.100530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.101854Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241465149302559:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.107730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241465149302552:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.107749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.107758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241465149302552:2109] [node 5] ICP22 created new session: [3:7587241465149302578:2109]
2025-12-24T02:11:53.107766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241465149302578:2109] [node 5] ICS09 handshake done sender: [3:7587241465149302559:2114] self: [3:7587241465149302558:0] peer: [5:7587241462623316323:0] socket: 563 qp: -1
2025-12-24T02:11:53.107774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241465149302578:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:53.107785Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.118928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457940734046:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.119068Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701428:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
2025-12-24T02:11:53.119241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701425:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.119413Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464450848847:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.120445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659081:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.120458Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241464314659081:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.121078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241464314659099:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45623
2025-12-24T02:11:53.121279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316303:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.121291Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241462623316303:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.121788Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316324:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.123135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241462235701428:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.123177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241457940734048:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.123300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889761:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.123310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241459188889761:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.123453Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241464314659081:2114] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.124517Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241465149302576:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.124559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241465149302561:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.124563Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241465149302561:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.125184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241460854335188:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.125268Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302580:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
2025-12-24T02:11:53.125778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241462623316318:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.125834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241464314659073:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.125853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.125862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241464314659073:2108] [node 5] ICP22 created new session: [2:7587241464314659101:2108]
2025-12-24T02:11:53.125869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241464314659101:2108] [node 5] ICS09 handshake done sender: [2:7587241464314659081:2114] self: [2:7587241464314659080:0] peer: [5:7587241462623316319:0] socket: 549 qp: -1
2025-12-24T02:11:53.125877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241464314659101:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:53.125890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.126222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241460155881465:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241464450848835:2115] held: no
2025-12-24T02:11:53.126512Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848854:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.127057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659093:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.127086Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659099:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.127122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241464314659073:2108] [node 5] ICP18 handshake (actor [2:7587241464314659093:2117]) is too old
2025-12-24T02:11:53.127137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241460019691711:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.127455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241460155881466:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.127516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464450848835:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.127757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241458328348943:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.127775Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.127781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241458328348943:2050] [node 3] ICP22 created new session: [5:7587241462623316330:2050]
2025-12-24T02:11:53.127787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241462623316330:2050] [node 3] ICS09 handshake done sender: [5:7587241462623316318:2118] self: [5:7587241462623316323:0] peer: [3:7587241465149302558:0] socket: 570 qp: -1
2025-12-24T02:11:53.127794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241462623316330:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.127801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.127788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241462235701425:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.127842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701432:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
2025-12-24T02:11:53.127914Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889771:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.127925Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241459188889771:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.128611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460155881465:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.128625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.128631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460155881465:2048] [node 1] ICP22 created new session: [6:7587241464450848857:2048]
2025-12-24T02:11:53.128637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464450848857:2048] [node 1] ICS09 handshake done sender: [6:7587241464450848835:2115] self: [6:7587241464450848851:0] peer: [1:7587241459188889766:0] socket: 538 qp: -1
2025-12-24T02:11:53.128642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464450848857:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.130689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.131231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241464314659083:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.131241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241464314659083:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.132071Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848859:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.132318Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241459188889769:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.132328Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241459188889769:2114] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.133060Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241464314659083:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.133252Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464450848847:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.133319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460155881466:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.133333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.133339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460155881466:2049] [node 2] ICP22 created new session: [6:7587241464450848860:2049]
2025-12-24T02:11:53.133346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464450848860:2049] [node 2] ICS09 handshake done sender: [6:7587241464450848847:2121] self: [6:7587241464450848856:0] peer: [2:7587241464314659082:0] socket: 569 qp: -1
2025-12-24T02:11:53.133353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464450848860:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.133359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.131973Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241465149302561:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.134936Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241462235701428:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.134445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241464314659074:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.134465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.134473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241464314659074:2109] [node 6] ICP22 created new session: [2:7587241464314659106:2109]
2025-12-24T02:11:53.134480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241464314659106:2109] [node 6] ICS09 handshake done sender: [2:7587241464314659083:2115] self: [2:7587241464314659082:0] peer: [6:7587241464450848856:0] socket: 531 qp: -1
2025-12-24T02:11:53.134485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241464314659106:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:53.134494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.135147Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241459188889761:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.135288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241459188889771:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.135407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241459188889769:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.135481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241459188889757:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.135497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.135503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241459188889757:2107] [node 4] ICP22 created new session: [1:7587241463483857089:2107]
2025-12-24T02:11:53.135509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463483857089:2107] [node 4] ICS09 handshake done sender: [1:7587241459188889761:2110] self: [1:7587241459188889759:0] peer: [4:7587241462235701430:0] socket: 523 qp: -1
2025-12-24T02:11:53.135514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463483857089:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:53.135522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.135282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241465149302553:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.135300Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.135313Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241465149302553:2110] [node 4] ICP22 created new session: [3:7587241465149302584:2110]
2025-12-24T02:11:53.135319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241465149302584:2110] [node 4] ICS09 handshake done sender: [3:7587241465149302561:2115] self: [3:7587241465149302560:0] peer: [4:7587241462235701431:0] socket: 529 qp: -1
2025-12-24T02:11:53.135324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241465149302584:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:53.135332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.135021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241462235701434:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28227
2025-12-24T02:11:53.135032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457940734048:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.135041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.135047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457940734048:2050] [node 3] ICP22 created new session: [4:7587241462235701435:2050]
2025-12-24T02:11:53.135052Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241462235701435:2050] [node 3] ICS09 handshake done sender: [4:7587241462235701428:2118] self: [4:7587241462235701431:0] peer: [3:7587241465149302560:0] socket: 573 qp: -1
2025-12-24T02:11:53.135057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241462235701435:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.135066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.136357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241459188889696:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.136375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.136380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241459188889696:2049] [node 3] ICP22 created new session: [1:7587241463483857091:2049]
2025-12-24T02:11:53.136385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463483857091:2049] [node 3] ICS09 handshake done sender: [1:7587241459188889771:2115] self: [1:7587241459188889770:0] peer: [3:7587241465149302582:0] socket: 572 qp: -1
2025-12-24T02:11:53.136390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463483857091:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.136395Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.136588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241459188889695:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.136598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.136602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241459188889695:2048] [node 2] ICP22 created new session: [1:7587241463483857093:2048]
2025-12-24T02:11:53.136606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463483857093:2048] [node 2] ICS09 handshake done sender: [1:7587241459188889769:2114] self: [1:7587241459188889768:0] peer: [2:7587241464314659103:0] socket: 568 qp: -1
2025-12-24T02:11:53.136610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463483857093:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:53.136614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.137150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241464314659105:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45623
2025-12-24T02:11:53.137646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241464314659099:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.137704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241460019691711:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.137718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.137722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241460019691711:2048] [node 1] ICP22 created new session: [2:7587241464314659108:2048]
2025-12-24T02:11:53.137727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241464314659108:2048] [node 1] ICS09 handshake done sender: [2:7587241464314659099:2118] self: [2:7587241464314659103:0] peer: [1:7587241459188889768:0] socket: 581 qp: -1
2025-12-24T02:11:53.137732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241464314659108:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.137739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.138841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316329:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.139222Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241462623316312:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.139627Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241462235701415:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.139699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241458328348942:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.139726Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.139731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241458328348942:2049] [node 2] ICP22 created new session: [5:7587241462623316333:2049]
2025-12-24T02:11:53.139736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241462623316333:2049] [node 2] ICS09 handshake done sender: [5:7587241462623316312:2116] self: [5:7587241462623316319:0] peer: [2:7587241464314659080:0] socket: 554 qp: -1
2025-12-24T02:11:53.139743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241462623316333:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.139751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.143036Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241465149302552:2109] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.143054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241460854335188:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4419) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241465149302576:2119] held: no
2025-12-24T02:11:53.143269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241465149302583:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51000
2025-12-24T02:11:53.143366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241457940734046:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.143384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.143391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241457940734046:2048] [node 1] ICP22 created new session: [4:7587241462235701437:2048]
2025-12-24T02:11:53.143397Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241462235701437:2048] [node 1] ICS09 handshake done sender: [4:7587241462235701415:2113] self: [4:7587241462235701430:0] peer: [1:7587241459188889759:0] socket: 542 qp: -1
2025-12-24T02:11:53.143404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241462235701437:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.143411Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.143575Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241457940734046:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.144336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241465149302576:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.144603Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316328:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.144651Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241458328348942:2049] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.144864Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464450848862:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51578
2025-12-24T02:11:53.144953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241460854335188:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.144969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.144975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241460854335188:2048] [node 1] ICP22 created new session: [3:7587241465149302586:2048]
2025-12-24T02:11:53.144980Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241465149302586:2048] [node 1] ICS09 handshake done sender: [3:7587241465149302576:2119] self: [3:7587241465149302582:0] peer: [1:7587241459188889770:0] socket: 577 qp: -1
2025-12-24T02:11:53.144985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241465149302586:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.144992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.145624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241462623316303:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.145805Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464450848845:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.145871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241460155881484:2060] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.145882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.145903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241460155881484:2060] [node 5] ICP22 created new session: [6:7587241464450848863:2060]
2025-12-24T02:11:53.145907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464450848863:2060] [node 5] ICS09 handshake done sender: [6:7587241464450848845:2119] self: [6:7587241464450848855:0] peer: [5:7587241458328348994:0] socket: 566 qp: -1
2025-12-24T02:11:53.145912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464450848863:2060] [node 5] ICS10 traffic start
2025-12-24T02:11:53.145920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.147006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241462623316328:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.147021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241458328348992:2090] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.147034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.147041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241458328348992:2090] [node 6] ICP22 created new session: [5:7587241462623316335:2090]
2025-12-24T02:11:53.147045Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241462623316335:2090] [node 6] ICS09 handshake done sender: [5:7587241462623316303:2110] self: [5:7587241458328348994:0] peer: [6:7587241464450848855:0] socket: 555 qp: -1
2025-12-24T02:11:53.147050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241462623316335:2090] [node 6] ICS10 traffic start
2025-12-24T02:11:53.147056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.148255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241458328348993:2092] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.148400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241462235701419:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.148411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241462235701419:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.149170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241462623316338:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45898
2025-12-24T02:11:53.149280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241462235701419:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.149447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241462623316328:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.149502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241458328348993:2092] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.149514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.149519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241458328348993:2092] [node 4] ICP22 created new session: [5:7587241462623316339:2092]
2025-12-24T02:11:53.149524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241462623316339:2092] [node 4] ICS09 handshake done sender: [5:7587241462623316328:2121] self: [5:7587241462623316337:0] peer: [4:7587241462235701418:0] socket: 585 qp: -1
2025-12-24T02:11:53.149530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241462623316339:2092] [node 4] ICS10 traffic start
2025-12-24T02:11:53.149537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.150427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241462235701417:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.150443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.150448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241462235701417:2111] [node 5] ICP22 created new session: [4:7587241462235701440:2111]
2025-12-24T02:11:53.150454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241462235701440:2111] [node 5] ICS09 handshake done sender: [4:7587241462235701419:2115] self: [4:7587241462235701418:0] peer: [5:7587241462623316337:0] socket: 580 qp: -1
2025-12-24T02:11:53.150458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241462235701440:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:53.150466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.160952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:53.161062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241465149302586:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:53.161074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241460854335188:2048] [node 1] ICP30 unregister session Session# [3:7587241465149302586:2048] VirtualId# [3:7587241465149302582:0]
2025-12-24T02:11:53.161082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241465149302586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.161120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.161155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.162759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:53.162858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241464314659108:2048] [node 1] ICS01 socket: 581 reason# EndOfStream
2025-12-24T02:11:53.162865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241460019691711:2048] [node 1] ICP30 unregister session Session# [2:7587241464314659108:2048] VirtualId# [2:7587241464314659103:0]
2025-12-24T02:11:53.162871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241464314659108:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.162912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.162943Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.166665Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:53.166760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241462235701437:2048] [node 1] ICS01 socket: 542 reason# EndOfStream
2025-12-24T02:11:53.166766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241457940734046:2048] [node 1] ICP30 unregister session Session# [4:7587241462235701437:2048] VirtualId# [4:7587241462235701430:0]
2025-12-24T02:11:53.166773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241462235701437:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.166822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.166845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.166884Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:53.166944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464450848857:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-12-24T02:11:53.166948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460155881465:2048] [node 1] ICP30 unregister session Session# [6:7587241464450848857:2048] VirtualId# [6:7587241464450848851:0]
2025-12-24T02:11:53.166952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464450848857:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.166987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.167006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.170752Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:53.170847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241462623316325:2048] [node 1] ICS01 socket: 550 reason# EndOfStream
2025-12-24T02:11:53.170855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241458328348941:2048] [node 1] ICP30 unregister session Session# [5:7587241462623316325:2048] VirtualId# [5:7587241462623316320:0]
2025-12-24T02:11:53.170860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241462623316325:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.170914Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.170938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.173007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:53.173087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241465149302572:2049] [node 2] ICS01 socket: 545 reason# EndOfStream
2025-12-24T02:11:53.173093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241460854335189:2049] [node 2] ICP30 unregister session Session# [3:7587241465149302572:2049] VirtualId# [3:7587241465149302566:0]
2025-12-24T02:11:53.173099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241465149302572:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.173130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.173149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.182827Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:53.182913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241462235701422:2049] [node 2] ICS07 socket disconnect 543 reason# EndOfStream
2025-12-24T02:11:53.182919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241462235701422:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.182967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.182987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241462235701422:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:53.183198Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:53.183263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464450848860:2049] [node 2] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:53.183269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460155881466:2049] [node 2] ICP30 unregister session Session# [6:7587241464450848860:2049] VirtualId# [6:7587241464450848856:0]
2025-12-24T02:11:53.183274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464450848860:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.183301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.183322Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.186833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:53.186916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241462623316333:2049] [node 2] ICS01 socket: 554 reason# EndOfStream
2025-12-24T02:11:53.186923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241458328348942:2049] [node 2] ICP30 unregister session Session# [5:7587241462623316333:2049] VirtualId# [5:7587241462623316319:0]
2025-12-24T02:11:53.186928Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241462623316333:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.186971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.186996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.187232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241457940734047:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14258) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:53.187248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241462235701422:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:53.187253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241457940734047:2049] [node 2] ICP30 unregister session Session# [4:7587241462235701422:2049] VirtualId# [4:7587241462235701420:0]
2025-12-24T02:11:53.187306Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.187313Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241457940734047:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14258) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.187316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14258) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.198873Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:53.199051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464450848852:2050] [node 3] ICS01 socket: 536 reason# EndOfStream
2025-12-24T02:11:53.199061Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241460155881467:2050] [node 3] ICP30 unregister session Session# [6:7587241464450848852:2050] VirtualId# [6:7587241464450848843:0]
2025-12-24T02:11:53.199069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464450848852:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.199123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.199146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.202764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:53.202862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241462623316330:2050] [node 3] ICS01 socket: 570 reason# EndOfStream
2025-12-24T02:11:53.202869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241458328348943:2050] [node 3] ICP30 unregister session Session# [5:7587241462623316330:2050] VirtualId# [5:7587241462623316323:0]
2025-12-24T02:11:53.202877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241462623316330:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.202923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.202944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.206821Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:53.206902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464450848848:2069] [node 4] ICS01 socket: 537 reason# EndOfStream
2025-12-24T02:11:53.206909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241464450848832:2069] [node 4] ICP30 unregister session Session# [6:7587241464450848848:2069] VirtualId# [6:7587241464450848842:0]
2025-12-24T02:11:53.206915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464450848848:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.206963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:53.206983Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:53.258791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241462950760014:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:53.258807Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241462950760015:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.262721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463845353295:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.262740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463845353296:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.254782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241462768752432:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.254907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241462768752433:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.286803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241463033815459:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.286820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241463033815460:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.286826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241463033815461:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.286853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464659649850:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.286867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464659649851:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.286873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464659649852:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.290765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241462950760036:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:53.290882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241462950760042:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:53.291292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241462950760045:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:53.294300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241462950760053:2076] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46890
2025-12-24T02:11:53.290869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241462768752447:2059] [node 4] ICP01 ready to work
2025-12-24T02:11:53.294925Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241462768752463:2072] [node 5] ICP01 ready to work
2025-12-24T02:11:53.294942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463845353307:2058] [node 6] ICP01 ready to work
2025-12-24T02:11:53.295424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241462768752465:2068] [node 6] ICP01 ready to work
2025-12-24T02:11:53.302334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241463033815495:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:53.302608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464659649877:2072] [node 6] ICP01 ready to work
2025-12-24T02:11:53.302525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241463033815498:2069] [node 6] ICP01 ready to work
2025-12-24T02:11:53.303126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463845353345:2065] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.307606Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815472:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.308773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815514:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.302844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241465182423128:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.302901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241465182423129:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.302910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241465182423130:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.303048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241465182423163:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:53.303253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241465182423170:2067] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.327339Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241462950760036:2064] [node 6] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30528) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:53.327400Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [1:7587241462950760036:2064] [node 6] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:30528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.327404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [6] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:30528) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.327980Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241464659649877:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(25309741ns) {Configure:104(25309371ns) {ProcessConfigured:135(24875791ns) {StartInitialHandshake:157(24875465ns) {DropHandshakes:470(624ns) {DropIncomingHandshake:442(171ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:477(1039ns) {SwitchToState:179(780ns)}}}}}
2025-12-24T02:11:53.329247Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649915:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.329311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649916:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.329982Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241463845353295:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26681050ns) {Configure:104(26680397ns) {ProcessConfigured:135(26303937ns) {StartInitialHandshake:157(26303483ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(264ns) PrepareNewSessionHandshake:477(1329ns) {SwitchToState:179(737ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.339885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [1:7587241462950760036:2064] [node 6] ICP33 wake up from error state
2025-12-24T02:11:53.341081Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241462768752463:2072] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(46090442ns) {Configure:104(46089460ns) {ProcessConfigured:135(45756713ns) {StartInitialHandshake:157(45756175ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(74ns) DropOutgoingHandshake:457(64ns)} GenerateSessionVirtualId:988(96ns) PrepareNewSessionHandshake:477(1459ns) {SwitchToState:179(1015ns)}}}}}
2025-12-24T02:11:53.341453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241462768752510:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25709
2025-12-24T02:11:53.338734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464659649887:2061] [node 4] ICP01 ready to work
2025-12-24T02:11:53.338834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649915:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.341640Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241463033815459:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(39013376ns) {Configure:104(39004069ns) {ProcessConfigured:135(38543337ns) {StartInitialHandshake:157(38543171ns) {DropHandshakes:470(776ns) {DropIncomingHandshake:442(185ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(273ns) PrepareNewSessionHandshake:477(948ns) {SwitchToState:179(218ns)}}}}}
2025-12-24T02:11:53.341748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241463033815472:2060] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.341776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241463033815460:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.339037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463845353338:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:53.342302Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241462768752499:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25709
2025-12-24T02:11:53.341941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760053:2076] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.342034Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241462950760095:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46890
2025-12-24T02:11:53.342681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241463033815514:2099] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.342715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241463033815459:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.346833Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752499:2105] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.347885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241462768752432:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.348033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760029:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.348041Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241462950760029:2061] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.348563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241462950760029:2061] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.348639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241462950760014:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.348661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.348668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241462950760014:2048] [node 2] ICP22 created new session: [1:7587241462950760098:2048]
2025-12-24T02:11:53.348675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241462950760098:2048] [node 2] ICS09 handshake done sender: [1:7587241462950760029:2061] self: [1:7587241462950760028:0] peer: [2:7587241462768752512:0] socket: 518 qp: -1
2025-12-24T02:11:53.348683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241462950760098:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:53.348702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.349944Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241465182423128:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(46601752ns) {Configure:104(46601118ns) {ProcessConfigured:135(46186220ns) {StartInitialHandshake:157(46185814ns) {DropHandshakes:470(411ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1538ns) {SwitchToState:179(1066ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.350947Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241463845353296:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20400636ns) {Configure:104(20400145ns) {ProcessConfigured:135(20156625ns) {StartInitialHandshake:157(20156503ns) {DropHandshakes:470(273ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(106ns)} GenerateSessionVirtualId:988(214ns) PrepareNewSessionHandshake:477(1149ns) {SwitchToState:179(84ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.353787Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752450:2063] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.353803Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241462768752450:2063] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.353608Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353374:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.353673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353375:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.353706Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353376:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.353162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423197:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.354932Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353374:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.355007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760047:2073] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.355017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241462950760047:2073] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.358170Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241462768752432:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241462768752499:2105] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.358357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241462768752513:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25709
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.359158Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241462768752499:2105] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.360374Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241462768752432:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.360392Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.360398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241462768752432:2048] [node 1] ICP22 created new session: [2:7587241462768752517:2048]
2025-12-24T02:11:53.360404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241462768752517:2048] [node 1] ICS09 handshake done sender: [2:7587241462768752499:2105] self: [2:7587241462768752512:0] peer: [1:7587241462950760028:0] socket: 535 qp: -1
2025-12-24T02:11:53.360411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241462768752517:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.360428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.360820Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241462768752450:2063] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.360898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241462768752447:2059] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.360909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.360914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241462768752447:2059] [node 4] ICP22 created new session: [2:7587241462768752519:2059]
2025-12-24T02:11:53.360918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241462768752519:2059] [node 4] ICS09 handshake done sender: [2:7587241462768752450:2063] self: [2:7587241462768752449:0] peer: [4:7587241463033815538:0] socket: 519 qp: -1
2025-12-24T02:11:53.360922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241462768752519:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:53.360926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.362421Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353375:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.362507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241463845353295:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.362577Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353378:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.358188Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241464659649887:2061] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19202090ns) {Configure:104(19201454ns) {ProcessConfigured:135(18817106ns) {StartInitialHandshake:157(18816881ns) {DropHandshakes:470(437ns) {DropIncomingHandshake:442(87ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(1292ns) {SwitchToState:179(914ns)}}}}}
2025-12-24T02:11:53.358381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464659649887:2061] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.362811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241463033815516:2100] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.362834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241463033815516:2100] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.362878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241465182423197:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.363186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241462950760102:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46890
2025-12-24T02:11:53.363363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760093:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.363369Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241462950760093:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.363828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241462950760101:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46890
2025-12-24T02:11:53.362971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815541:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.363983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241463033815472:2060] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.364073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241463033815460:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.364084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.364091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241463033815460:2049] [node 2] ICP22 created new session: [4:7587241463033815542:2049]
2025-12-24T02:11:53.364096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241463033815542:2049] [node 2] ICS09 handshake done sender: [4:7587241463033815472:2060] self: [4:7587241463033815538:0] peer: [2:7587241462768752449:0] socket: 525 qp: -1
2025-12-24T02:11:53.364101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241463033815542:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.364108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.368154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649931:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.368204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649916:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.368277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464659649850:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.368330Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649930:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.368392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649933:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.370780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760101:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.370839Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241462950760093:2115] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.372939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241465182423163:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.373018Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423208:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.373049Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423209:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.373080Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423206:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.374014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423211:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.374129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241465182423197:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.374165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241465182423163:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.374175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.374179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241465182423163:2060] [node 4] ICP22 created new session: [6:7587241465182423212:2060]
2025-12-24T02:11:53.374184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241465182423212:2060] [node 4] ICS09 handshake done sender: [6:7587241465182423197:2110] self: [6:7587241465182423210:0] peer: [4:7587241463033815519:0] socket: 550 qp: -1
2025-12-24T02:11:53.374188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241465182423212:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:53.374202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.373166Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241463033815520:2102] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.373176Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241463033815520:2102] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.373593Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241463033815520:2102] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.373653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241463033815498:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.373664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.373671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241463033815498:2069] [node 6] ICP22 created new session: [4:7587241463033815546:2069]
2025-12-24T02:11:53.373678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241463033815546:2069] [node 6] ICS09 handshake done sender: [4:7587241463033815520:2102] self: [4:7587241463033815519:0] peer: [6:7587241465182423210:0] socket: 532 qp: -1
2025-12-24T02:11:53.373683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241463033815546:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:53.373692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.374816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649930:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.374887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464659649852:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.375524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353362:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.375534Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463845353362:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.376019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353323:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.376023Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463845353323:2073] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.376389Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353376:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.376673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241463033815516:2100] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.376782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241463033815495:2059] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.376795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.376802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241463033815495:2059] [node 5] ICP22 created new session: [4:7587241463033815549:2059]
2025-12-24T02:11:53.376808Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241463033815549:2059] [node 5] ICS09 handshake done sender: [4:7587241463033815516:2100] self: [4:7587241463033815515:0] peer: [5:7587241464659649932:0] socket: 530 qp: -1
2025-12-24T02:11:53.376814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241463033815549:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:53.376822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.377176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463845353362:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.377295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463845353323:2073] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.377365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463845353345:2065] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.377378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.377383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463845353345:2065] [node 5] ICP22 created new session: [3:7587241463845353384:2065]
2025-12-24T02:11:53.377388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463845353384:2065] [node 5] ICS09 handshake done sender: [3:7587241463845353362:2108] self: [3:7587241463845353361:0] peer: [5:7587241464659649935:0] socket: 552 qp: -1
2025-12-24T02:11:53.377394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463845353384:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:53.377399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.377594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463845353307:2058] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.377604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.377609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463845353307:2058] [node 6] ICP22 created new session: [3:7587241463845353386:2058]
2025-12-24T02:11:53.377613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463845353386:2058] [node 6] ICS09 handshake done sender: [3:7587241463845353323:2073] self: [3:7587241463845353308:0] peer: [6:7587241465182423215:0] socket: 544 qp: -1
2025-12-24T02:11:53.377616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463845353386:2058] [node 6] ICS10 traffic start
2025-12-24T02:11:53.377621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.378319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815551:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.374954Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241465182423208:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.375003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241465182423206:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.375068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241465182423130:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.375077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241465182423170:2067] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.379091Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649931:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.379162Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353380:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.380487Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463845353375:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.380990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241462768752522:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25709
2025-12-24T02:11:53.381012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241462768752523:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25709
2025-12-24T02:11:53.380703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241463845353295:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.380717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.380724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463845353295:2048] [node 1] ICP22 created new session: [3:7587241463845353390:2048]
2025-12-24T02:11:53.380729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463845353390:2048] [node 1] ICS09 handshake done sender: [3:7587241463845353375:2116] self: [3:7587241463845353379:0] peer: [1:7587241462950760092:0] socket: 557 qp: -1
2025-12-24T02:11:53.380735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463845353390:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.380741Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.381525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241463033815551:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.381606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241463033815461:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.381802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353371:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.381809Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463845353371:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.382419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815553:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.382536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463845353371:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.382739Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241463033815551:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.382840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241463033815461:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.382854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.382861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241463033815461:2050] [node 3] ICP22 created new session: [4:7587241463033815554:2050]
2025-12-24T02:11:53.382867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241463033815554:2050] [node 3] ICS09 handshake done sender: [4:7587241463033815551:2118] self: [4:7587241463033815552:0] peer: [3:7587241463845353370:0] socket: 583 qp: -1
2025-12-24T02:11:53.382873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241463033815554:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.382880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.383529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241462950760015:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.383548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.383556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241462950760015:2049] [node 3] ICP22 created new session: [1:7587241462950760106:2049]
2025-12-24T02:11:53.383562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241462950760106:2049] [node 3] ICS09 handshake done sender: [1:7587241462950760093:2115] self: [1:7587241462950760092:0] peer: [3:7587241463845353379:0] socket: 524 qp: -1
2025-12-24T02:11:53.383567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241462950760106:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.383575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.384075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241462950760104:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:46890
2025-12-24T02:11:53.384114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241462950760015:2049] [node 3] ICP18 handshake (actor [1:7587241462950760101:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.384407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241465182423209:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.384444Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241465182423130:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5288) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241465182423208:2117] held: no
2025-12-24T02:11:53.384598Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423217:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.384752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241465182423129:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.384765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241465182423208:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.384853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241465182423130:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.384864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.384870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241465182423130:2050] [node 3] ICP22 created new session: [6:7587241465182423219:2050]
2025-12-24T02:11:53.384874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241465182423219:2050] [node 3] ICS09 handshake done sender: [6:7587241465182423208:2117] self: [6:7587241465182423215:0] peer: [3:7587241463845353308:0] socket: 569 qp: -1
2025-12-24T02:11:53.384879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241465182423219:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.384897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.386823Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752498:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.386834Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241462768752498:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.390743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760049:2074] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.390760Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241462950760049:2074] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.390909Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241462950760047:2073] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.390801Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649914:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.390817Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241464659649914:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.391459Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241464659649933:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.391578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464659649851:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.391778Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649939:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.392121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463845353338:2060] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.392144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.392151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463845353338:2060] [node 4] ICP22 created new session: [3:7587241463845353393:2060]
2025-12-24T02:11:53.392158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463845353393:2060] [node 4] ICS09 handshake done sender: [3:7587241463845353371:2114] self: [3:7587241463845353370:0] peer: [4:7587241463033815552:0] socket: 582 qp: -1
2025-12-24T02:11:53.392163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463845353393:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:53.392173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.392789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463845353378:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.394779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241462768752498:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.394803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752510:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.394842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752509:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.394845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241462768752509:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.395574Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241462768752509:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.395672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241462768752463:2072] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.395692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.395698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241462768752463:2072] [node 5] ICP22 created new session: [2:7587241462768752527:2072]
2025-12-24T02:11:53.395706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241462768752527:2072] [node 5] ICS09 handshake done sender: [2:7587241462768752509:2114] self: [2:7587241462768752464:0] peer: [5:7587241464659649938:0] socket: 565 qp: -1
2025-12-24T02:11:53.395712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241462768752527:2072] [node 5] ICS10 traffic start
2025-12-24T02:11:53.395719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.399239Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649940:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.402242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241462768752465:2068] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.402261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.402269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241462768752465:2068] [node 6] ICP22 created new session: [2:7587241462768752529:2068]
2025-12-24T02:11:53.402277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241462768752529:2068] [node 6] ICS09 handshake done sender: [2:7587241462768752498:2104] self: [2:7587241462768752466:0] peer: [6:7587241465182423218:0] socket: 547 qp: -1
2025-12-24T02:11:53.402285Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241462768752529:2068] [node 6] ICS10 traffic start
2025-12-24T02:11:53.402296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.402868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241463845353296:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.403867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353396:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.404154Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463845353378:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.404217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241463845353296:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.404232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.404241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463845353296:2049] [node 2] ICP22 created new session: [3:7587241463845353397:2049]
2025-12-24T02:11:53.404247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463845353397:2049] [node 2] ICS09 handshake done sender: [3:7587241463845353378:2118] self: [3:7587241463845353395:0] peer: [2:7587241462768752500:0] socket: 567 qp: -1
2025-12-24T02:11:53.404252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463845353397:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.404259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.403067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752523:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.403101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752501:2106] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.403104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241462768752501:2106] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.403984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241462768752501:2106] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.405116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241462768752433:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.405128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.405134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241462768752433:2049] [node 3] ICP22 created new session: [2:7587241462768752533:2049]
2025-12-24T02:11:53.405140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241462768752533:2049] [node 3] ICS09 handshake done sender: [2:7587241462768752501:2106] self: [2:7587241462768752500:0] peer: [3:7587241463845353395:0] socket: 566 qp: -1
2025-12-24T02:11:53.405145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241462768752533:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.405151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.407442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760095:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.407458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241462950760042:2068] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.407465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.407470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241462950760042:2068] [node 4] ICP22 created new session: [1:7587241462950760108:2068]
2025-12-24T02:11:53.407474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241462950760108:2068] [node 4] ICS09 handshake done sender: [1:7587241462950760047:2073] self: [1:7587241462950760046:0] peer: [4:7587241463033815539:0] socket: 522 qp: -1
2025-12-24T02:11:53.407479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241462950760108:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:53.407487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.407694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760104:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.407791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241462950760042:2068] [node 4] ICP18 handshake (actor [1:7587241462950760104:2119]) is too old
2025-12-24T02:11:53.410880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815556:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.410995Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241462950760049:2074] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.411891Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241464659649914:2108] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.414802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241462768752522:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.411148Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423221:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.412054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241465182423222:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16503
2025-12-24T02:11:53.412173Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241465182423209:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.412215Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241465182423206:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.412256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241465182423129:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.412267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.412274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241465182423129:2049] [node 2] ICP22 created new session: [6:7587241465182423223:2049]
2025-12-24T02:11:53.412281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241465182423223:2049] [node 2] ICS09 handshake done sender: [6:7587241465182423209:2118] self: [6:7587241465182423218:0] peer: [2:7587241462768752466:0] socket: 575 qp: -1
2025-12-24T02:11:53.412286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241465182423223:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.412293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.414280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241465182423170:2067] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.414292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.414300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241465182423170:2067] [node 5] ICP22 created new session: [6:7587241465182423225:2067]
2025-12-24T02:11:53.414306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241465182423225:2067] [node 5] ICS09 handshake done sender: [6:7587241465182423206:2115] self: [6:7587241465182423216:0] peer: [5:7587241464659649895:0] socket: 520 qp: -1
2025-12-24T02:11:53.414311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241465182423225:2067] [node 5] ICS10 traffic start
2025-12-24T02:11:53.414317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.415890Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241463033815459:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241463033815514:2099] held: no
2025-12-24T02:11:53.414999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241462768752465:2068] [node 6] ICP18 handshake (actor [2:7587241462768752522:2118]) is too old
2025-12-24T02:11:53.418899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241462950760102:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.418914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241462950760045:2070] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.418927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.418934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241462950760045:2070] [node 5] ICP22 created new session: [1:7587241462950760112:2070]
2025-12-24T02:11:53.418940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241462950760112:2070] [node 5] ICS09 handshake done sender: [1:7587241462950760049:2074] self: [1:7587241462950760048:0] peer: [5:7587241464659649934:0] socket: 536 qp: -1
2025-12-24T02:11:53.418945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241462950760112:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:53.418953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.419589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241462950760036:2064] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.420326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241464659649915:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.420401Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241464659649851:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241464659649933:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.418827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241463033815514:2099] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.419709Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241465182423192:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.419719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241465182423192:2107] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.420366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241465182423192:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.420471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241465182423128:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.420479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.420486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241465182423128:2048] [node 1] ICP22 created new session: [6:7587241465182423228:2048]
2025-12-24T02:11:53.420491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241465182423228:2048] [node 1] ICS09 handshake done sender: [6:7587241465182423192:2107] self: [6:7587241465182423172:0] peer: [1:7587241462950760114:0] socket: 554 qp: -1
2025-12-24T02:11:53.420496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241465182423228:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.420503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.422934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241463033815459:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.422950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.422957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241463033815459:2048] [node 1] ICP22 created new session: [4:7587241463033815557:2048]
2025-12-24T02:11:53.422963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241463033815557:2048] [node 1] ICS09 handshake done sender: [4:7587241463033815514:2099] self: [4:7587241463033815539:0] peer: [1:7587241462950760046:0] socket: 527 qp: -1
2025-12-24T02:11:53.422970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241463033815557:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.422978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.423523Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241463033815559:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:62583
2025-12-24T02:11:53.423539Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463845353399:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43028
2025-12-24T02:11:53.421058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241464659649930:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.421213Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241464659649850:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10935) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241464659649916:2110] held: no
2025-12-24T02:11:53.421292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241464659649877:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.421304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.421310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464659649877:2072] [node 6] ICP22 created new session: [5:7587241464659649945:2072]
2025-12-24T02:11:53.421314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241464659649945:2072] [node 6] ICS09 handshake done sender: [5:7587241464659649914:2108] self: [5:7587241464659649895:0] peer: [6:7587241465182423216:0] socket: 539 qp: -1
2025-12-24T02:11:53.421318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241464659649945:2072] [node 6] ICS10 traffic start
2025-12-24T02:11:53.421323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.422280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464659649887:2061] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.422290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.422294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464659649887:2061] [node 4] ICP22 created new session: [5:7587241464659649947:2061]
2025-12-24T02:11:53.422298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241464659649947:2061] [node 4] ICS09 handshake done sender: [5:7587241464659649915:2109] self: [5:7587241464659649932:0] peer: [4:7587241463033815515:0] socket: 537 qp: -1
2025-12-24T02:11:53.422303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241464659649947:2061] [node 4] ICS10 traffic start
2025-12-24T02:11:53.422307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.422770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464659649852:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.422780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.422784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464659649852:2050] [node 3] ICP22 created new session: [5:7587241464659649949:2050]
2025-12-24T02:11:53.422789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241464659649949:2050] [node 3] ICS09 handshake done sender: [5:7587241464659649930:2117] self: [5:7587241464659649935:0] peer: [3:7587241463845353361:0] socket: 561 qp: -1
2025-12-24T02:11:53.422793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241464659649949:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.422798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.423150Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649941:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.423377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241464659649942:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:40751
2025-12-24T02:11:53.423987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241464659649933:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.424050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241464659649916:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.424095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464659649851:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.424108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.424112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464659649850:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.424114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464659649851:2049] [node 2] ICP22 created new session: [5:7587241464659649951:2049]
2025-12-24T02:11:53.424117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241464659649951:2049] [node 2] ICS09 handshake done sender: [5:7587241464659649933:2119] self: [5:7587241464659649938:0] peer: [2:7587241462768752464:0] socket: 573 qp: -1
2025-12-24T02:11:53.424121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.424123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241464659649951:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.424125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464659649850:2048] [node 1] ICP22 created new session: [5:7587241464659649952:2048]
2025-12-24T02:11:53.424128Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.424130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241464659649952:2048] [node 1] ICS09 handshake done sender: [5:7587241464659649916:2110] self: [5:7587241464659649934:0] peer: [1:7587241462950760048:0] socket: 538 qp: -1
2025-12-24T02:11:53.424135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241464659649952:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.424139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.438812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:53.438919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241462768752517:2048] [node 1] ICS01 socket: 535 reason# EndOfStream
2025-12-24T02:11:53.438928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241462768752432:2048] [node 1] ICP30 unregister session Session# [2:7587241462768752517:2048] VirtualId# [2:7587241462768752512:0]
2025-12-24T02:11:53.438935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241462768752517:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.438976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.438999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.438998Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:53.439060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241463845353390:2048] [node 1] ICS01 socket: 557 reason# EndOfStream
2025-12-24T02:11:53.439066Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241463845353295:2048] [node 1] ICP30 unregister session Session# [3:7587241463845353390:2048] VirtualId# [3:7587241463845353379:0]
2025-12-24T02:11:53.439070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241463845353390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.439096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.439119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.452918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:53.453029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241465182423228:2048] [node 1] ICS01 socket: 554 reason# EndOfStream
2025-12-24T02:11:53.453038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241465182423128:2048] [node 1] ICP30 unregister session Session# [6:7587241465182423228:2048] VirtualId# [6:7587241465182423172:0]
2025-12-24T02:11:53.453047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241465182423228:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.453097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.453131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.454744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:53.454833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241463033815557:2048] [node 1] ICS01 socket: 527 reason# EndOfStream
2025-12-24T02:11:53.454840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241463033815459:2048] [node 1] ICP30 unregister session Session# [4:7587241463033815557:2048] VirtualId# [4:7587241463033815539:0]
2025-12-24T02:11:53.454846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241463033815557:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.454903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.454932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.454977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:53.455046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241464659649952:2048] [node 1] ICS01 socket: 538 reason# EndOfStream
2025-12-24T02:11:53.455051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464659649850:2048] [node 1] ICP30 unregister session Session# [5:7587241464659649952:2048] VirtualId# [5:7587241464659649934:0]
2025-12-24T02:11:53.455057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241464659649952:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.455089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.455113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.460485Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:53.460581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241463845353397:2049] [node 2] ICS01 socket: 567 reason# EndOfStream
2025-12-24T02:11:53.460587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241463845353296:2049] [node 2] ICP30 unregister session Session# [3:7587241463845353397:2049] VirtualId# [3:7587241463845353395:0]
2025-12-24T02:11:53.460593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241463845353397:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.460629Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.460648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.469563Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:53.469671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241463033815542:2049] [node 2] ICS01 socket: 525 reason# EndOfStream
2025-12-24T02:11:53.469682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241463033815460:2049] [node 2] ICP30 unregister session Session# [4:7587241463033815542:2049] VirtualId# [4:7587241463033815538:0]
2025-12-24T02:11:53.469690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241463033815542:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.469737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.469763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.469797Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:53.469866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241465182423223:2049] [node 2] ICS01 socket: 575 reason# EndOfStream
2025-12-24T02:11:53.469873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241465182423129:2049] [node 2] ICP30 unregister session Session# [6:7587241465182423223:2049] VirtualId# [6:7587241465182423218:0]
2025-12-24T02:11:53.469878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241465182423223:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.469905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.469926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.471027Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:53.471125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241464659649951:2049] [node 2] ICS01 socket: 573 reason# EndOfStream
2025-12-24T02:11:53.471133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464659649851:2049] [node 2] ICP30 unregister session Session# [5:7587241464659649951:2049] VirtualId# [5:7587241464659649938:0]
2025-12-24T02:11:53.471140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241464659649951:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.471193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.471221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.493346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:53.493451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241465182423219:2050] [node 3] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:53.493460Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241465182423130:2050] [node 3] ICP30 unregister session Session# [6:7587241465182423219:2050] VirtualId# [6:7587241465182423215:0]
2025-12-24T02:11:53.493468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241465182423219:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.493513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.493535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.498751Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:53.498850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241464659649949:2050] [node 3] ICS01 socket: 561 reason# EndOfStream
2025-12-24T02:11:53.498860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464659649852:2050] [node 3] ICP30 unregister session Session# [5:7587241464659649949:2050] VirtualId# [5:7587241464659649935:0]
2025-12-24T02:11:53.498867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241464659649949:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.498921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.498947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.513928Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:53.514022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241465182423212:2060] [node 4] ICS01 socket: 550 reason# EndOfStream
2025-12-24T02:11:53.514034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241465182423163:2060] [node 4] ICP30 unregister session Session# [6:7587241465182423212:2060] VirtualId# [6:7587241465182423210:0]
2025-12-24T02:11:53.514040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241465182423212:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.514090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:53.514114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:53.566897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241465060094959:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.566913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241465060094960:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.574778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241465031242770:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.574803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241465031242772:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.574835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241465031242771:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.574802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463979910836:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.574812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463979910837:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.567253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463990027730:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:53.567262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463990027731:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.575505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465380509655:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.575513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465380509656:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.575518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465380509657:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.582307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464420305262:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.582317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464420305263:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.582322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464420305264:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.615069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241465060094998:2077] [node 4] ICP01 ready to work
2025-12-24T02:11:53.615210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241465060094999:2080] [node 5] ICP01 ready to work
2025-12-24T02:11:53.615301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241465060095001:2084] [node 6] ICP01 ready to work
2025-12-24T02:11:53.615068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464420305284:2061] [node 5] ICP01 ready to work
2025-12-24T02:11:53.615212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241464420305286:2067] [node 4] ICP01 ready to work
2025-12-24T02:11:53.615480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463990027791:2102] [node 6] ICP01 ready to work
2025-12-24T02:11:53.616391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463990027797:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:53.615319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463979910869:2079] [node 6] ICP01 ready to work
2025-12-24T02:11:53.616719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463979910880:2076] [node 5] ICP01 ready to work
2025-12-24T02:11:53.618860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241465031242797:2069] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.626854Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241465031242810:2071] [node 5] ICP01 ready to work
2025-12-24T02:11:53.627924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242811:2084] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.627950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242812:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.634794Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241465031242771:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(19806003ns) {Configure:104(19804535ns) {ProcessConfigured:135(16535737ns) {StartInitialHandshake:157(16534350ns) {DropHandshakes:470(589ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(274ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(1607ns) {SwitchToState:179(1122ns)}}}}}
2025-12-24T02:11:53.636126Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241465031242770:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20917393ns) {Configure:104(20916162ns) {ProcessConfigured:135(17779023ns) {StartInitialHandshake:157(17778483ns) {DropHandshakes:470(334ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(125ns)} GenerateSessionVirtualId:988(337ns) PrepareNewSessionHandshake:477(638ns) {SwitchToState:179(266ns)}}}}}
2025-12-24T02:11:53.643171Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241463979910880:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(26397223ns) {Configure:104(26396713ns) {ProcessConfigured:135(26139019ns) {StartInitialHandshake:157(26138321ns) {DropHandshakes:470(332ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(82ns) PrepareNewSessionHandshake:477(1688ns) {SwitchToState:179(1021ns)}}}}}
2025-12-24T02:11:53.647051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241465031242811:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.647109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241465031242771:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.651200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463990027808:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:53.651764Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463990027805:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-12-24T02:11:53.651835Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463990027806:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-12-24T02:11:53.651884Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463990027807:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-12-24T02:11:53.652694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465380509722:2073] [node 6] ICP01 ready to work
2025-12-24T02:11:53.656564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241463979910895:2086] [node 4] ICP01 ready to work
2025-12-24T02:11:53.652232Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241463990027731:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34568558ns) {Configure:104(34568068ns) {ProcessConfigured:135(34308817ns) {StartInitialHandshake:157(34308547ns) {DropHandshakes:470(512ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(31ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(617ns) {SwitchToState:179(72ns)}}}}}
2025-12-24T02:11:53.657525Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027805:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.658834Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095003:2087] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.658861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241465060095003:2087] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.655540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465380509687:2065] [node 4] ICP01 ready to work
2025-12-24T02:11:53.659169Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027806:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.663431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509731:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.663507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509732:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.663545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509733:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.666953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509731:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.669491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027807:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.668232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305343:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.668254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305340:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.668264Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305341:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.668313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305342:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.670942Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509732:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.670978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465380509656:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.670949Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464420305342:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.670989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464420305286:2067] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.675811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464420305341:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.675871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464420305264:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.677035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027793:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.677046Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241463990027793:2107] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.677646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241463990027793:2107] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.675469Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241463979910895:2086] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(18643761ns) {Configure:104(18643055ns) {ProcessConfigured:135(18223443ns) {StartInitialHandshake:157(18222832ns) {DropHandshakes:470(687ns) {DropIncomingHandshake:442(61ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(310ns) PrepareNewSessionHandshake:477(1708ns) {SwitchToState:179(845ns)}}}}}
2025-12-24T02:11:53.678792Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910873:2081] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.678797Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463979910873:2081] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.679155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241465060095038:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30843
2025-12-24T02:11:53.679176Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241465060095039:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30843
2025-12-24T02:11:53.679189Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241465060095040:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30843
2025-12-24T02:11:53.676922Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464420305340:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.676957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464420305262:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.677576Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305347:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.677819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464420305340:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.677857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464420305262:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.677895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.677902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464420305262:2048] [node 1] ICP22 created new session: [6:7587241464420305348:2048]
2025-12-24T02:11:53.677909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464420305348:2048] [node 1] ICS09 handshake done sender: [6:7587241464420305340:2115] self: [6:7587241464420305346:0] peer: [1:7587241463990027792:0] socket: 554 qp: -1
2025-12-24T02:11:53.677917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464420305348:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.677926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.678960Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464420305343:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.680210Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241465031242814:2086] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.680233Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241465031242814:2086] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.680401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464420305284:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.679329Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910915:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
2025-12-24T02:11:53.679392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910916:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
2025-12-24T02:11:53.679440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910917:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
2025-12-24T02:11:53.680791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910916:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.680811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910917:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.682865Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463990027816:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-12-24T02:11:53.682994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463990027791:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.683013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.683020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463990027791:2102] [node 6] ICP22 created new session: [1:7587241463990027817:2102]
2025-12-24T02:11:53.683027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463990027817:2102] [node 6] ICS09 handshake done sender: [1:7587241463990027793:2107] self: [1:7587241463990027792:0] peer: [6:7587241464420305346:0] socket: 531 qp: -1
2025-12-24T02:11:53.683033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463990027817:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:53.683063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.683050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509726:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.683059Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241465380509726:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.683246Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509733:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.683315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465380509687:2065] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.682818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095005:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.682832Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241465060095005:2088] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.682949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241465060095003:2087] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.683600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095039:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.683675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241465060094998:2077] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.683684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.683689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241465060094998:2077] [node 4] ICP22 created new session: [2:7587241465060095043:2077]
2025-12-24T02:11:53.683694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241465060095043:2077] [node 4] ICS09 handshake done sender: [2:7587241465060095003:2087] self: [2:7587241465060095002:0] peer: [4:7587241465031242846:0] socket: 538 qp: -1
2025-12-24T02:11:53.683699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241465060095043:2077] [node 4] ICS10 traffic start
2025-12-24T02:11:53.683707Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.683495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910915:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.686767Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241465031242812:2085] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.686801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241465031242770:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.695174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027799:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.695187Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241463990027799:2109] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.698733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241465031242817:2088] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.698750Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241465031242817:2088] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.699022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242852:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.699889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241465031242811:2084] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.699985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241465031242771:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.700007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.700015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241465031242771:2049] [node 2] ICP22 created new session: [4:7587241465031242853:2049]
2025-12-24T02:11:53.700023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241465031242853:2049] [node 2] ICS09 handshake done sender: [4:7587241465031242811:2084] self: [4:7587241465031242846:0] peer: [2:7587241465060095002:0] socket: 548 qp: -1
2025-12-24T02:11:53.700031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241465031242853:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.700040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.698916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463979910873:2081] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.700624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241465060095005:2088] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.700711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241465060094999:2080] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.700721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.700728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241465060094999:2080] [node 5] ICP22 created new session: [2:7587241465060095047:2080]
2025-12-24T02:11:53.700734Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241465060095047:2080] [node 5] ICS09 handshake done sender: [2:7587241465060095005:2088] self: [2:7587241465060095004:0] peer: [5:7587241465380509737:0] socket: 540 qp: -1
2025-12-24T02:11:53.700738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241465060095047:2080] [node 5] ICS10 traffic start
2025-12-24T02:11:53.700743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.706809Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465380509726:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.706985Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305352:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.706955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463979910869:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.706978Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.706987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463979910869:2079] [node 6] ICP22 created new session: [3:7587241463979910922:2079]
2025-12-24T02:11:53.706994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463979910922:2079] [node 6] ICS09 handshake done sender: [3:7587241463979910873:2081] self: [3:7587241463979910872:0] peer: [6:7587241464420305345:0] socket: 533 qp: -1
2025-12-24T02:11:53.707000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463979910922:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:53.707026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.707149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241465380509722:2073] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.707168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.707174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465380509722:2073] [node 6] ICP22 created new session: [5:7587241465380509740:2073]
2025-12-24T02:11:53.707180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241465380509740:2073] [node 6] ICS09 handshake done sender: [5:7587241465380509726:2112] self: [5:7587241465380509725:0] peer: [6:7587241464420305351:0] socket: 560 qp: -1
2025-12-24T02:11:53.707185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241465380509740:2073] [node 6] ICS10 traffic start
2025-12-24T02:11:53.707191Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.711761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241464420305284:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29908) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241464420305343:2118] held: no
2025-12-24T02:11:53.711939Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464420305341:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.710802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095040:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.713005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241465060094998:2077] [node 4] ICP35 event processing took too much time PendingConnection:279(29210500ns) {HandleHandshakeStatus:329(29209972ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(180ns) SwitchToState:179(35ns) ProcessPendingSessionEvents:495(15535547ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15513431ns max# 15430209ns}}}
2025-12-24T02:11:53.714583Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095038:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.712599Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241463990027819:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26740
2025-12-24T02:11:53.712653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027816:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.714129Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241463990027791:2102] [node 6] ICP35 event processing took too much time PendingConnection:279(31028247ns) {HandleHandshakeStatus:329(31027289ns) {DropIncomingHandshake:442(3974ns) DropOutgoingHandshake:457(280ns) SwitchToState:179(324ns) ProcessPendingSessionEvents:495(15232376ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15205555ns max# 15121995ns}}}
2025-12-24T02:11:53.714988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241463990027799:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.715064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463990027797:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.715081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.715086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463990027797:2108] [node 4] ICP22 created new session: [1:7587241463990027823:2108]
2025-12-24T02:11:53.715093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463990027823:2108] [node 4] ICS09 handshake done sender: [1:7587241463990027799:2109] self: [1:7587241463990027798:0] peer: [4:7587241465031242851:0] socket: 539 qp: -1
2025-12-24T02:11:53.715098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463990027823:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:53.715119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.714913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464420305264:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.714938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.714944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464420305264:2050] [node 3] ICP22 created new session: [6:7587241464420305353:2050]
2025-12-24T02:11:53.716054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241465031242817:2088] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.716316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241465031242810:2071] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.716327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.716333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241465031242810:2071] [node 5] ICP22 created new session: [4:7587241465031242859:2071]
2025-12-24T02:11:53.716337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241465031242859:2071] [node 5] ICS09 handshake done sender: [4:7587241465031242817:2088] self: [4:7587241465031242816:0] peer: [5:7587241465380509738:0] socket: 550 qp: -1
2025-12-24T02:11:53.716342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241465031242859:2071] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.716784Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027819:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.716816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027790:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.717378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509743:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.714950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464420305353:2050] [node 3] ICS09 handshake done sender: [6:7587241464420305341:2116] self: [6:7587241464420305345:0] peer: [3:7587241463979910872:0] socket: 564 qp: -1
2025-12-24T02:11:53.714957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464420305353:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.714964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.718848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465380509732:2115] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.718971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509745:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.719674Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910926:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
2025-12-24T02:11:53.720909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465380509656:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.720927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.720932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465380509656:2049] [node 2] ICP22 created new session: [5:7587241465380509746:2049]
2025-12-24T02:11:53.720938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241465380509746:2049] [node 2] ICS09 handshake done sender: [5:7587241465380509732:2115] self: [5:7587241465380509737:0] peer: [2:7587241465060095004:0] socket: 562 qp: -1
2025-12-24T02:11:53.720944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241465380509746:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.720951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.722494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910924:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.723803Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305356:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.724052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305357:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.724388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305358:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
2025-12-24T02:11:53.716346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.716562Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241465031242814:2086] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:53.716641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241465031242797:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.716648Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.716654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241465031242797:2069] [node 6] ICP22 created new session: [4:7587241465031242861:2069]
2025-12-24T02:11:53.716661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241465031242861:2069] [node 6] ICS09 handshake done sender: [4:7587241465031242814:2086] self: [4:7587241465031242813:0] peer: [6:7587241464420305344:0] socket: 552 qp: -1
2025-12-24T02:11:53.716665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241465031242861:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:53.716680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.721755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242858:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.724672Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241465031242770:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241465031242812:2085] held: no
2025-12-24T02:11:53.724770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241465031242812:2085] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.725080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241464420305356:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.725097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464420305343:2118] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.716819Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241463990027790:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.717345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241463990027797:2108] [node 4] ICP18 handshake (actor [1:7587241463990027819:2119]) is too old
No RDMA context found for GID: 2025-12-24T02:11:53.717699Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241463990027790:2106] [node 2] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:53.717849Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463990027730:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.717861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.717868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463990027730:2048] [node 2] ICP22 created new session: [1:7587241463990027827:2048]
2025-12-24T02:11:53.717874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463990027827:2048] [node 2] ICS09 handshake done sender: [1:7587241463990027790:2106] self: [1:7587241463990027789:0] peer: [2:7587241465060095051:0] socket: 529 qp: -1
2025-12-24T02:11:53.717879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463990027827:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:53.717886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.725354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242863:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.725515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464420305342:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.725735Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910926:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.725766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910924:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.725797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241463979910836:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.725808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241463979910837:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.725447Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509744:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.725666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465380509733:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.725589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464420305284:2061] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.725603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.725608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464420305284:2061] [node 5] ICP22 created new session: [6:7587241464420305359:2061]
2025-12-24T02:11:53.725613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464420305359:2061] [node 5] ICS09 handshake done sender: [6:7587241464420305343:2118] self: [6:7587241464420305351:0] peer: [5:7587241465380509725:0] socket: 566 qp: -1
2025-12-24T02:11:53.725619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464420305359:2061] [node 5] ICS10 traffic start
2025-12-24T02:11:53.725626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.725827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464420305286:2067] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.725835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.725839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464420305286:2067] [node 4] ICP22 created new session: [6:7587241464420305361:2067]
2025-12-24T02:11:53.725843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464420305361:2067] [node 4] ICS09 handshake done sender: [6:7587241464420305342:2117] self: [6:7587241464420305344:0] peer: [4:7587241465031242813:0] socket: 565 qp: -1
2025-12-24T02:11:53.725846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464420305361:2067] [node 4] ICS10 traffic start
2025-12-24T02:11:53.725850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.726145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241464420305263:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.723227Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241465060094959:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.723262Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241465060094959:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241465060095038:2115] held: no
2025-12-24T02:11:53.726355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095009:2090] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.726359Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241465060095009:2090] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.726790Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241465060095014:2091] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.726799Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241465060095014:2091] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.727339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241465031242770:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.727354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.727360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241465031242770:2048] [node 1] ICP22 created new session: [4:7587241465031242864:2048]
2025-12-24T02:11:53.727366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241465031242864:2048] [node 1] ICS09 handshake done sender: [4:7587241465031242812:2085] self: [4:7587241465031242851:0] peer: [1:7587241463990027798:0] socket: 549 qp: -1
2025-12-24T02:11:53.727371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241465031242864:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.727377Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.730079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241465060095009:2090] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.732123Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509748:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.732273Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509744:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.732286Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465380509687:2065] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.732297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.732303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465380509687:2065] [node 4] ICP22 created new session: [5:7587241465380509749:2065]
2025-12-24T02:11:53.732309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241465380509749:2065] [node 4] ICS09 handshake done sender: [5:7587241465380509733:2116] self: [5:7587241465380509738:0] peer: [4:7587241465031242816:0] socket: 563 qp: -1
2025-12-24T02:11:53.732314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241465380509749:2065] [node 4] ICS10 traffic start
2025-12-24T02:11:53.732320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.732594Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241465031242797:2069] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.731921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241465060095053:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30843
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.736837Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241463979910836:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29800) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241463979910926:2119] held: no
2025-12-24T02:11:53.736947Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910929:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
2025-12-24T02:11:53.737328Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463979910926:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.737411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241463979910836:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.737425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.737430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463979910836:2048] [node 1] ICP22 created new session: [3:7587241463979910930:2048]
2025-12-24T02:11:53.737437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463979910930:2048] [node 1] ICS09 handshake done sender: [3:7587241463979910926:2119] self: [3:7587241463979910927:0] peer: [1:7587241463990027801:0] socket: 587 qp: -1
2025-12-24T02:11:53.737442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463979910930:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.737448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.733955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027811:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.733976Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241463990027811:2117] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.734777Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241463990027811:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.734841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463990027731:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.734854Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.734859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463990027731:2049] [node 3] ICP22 created new session: [1:7587241463990027830:2049]
2025-12-24T02:11:53.734865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463990027830:2049] [node 3] ICS09 handshake done sender: [1:7587241463990027811:2117] self: [1:7587241463990027801:0] peer: [3:7587241463979910927:0] socket: 535 qp: -1
2025-12-24T02:11:53.734871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463990027830:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.734888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.739371Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241464420305364:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48477
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.740978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242866:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.741747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241464420305356:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.739056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465380509748:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.739090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465380509655:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.739138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465380509657:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.742175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241463990027810:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.742185Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241463990027810:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.742963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241463990027810:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.742978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241464420305263:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.742995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.743001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241464420305263:2049] [node 2] ICP22 created new session: [6:7587241464420305365:2049]
2025-12-24T02:11:53.743006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241464420305365:2049] [node 2] ICS09 handshake done sender: [6:7587241464420305356:2121] self: [6:7587241464420305363:0] peer: [2:7587241465060095008:0] socket: 584 qp: -1
2025-12-24T02:11:53.743012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241464420305365:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.743019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.743076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463990027808:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.743094Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.743101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463990027808:2112] [node 5] ICP22 created new session: [1:7587241463990027833:2112]
2025-12-24T02:11:53.743107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241463990027833:2112] [node 5] ICS09 handshake done sender: [1:7587241463990027810:2116] self: [1:7587241463990027809:0] peer: [5:7587241465380509752:0] socket: 589 qp: -1
2025-12-24T02:11:53.743113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241463990027833:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:53.743120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.743034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910910:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.743044Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463979910910:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:53.743524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241463979910837:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31608) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241463979910924:2118] held: no
2025-12-24T02:11:53.743751Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241463979910933:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44620
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.744421Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509754:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.739918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241465060095054:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30843
2025-12-24T02:11:53.740034Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241465060094999:2080] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.740041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241465060095001:2084] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.740054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:53.740060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241465060095001:2084] [node 6] ICP22 created new session: [2:7587241465060095056:2084]
2025-12-24T02:11:53.740066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241465060095056:2084] [node 6] ICS09 handshake done sender: [2:7587241465060095009:2090] self: [2:7587241465060095008:0] peer: [6:7587241464420305363:0] socket: 578 qp: -1
2025-12-24T02:11:53.740071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241465060095056:2084] [node 6] ICS10 traffic start
2025-12-24T02:11:53.740079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:53.740471Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241465060095014:2091] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.741194Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241465060095038:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.741240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241465060094960:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.741252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.741257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241465060094960:2049] [node 3] ICP22 created new session: [2:7587241465060095058:2049]
2025-12-24T02:11:53.741263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241465060095058:2049] [node 3] ICS09 handshake done sender: [2:7587241465060095014:2091] self: [2:7587241465060095011:0] peer: [3:7587241463979910928:0] socket: 580 qp: -1
2025-12-24T02:11:53.741268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241465060095058:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:53.741274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.741879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241465060094959:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.741887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.741893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241465060094959:2048] [node 1] ICP22 created new session: [2:7587241465060095060:2048]
2025-12-24T02:11:53.741897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241465060095060:2048] [node 1] ICS09 handshake done sender: [2:7587241465060095038:2115] self: [2:7587241465060095051:0] peer: [1:7587241463990027789:0] socket: 543 qp: -1
2025-12-24T02:11:53.741902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241465060095060:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.741906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:53.746776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465380509744:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:53.747728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465380509655:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.747745Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:53.747751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465380509655:2048] [node 1] ICP22 created new session: [5:7587241465380509756:2048]
2025-12-24T02:11:53.747756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241465380509756:2048] [node 1] ICS09 handshake done sender: [5:7587241465380509744:2119] self: [5:7587241465380509752:0] peer: [1:7587241463990027809:0] socket: 591 qp: -1
2025-12-24T02:11:53.747763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241465380509756:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:53.747770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.749477Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465380509755:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54388
2025-12-24T02:11:53.749613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465380509748:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.747410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463979910910:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:53.747598Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463979910924:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:53.748660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463979910880:2076] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.748673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:53.748678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463979910880:2076] [node 5] ICP22 created new session: [3:7587241463979910935:2076]
2025-12-24T02:11:53.748684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463979910935:2076] [node 5] ICS09 handshake done sender: [3:7587241463979910910:2111] self: [3:7587241463979910881:0] peer: [5:7587241465380509753:0] socket: 594 qp: -1
2025-12-24T02:11:53.748688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463979910935:2076] [node 5] ICS10 traffic start
2025-12-24T02:11:53.748694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:53.749137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241463979910837:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.749145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:53.749150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463979910837:2049] [node 2] ICP22 created new session: [3:7587241463979910937:2049]
2025-12-24T02:11:53.749155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463979910937:2049] [node 2] ICS09 handshake done sender: [3:7587241463979910924:2118] self: [3:7587241463979910928:0] peer: [2:7587241465060095011:0] socket: 583 qp: -1
2025-12-24T02:11:53.749159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463979910937:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:53.749165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:53.751461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465380509657:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.751489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.751495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465380509657:2050] [node 3] ICP22 created new session: [5:7587241465380509758:2050]
2025-12-24T02:11:53.751501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241465380509758:2050] [node 3] ICS09 handshake done sender: [5:7587241465380509748:2121] self: [5:7587241465380509753:0] peer: [3:7587241463979910881:0] socket: 571 qp: -1
2025-12-24T02:11:53.751507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241465380509758:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.751514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.753554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241463979910914:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.753562Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241463979910914:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:53.754353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241463979910914:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:53.752484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241465031242866:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:53.752528Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241465031242772:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:53.754280Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241465031242868:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23358
2025-12-24T02:11:53.754504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241465031242866:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:53.754556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241465031242772:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:53.754565Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:53.754575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241465031242772:2050] [node 3] ICP22 created new session: [4:7587241465031242869:2050]
2025-12-24T02:11:53.754580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241465031242869:2050] [node 3] ICS09 handshake done sender: [4:7587241465031242866:2120] self: [4:7587241465031242867:0] peer: [3:7587241463979910911:0] socket: 546 qp: -1
2025-12-24T02:11:53.754586Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241465031242869:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:53.754592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:53.754881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241463979910895:2086] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:53.754908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:53.754912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241463979910895:2086] [node 4] ICP22 created new session: [3:7587241463979910940:2086]
2025-12-24T02:11:53.754916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241463979910940:2086] [node 4] ICS09 handshake done sender: [3:7587241463979910914:2114] self: [3:7587241463979910911:0] peer: [4:7587241465031242867:0] socket: 536 qp: -1
2025-12-24T02:11:53.754920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241463979910940:2086] [node 4] ICS10 traffic start
2025-12-24T02:11:53.754926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:53.755507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241465380509657:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:53.779437Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:53.779543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241463979910930:2048] [node 1] ICS01 socket: 587 reason# EndOfStream
2025-12-24T02:11:53.779554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241463979910836:2048] [node 1] ICP30 unregister session Session# [3:7587241463979910930:2048] VirtualId# [3:7587241463979910927:0]
2025-12-24T02:11:53.779565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241463979910930:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.779590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.779610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.782145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:53.782214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241465031242864:2048] [node 1] ICS01 socket: 549 reason# EndOfStream
2025-12-24T02:11:53.782220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241465031242770:2048] [node 1] ICP30 unregister session Session# [4:7587241465031242864:2048] VirtualId# [4:7587241465031242851:0]
2025-12-24T02:11:53.782225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241465031242864:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.782250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.782268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.782417Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:53.782487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464420305348:2048] [node 1] ICS01 socket: 554 reason# EndOfStream
2025-12-24T02:11:53.782492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241464420305262:2048] [node 1] ICP30 unregister session Session# [6:7587241464420305348:2048] VirtualId# [6:7587241464420305346:0]
2025-12-24T02:11:53.782496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464420305348:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.782522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.782559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.790830Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:53.790947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241465380509756:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-12-24T02:11:53.790960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465380509655:2048] [node 1] ICP30 unregister session Session# [5:7587241465380509756:2048] VirtualId# [5:7587241465380509752:0]
2025-12-24T02:11:53.790968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241465380509756:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.791010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:53.791037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:53.799376Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:53.799472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241463979910937:2049] [node 2] ICS01 socket: 583 reason# EndOfStream
2025-12-24T02:11:53.799482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241463979910837:2049] [node 2] ICP30 unregister session Session# [3:7587241463979910937:2049] VirtualId# [3:7587241463979910928:0]
2025-12-24T02:11:53.799490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241463979910937:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.799522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.799540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.811018Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:53.811078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241464420305365:2049] [node 2] ICS07 socket disconnect 584 reason# EndOfStream
2025-12-24T02:11:53.811082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464420305365:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.811105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.811124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241464420305365:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:53.810594Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:53.810715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241465031242853:2049] [node 2] ICS01 socket: 548 reason# EndOfStream
2025-12-24T02:11:53.810725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241465031242771:2049] [node 2] ICP30 unregister session Session# [4:7587241465031242853:2049] VirtualId# [4:7587241465031242846:0]
2025-12-24T02:11:53.810732Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241465031242853:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.810784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.810824Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.810863Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:53.810933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241465380509746:2049] [node 2] ICS01 socket: 562 reason# EndOfStream
2025-12-24T02:11:53.810939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465380509656:2049] [node 2] ICP30 unregister session Session# [5:7587241465380509746:2049] VirtualId# [5:7587241465380509737:0]
2025-12-24T02:11:53.810943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241465380509746:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.810967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:53.810987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.815130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241464420305263:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31608) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:53.815151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464420305365:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:53.815158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241464420305263:2049] [node 2] ICP30 unregister session Session# [6:7587241464420305365:2049] VirtualId# [6:7587241464420305363:0]
2025-12-24T02:11:53.815213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:53.815220Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241464420305263:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.815223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31608) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:53.826800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241464420305263:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:53.838853Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:53.838967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464420305353:2050] [node 3] ICS01 socket: 564 reason# EndOfStream
2025-12-24T02:11:53.838978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241464420305264:2050] [node 3] ICP30 unregister session Session# [6:7587241464420305353:2050] VirtualId# [6:7587241464420305345:0]
2025-12-24T02:11:53.838985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464420305353:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.839040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.839067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.839169Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:53.839237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241465380509758:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-12-24T02:11:53.839243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465380509657:2050] [node 3] ICP30 unregister session Session# [5:7587241465380509758:2050] VirtualId# [5:7587241465380509753:0]
2025-12-24T02:11:53.839247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241465380509758:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.839276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:53.839299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:53.870889Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:53.870997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241464420305361:2067] [node 4] ICS01 socket: 565 reason# EndOfStream
2025-12-24T02:11:53.871010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241464420305286:2067] [node 4] ICP30 unregister session Session# [6:7587241464420305361:2067] VirtualId# [6:7587241464420305344:0]
2025-12-24T02:11:53.871019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241464420305361:2067] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:53.871070Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:53.871097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:53.938893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241461959710151:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.938908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241461959710152:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.942801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241464022724756:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.942811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241464022724757:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.931073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463579888635:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:53.931082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241463579888636:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:53.962886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464395435992:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.962897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464395435993:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.962902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241464395435994:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.962905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241462296198474:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.962916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241462296198475:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.962921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241462296198476:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:53.982820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241463116837568:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:53.982840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241463116837569:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:53.982845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241463116837570:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.034997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241467411804878:2059] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.035899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804879:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.035055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466591165827:2103] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.038923Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677507:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.042744Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467874855993:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64577
2025-12-24T02:11:54.043852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677507:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.047701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467874855998:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:54.048021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466254677510:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:54.048340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241466254677510:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.050936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241467411804879:2061] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.051071Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241467411804889:2066] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.051074Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241467411804889:2066] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.051603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241467411804878:2059] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.052534Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241467411804889:2066] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.052549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241467411804885:2064] [node 5] ICP01 ready to work
2025-12-24T02:11:54.052469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692112:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
2025-12-24T02:11:54.052618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241463116837569:2049] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.052636Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.052645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241463116837569:2049] [node 2] ICP22 created new session: [6:7587241467411804931:2049]
2025-12-24T02:11:54.052651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241467411804931:2049] [node 2] ICS09 handshake done sender: [6:7587241467411804889:2066] self: [6:7587241467411804888:0] peer: [2:7587241466254677511:0] socket: 541 qp: -1
2025-12-24T02:11:54.052659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241467411804931:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.052682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.058788Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241463579888636:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16426601ns) {Configure:104(16425761ns) {ProcessConfigured:135(16005038ns) {StartInitialHandshake:157(16004825ns) {DropHandshakes:470(413ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(355ns) PrepareNewSessionHandshake:477(754ns) {SwitchToState:179(574ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.060888Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467874856007:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64577
2025-12-24T02:11:54.061520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467874855997:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:54.061910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165829:2104] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.061928Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466591165829:2104] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.062868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241468690403329:2062] [node 4] ICP01 ready to work
2025-12-24T02:11:54.063341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467874855996:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:54.063720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241468690403331:2072] [node 6] ICP01 ready to work
2025-12-24T02:11:54.065483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692112:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.073461Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677516:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.073541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677517:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.073571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677515:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.074863Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467874856008:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64577
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.078293Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241467874855997:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16259849ns) {Configure:104(16259203ns) {ProcessConfigured:135(15966503ns) {StartInitialHandshake:157(15966037ns) {DropHandshakes:470(385ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(730ns) {SwitchToState:179(229ns)}}}}}
2025-12-24T02:11:54.078363Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874855993:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.077344Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241468317692119:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:54.077487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403364:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.078843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804948:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.078993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856007:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.078977Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241463116837569:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(26286101ns) {HandleHandshakeStatus:329(26285441ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(305ns) SwitchToState:179(85ns) ProcessPendingSessionEvents:495(15903077ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 15889545ns max# 15846645ns}}}
2025-12-24T02:11:54.080054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466591165829:2104] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.080392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241467411804879:2061] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.080339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466591165827:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.080361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.080369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466591165827:2103] [node 6] ICP22 created new session: [4:7587241466591165842:2103]
2025-12-24T02:11:54.080376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466591165842:2103] [node 6] ICS09 handshake done sender: [4:7587241466591165829:2104] self: [4:7587241466591165828:0] peer: [6:7587241467411804926:0] socket: 534 qp: -1
2025-12-24T02:11:54.080384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466591165842:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:54.080391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.080490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241467411804878:2059] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.080503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.080510Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241467411804878:2059] [node 4] ICP22 created new session: [6:7587241467411804949:2059]
2025-12-24T02:11:54.080515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241467411804949:2059] [node 4] ICS09 handshake done sender: [6:7587241467411804879:2061] self: [6:7587241467411804926:0] peer: [4:7587241466591165828:0] socket: 535 qp: -1
2025-12-24T02:11:54.080521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241467411804949:2059] [node 4] ICS10 traffic start
2025-12-24T02:11:54.080529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.079260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241468690403329:2062] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16253787ns) {Configure:104(16253306ns) {ProcessConfigured:135(15911670ns) {StartInitialHandshake:157(15911093ns) {DropHandshakes:470(408ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(237ns) PrepareNewSessionHandshake:477(754ns) {SwitchToState:179(433ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.083529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241468317692114:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:54.082270Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241468690403331:2072] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18419283ns) {Configure:104(18418789ns) {ProcessConfigured:135(18145109ns) {StartInitialHandshake:157(18144647ns) {DropHandshakes:470(300ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1077ns) {SwitchToState:179(399ns)}}}}}
2025-12-24T02:11:54.085521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165837:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.085673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241466591165827:2103] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:54.095013Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241462296198474:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(20924073ns) {Configure:104(20923183ns) {ProcessConfigured:135(20424385ns) {StartInitialHandshake:157(20423615ns) {DropHandshakes:470(367ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(224ns) PrepareNewSessionHandshake:477(1342ns) {SwitchToState:179(229ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.103596Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467874856016:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64577
2025-12-24T02:11:54.103634Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467874856015:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:64577
2025-12-24T02:11:54.097833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241468317692123:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:54.104003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403364:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.102794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466254677521:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:54.103030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677516:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.106848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692121:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.106713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466254677520:2113] [node 5] ICP01 ready to work
2025-12-24T02:11:54.106776Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466254677507:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.107823Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677522:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.108318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677523:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.108707Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403368:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.108736Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403369:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.108757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403370:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.110022Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856001:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.110033Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467874856001:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.110143Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856016:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.110191Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856015:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.110434Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403368:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.110730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692121:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.110860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677517:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.110882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241466254677510:2108] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.110894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.110903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466254677510:2108] [node 6] ICP22 created new session: [2:7587241466254677531:2108]
2025-12-24T02:11:54.110909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466254677531:2108] [node 6] ICS09 handshake done sender: [2:7587241466254677507:2107] self: [2:7587241466254677511:0] peer: [6:7587241467411804888:0] socket: 542 qp: -1
2025-12-24T02:11:54.110917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466254677531:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:54.110939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.114963Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677522:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.115002Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677523:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.117556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241461959710151:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.117739Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241461959710152:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(51111365ns) {Configure:104(51110566ns) {ProcessConfigured:135(50753681ns) {StartInitialHandshake:157(50753007ns) {DropHandshakes:470(433ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(375ns) PrepareNewSessionHandshake:477(688ns) {SwitchToState:179(106ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.118656Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466254677537:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38142
2025-12-24T02:11:54.118822Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241468317692119:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(41418983ns) {Configure:104(41418421ns) {ProcessConfigured:135(41074360ns) {StartInitialHandshake:157(41073941ns) {DropHandshakes:470(420ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(129ns) PrepareNewSessionHandshake:477(1016ns) {SwitchToState:179(528ns)}}}}}
2025-12-24T02:11:54.117974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856003:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.117984Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467874856003:2113] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.118761Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467874856003:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.118876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463579888635:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.118892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.118891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856008:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.118899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463579888635:2048] [node 2] ICP22 created new session: [1:7587241467874856020:2048]
2025-12-24T02:11:54.118906Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467874856020:2048] [node 2] ICS09 handshake done sender: [1:7587241467874856003:2113] self: [1:7587241467874856002:0] peer: [2:7587241466254677533:0] socket: 553 qp: -1
2025-12-24T02:11:54.118914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467874856020:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:54.118933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.119389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466254677517:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.119560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403370:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.119589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464395435993:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.119111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241463579888635:2048] [node 2] ICP18 handshake (actor [1:7587241467874856008:2115]) is too old
2025-12-24T02:11:54.118909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241464022724756:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.109548Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241462296198476:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23114107ns) {Configure:104(23112524ns) {ProcessConfigured:135(22733143ns) {StartInitialHandshake:157(22732681ns) {DropHandshakes:470(580ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(50ns)} GenerateSessionVirtualId:988(182ns) PrepareNewSessionHandshake:477(751ns) {SwitchToState:179(397ns)}}}}}
2025-12-24T02:11:54.109984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466591165846:2112] [node 5] ICP01 ready to work
2025-12-24T02:11:54.122834Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692133:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.123113Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692136:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.127145Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467874856001:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.127256Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692133:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.127281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692127:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.127286Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241468317692127:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.126879Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241466591165846:2112] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16730994ns) {Configure:104(16730432ns) {ProcessConfigured:135(16372719ns) {StartInitialHandshake:157(16371797ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(1444ns) {SwitchToState:179(1086ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.122756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464395435992:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.122789Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241464395435992:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(18729807ns) {IncomingHandshake:261(18729320ns) {DropIncomingHandshake:442(59ns) PrepareNewSessionHandshake:477(627ns) {SwitchToState:179(103ns)} GenerateSessionVirtualId:988(646ns) IssueIncomingHandshakeReply:193(18724069ns) {SwitchToState:179(1061ns)}}}
2025-12-24T02:11:54.126850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403369:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.126988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241464395435994:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.127047Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403373:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.127447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241468690403364:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.127514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464395435992:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.127532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.127539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464395435992:2048] [node 1] ICP22 created new session: [5:7587241468690403375:2048]
2025-12-24T02:11:54.127545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241468690403375:2048] [node 1] ICS09 handshake done sender: [5:7587241468690403364:2113] self: [5:7587241468690403367:0] peer: [1:7587241467874856000:0] socket: 559 qp: -1
2025-12-24T02:11:54.127551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241468690403375:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.127566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.128434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165853:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.128489Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165854:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.127231Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677527:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.127248Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466254677527:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.127678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241461959710151:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.127693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.127701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241461959710151:2048] [node 1] ICP22 created new session: [2:7587241466254677538:2048]
2025-12-24T02:11:54.127707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466254677538:2048] [node 1] ICS09 handshake done sender: [2:7587241466254677517:2112] self: [2:7587241466254677533:0] peer: [1:7587241467874856002:0] socket: 563 qp: -1
2025-12-24T02:11:54.127714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466254677538:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.127731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.128997Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241466254677510:2108] [node 6] ICP35 event processing took too much time PendingConnection:279(18049433ns) {HandleHandshakeStatus:329(18048405ns) {DropIncomingHandshake:442(339ns) DropOutgoingHandshake:457(176ns) SwitchToState:179(82ns) ProcessPendingSessionEvents:495(17929725ns) {unordered ForwardSessionEventToSession:612 num# 40 sum# 17920202ns max# 17885285ns}}}
2025-12-24T02:11:54.129012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241461959710151:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:54.130980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874855999:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.130991Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467874855999:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.130952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466254677527:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.133416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165853:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.133470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241462296198475:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.134839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466254677520:2113] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.134868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.134876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466254677520:2113] [node 5] ICP22 created new session: [2:7587241466254677541:2113]
2025-12-24T02:11:54.134882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466254677541:2113] [node 5] ICS09 handshake done sender: [2:7587241466254677527:2118] self: [2:7587241466254677526:0] peer: [5:7587241468690403372:0] socket: 578 qp: -1
2025-12-24T02:11:54.134890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466254677541:2113] [node 5] ICS10 traffic start
2025-12-24T02:11:54.134898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.135161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677525:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.135165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466254677525:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.134955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692136:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.138942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165856:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.138940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467874855998:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.138970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.138978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467874855998:2110] [node 5] ICP22 created new session: [1:7587241467874856024:2110]
2025-12-24T02:11:54.138985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467874856024:2110] [node 5] ICS09 handshake done sender: [1:7587241467874856001:2112] self: [1:7587241467874856000:0] peer: [5:7587241468690403367:0] socket: 552 qp: -1
2025-12-24T02:11:54.138992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467874856024:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:54.139000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.140134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466254677525:2117] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.139051Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692139:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
2025-12-24T02:11:54.140178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.140399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466254677521:2114] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.140421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.140427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466254677521:2114] [node 4] ICP22 created new session: [2:7587241466254677545:2114]
2025-12-24T02:11:54.140432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466254677545:2114] [node 4] ICS09 handshake done sender: [2:7587241466254677525:2117] self: [2:7587241466254677524:0] peer: [4:7587241466591165855:0] socket: 558 qp: -1
2025-12-24T02:11:54.140437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466254677545:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:54.140444Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.144227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241462296198475:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241466591165853:2116] held: no
2025-12-24T02:11:54.144584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466591165853:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.143476Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241468317692127:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.144686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241468317692123:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.144698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.144704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241468317692123:2112] [node 5] ICP22 created new session: [3:7587241468317692141:2112]
2025-12-24T02:11:54.144709Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241468317692141:2112] [node 5] ICS09 handshake done sender: [3:7587241468317692127:2114] self: [3:7587241468317692126:0] peer: [5:7587241468690403374:0] socket: 573 qp: -1
2025-12-24T02:11:54.144714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241468317692141:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:54.144720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.146005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692139:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.146036Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241464022724757:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.146134Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165854:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.146156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241462296198474:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.146296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241462296198475:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.146310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.146318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241462296198475:2049] [node 2] ICP22 created new session: [4:7587241466591165858:2049]
2025-12-24T02:11:54.146323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466591165858:2049] [node 2] ICS09 handshake done sender: [4:7587241466591165853:2116] self: [4:7587241466591165855:0] peer: [2:7587241466254677524:0] socket: 583 qp: -1
2025-12-24T02:11:54.146328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466591165858:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.146334Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.145228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.158445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692145:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
2025-12-24T02:11:54.158780Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467874855999:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.159100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241463579888636:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.159122Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.159129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241463579888636:2049] [node 3] ICP22 created new session: [1:7587241467874856028:2049]
2025-12-24T02:11:54.159135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467874856028:2049] [node 3] ICS09 handshake done sender: [1:7587241467874855999:2111] self: [1:7587241467874855994:0] peer: [3:7587241468317692135:0] socket: 550 qp: -1
2025-12-24T02:11:54.159141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467874856028:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:54.159148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.158998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241468317692121:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.159480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241464022724756:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.159492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.159498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241464022724756:2048] [node 1] ICP22 created new session: [3:7587241468317692146:2048]
2025-12-24T02:11:54.159503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241468317692146:2048] [node 1] ICS09 handshake done sender: [3:7587241468317692121:2111] self: [3:7587241468317692135:0] peer: [1:7587241467874855994:0] socket: 561 qp: -1
2025-12-24T02:11:54.159507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241468317692146:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.159514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.159160Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241464395435992:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(31512920ns) {HandleHandshakeStatus:329(31512377ns) {DropIncomingHandshake:442(302ns) DropOutgoingHandshake:457(3956ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(25178314ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 25152107ns max# 25067462ns}}}
2025-12-24T02:11:54.162887Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241464395435993:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241468690403370:2118] held: no
2025-12-24T02:11:54.163460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241464395435994:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241468690403369:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.166493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403379:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.166544Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403380:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.166597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466254677535:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.166626Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466254677535:2119] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.166769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856011:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.166784Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467874856011:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.167053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241467411804952:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.176215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241467411804953:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.176258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241463116837570:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.176383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692129:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.176394Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241468317692129:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.177003Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241468317692129:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.177068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241468317692114:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.177084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.177091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241468317692114:2108] [node 6] ICP22 created new session: [3:7587241468317692150:2108]
2025-12-24T02:11:54.177098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241468317692150:2108] [node 6] ICS09 handshake done sender: [3:7587241468317692129:2115] self: [3:7587241468317692128:0] peer: [6:7587241467411804955:0] socket: 577 qp: -1
2025-12-24T02:11:54.177105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241468317692150:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:54.177114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.184518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466254677535:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.184699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241461959710152:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.184721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.184728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241461959710152:2049] [node 3] ICP22 created new session: [2:7587241466254677548:2049]
2025-12-24T02:11:54.184735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466254677548:2049] [node 3] ICS09 handshake done sender: [2:7587241466254677535:2119] self: [2:7587241466254677514:0] peer: [3:7587241468317692144:0] socket: 591 qp: -1
2025-12-24T02:11:54.184744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466254677548:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:54.184752Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.183968Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165861:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.186821Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403381:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.186933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241468690403369:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.187029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241468690403370:2118] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.186911Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241464022724757:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65172) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241468317692139:2119] held: no
2025-12-24T02:11:54.184020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165862:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.190863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165862:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.190900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165861:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.190934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241462296198476:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.194935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165865:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.195035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467874856011:2116] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.195163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467874855997:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.195188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.195196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467874855997:2108] [node 4] ICP22 created new session: [1:7587241467874856031:2108]
2025-12-24T02:11:54.195203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467874856031:2108] [node 4] ICS09 handshake done sender: [1:7587241467874856011:2116] self: [1:7587241467874856009:0] peer: [4:7587241466591165857:0] socket: 586 qp: -1
2025-12-24T02:11:54.195210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467874856031:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:54.195221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.195197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241468317692134:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.195208Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241468317692134:2117] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.197568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241468317692134:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.195065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804956:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.196176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466591165854:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.196208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466591165866:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47941
2025-12-24T02:11:54.196275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241462296198474:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.196288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.196294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241462296198474:2048] [node 1] ICP22 created new session: [4:7587241466591165867:2048]
2025-12-24T02:11:54.196300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466591165867:2048] [node 1] ICS09 handshake done sender: [4:7587241466591165854:2117] self: [4:7587241466591165857:0] peer: [1:7587241467874856009:0] socket: 588 qp: -1
2025-12-24T02:11:54.196308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466591165867:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.196316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.197750Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466591165862:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.197803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241462296198476:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.197813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.197819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241462296198476:2050] [node 3] ICP22 created new session: [4:7587241466591165869:2050]
2025-12-24T02:11:54.197826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466591165869:2050] [node 3] ICS09 handshake done sender: [4:7587241466591165862:2120] self: [4:7587241466591165863:0] peer: [3:7587241468317692120:0] socket: 602 qp: -1
2025-12-24T02:11:54.197832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466591165869:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.197839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.198930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241468317692119:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.198948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.198954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241468317692119:2110] [node 4] ICP22 created new session: [3:7587241468317692154:2110]
2025-12-24T02:11:54.198959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241468317692154:2110] [node 4] ICS09 handshake done sender: [3:7587241468317692134:2117] self: [3:7587241468317692120:0] peer: [4:7587241466591165863:0] socket: 575 qp: -1
2025-12-24T02:11:54.198963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241468317692154:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:54.198985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.201515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241468317692152:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24867
2025-12-24T02:11:54.202783Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804957:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.203936Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466591165852:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.203945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466591165852:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.202902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464395435993:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.202920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.202929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464395435993:2049] [node 2] ICP22 created new session: [5:7587241468690403382:2049]
2025-12-24T02:11:54.202935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241468690403382:2049] [node 2] ICS09 handshake done sender: [5:7587241468690403370:2118] self: [5:7587241468690403372:0] peer: [2:7587241466254677526:0] socket: 580 qp: -1
2025-12-24T02:11:54.202941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241468690403382:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.202949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.203174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241464395435994:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.203181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.203186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241464395435994:2050] [node 3] ICP22 created new session: [5:7587241468690403384:2050]
2025-12-24T02:11:54.203192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241468690403384:2050] [node 3] ICS09 handshake done sender: [5:7587241468690403369:2117] self: [5:7587241468690403374:0] peer: [3:7587241468317692126:0] socket: 579 qp: -1
2025-12-24T02:11:54.203197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241468690403384:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.203202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.203405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403381:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.203612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241468690403329:2062] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.206966Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241468690403387:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54629
2025-12-24T02:11:54.207312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241468317692139:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.210303Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241464022724757:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.210325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.210334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241464022724757:2049] [node 2] ICP22 created new session: [3:7587241468317692156:2049]
2025-12-24T02:11:54.210341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241468317692156:2049] [node 2] ICS09 handshake done sender: [3:7587241468317692139:2119] self: [3:7587241468317692144:0] peer: [2:7587241466254677514:0] socket: 593 qp: -1
2025-12-24T02:11:54.210350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241468317692156:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.210359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.210905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466591165852:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.211037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467874856013:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.211053Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467874856013:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.210980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241463116837568:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.211294Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241464022724756:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(51697612ns) {HandleHandshakeStatus:329(51697258ns) {DropIncomingHandshake:442(117ns) DropOutgoingHandshake:457(2614ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(44326219ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 44304939ns max# 44223846ns}}}
2025-12-24T02:11:54.211030Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241463116837568:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43901869ns) {IncomingHandshake:261(43901295ns) {DropIncomingHandshake:442(182ns) PrepareNewSessionHandshake:477(827ns) {SwitchToState:179(264ns)} GenerateSessionVirtualId:988(383ns) IssueIncomingHandshakeReply:193(43893853ns) {SwitchToState:179(845ns)}}}
2025-12-24T02:11:54.211097Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241467411804956:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.211112Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241463116837570:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9057) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241467411804953:2118] held: no
2025-12-24T02:11:54.211158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241467411804885:2064] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.211777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466591165846:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.211797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.211804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466591165846:2112] [node 5] ICP22 created new session: [4:7587241466591165872:2112]
2025-12-24T02:11:54.211810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466591165872:2112] [node 5] ICS09 handshake done sender: [4:7587241466591165852:2115] self: [4:7587241466591165851:0] peer: [5:7587241468690403386:0] socket: 567 qp: -1
2025-12-24T02:11:54.211816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466591165872:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:54.211822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.213044Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804959:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.213318Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241463116837570:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(33355943ns) {HandleHandshakeStatus:395(33355392ns) {LogHandshakeFail:472(3416ns) {UpdateErrorStateLog:545(844ns)} DropOutgoingHandshake:457(532ns) ProcessPendingSessionEvents:495(85233ns) {unordered DropSessionEvent:512 num# 100 sum# 77811ns max# 3121ns}}}
2025-12-24T02:11:54.213353Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241467411804953:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.213426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241463116837570:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.213441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.213447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241463116837570:2050] [node 3] ICP22 created new session: [6:7587241467411804960:2050]
2025-12-24T02:11:54.213452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241467411804960:2050] [node 3] ICS09 handshake done sender: [6:7587241467411804953:2118] self: [6:7587241467411804955:0] peer: [3:7587241468317692128:0] socket: 596 qp: -1
2025-12-24T02:11:54.213457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241467411804960:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.213464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.211946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241468690403366:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.211956Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241468690403366:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.212693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241468690403381:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.212927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241468690403329:2062] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.212939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.212944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241468690403329:2062] [node 4] ICP22 created new session: [5:7587241468690403389:2062]
2025-12-24T02:11:54.212950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241468690403389:2062] [node 4] ICS09 handshake done sender: [5:7587241468690403381:2122] self: [5:7587241468690403386:0] peer: [4:7587241466591165851:0] socket: 600 qp: -1
2025-12-24T02:11:54.212955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241468690403389:2062] [node 4] ICS10 traffic start
2025-12-24T02:11:54.212961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.215352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241468690403366:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.215672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241468690403331:2072] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.215685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.215691Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241468690403331:2072] [node 6] ICP22 created new session: [5:7587241468690403391:2072]
2025-12-24T02:11:54.215696Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241468690403391:2072] [node 6] ICS09 handshake done sender: [5:7587241468690403366:2115] self: [5:7587241468690403363:0] peer: [6:7587241467411804958:0] socket: 598 qp: -1
2025-12-24T02:11:54.215701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241468690403391:2072] [node 6] ICS10 traffic start
2025-12-24T02:11:54.215706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.216590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241467411804956:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.216646Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241467411804962:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29453
2025-12-24T02:11:54.217471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241467411804885:2064] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26069) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241467411804956:2119] held: no
2025-12-24T02:11:54.217645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241467411804885:2064] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.217653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.217658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241467411804885:2064] [node 5] ICP22 created new session: [6:7587241467411804963:2064]
2025-12-24T02:11:54.217663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241467411804963:2064] [node 5] ICS09 handshake done sender: [6:7587241467411804956:2119] self: [6:7587241467411804958:0] peer: [5:7587241468690403363:0] socket: 601 qp: -1
2025-12-24T02:11:54.217668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241467411804963:2064] [node 5] ICS10 traffic start
2025-12-24T02:11:54.217674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.219013Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467874856013:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.219208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241467411804952:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.219551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467874855996:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.219575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.219581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467874855996:2107] [node 6] ICP22 created new session: [1:7587241467874856034:2107]
2025-12-24T02:11:54.219587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467874856034:2107] [node 6] ICS09 handshake done sender: [1:7587241467874856013:2117] self: [1:7587241467874856010:0] peer: [6:7587241467411804954:0] socket: 594 qp: -1
2025-12-24T02:11:54.219592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467874856034:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:54.219598Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.220154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241463116837568:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.220171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.220175Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241463116837568:2048] [node 1] ICP22 created new session: [6:7587241467411804965:2048]
2025-12-24T02:11:54.220181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241467411804965:2048] [node 1] ICS09 handshake done sender: [6:7587241467411804952:2117] self: [6:7587241467411804954:0] peer: [1:7587241467874856010:0] socket: 595 qp: -1
2025-12-24T02:11:54.220186Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241467411804965:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.220197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.243025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:54.243142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241466254677538:2048] [node 1] ICS01 socket: 563 reason# EndOfStream
2025-12-24T02:11:54.243154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241461959710151:2048] [node 1] ICP30 unregister session Session# [2:7587241466254677538:2048] VirtualId# [2:7587241466254677533:0]
2025-12-24T02:11:54.243163Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241466254677538:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.243206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.243230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.243565Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:54.243640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241468317692146:2048] [node 1] ICS07 socket disconnect 561 reason# EndOfStream
2025-12-24T02:11:54.243645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241468317692146:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.243669Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.243685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241468317692146:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:54.244479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241464022724756:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:54.244490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241468317692146:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:54.244495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241464022724756:2048] [node 1] ICP30 unregister session Session# [3:7587241468317692146:2048] VirtualId# [3:7587241468317692135:0]
2025-12-24T02:11:54.244544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.244551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241464022724756:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:54.244555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:54.249504Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:54.249596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241466591165867:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:54.249603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241462296198474:2048] [node 1] ICP30 unregister session Session# [4:7587241466591165867:2048] VirtualId# [4:7587241466591165857:0]
2025-12-24T02:11:54.249610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241466591165867:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.249658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.249682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.249716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:54.249784Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241468690403375:2048] [node 1] ICS07 socket disconnect 559 reason# EndOfStream
2025-12-24T02:11:54.249789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241468690403375:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.249814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.249832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241468690403375:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:54.250254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241464395435992:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:54.250262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241468690403375:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:54.250266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464395435992:2048] [node 1] ICP30 unregister session Session# [5:7587241468690403375:2048] VirtualId# [5:7587241468690403367:0]
2025-12-24T02:11:54.250315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.250321Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241464395435992:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:54.250323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16892) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:54.254529Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:54.254608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241467411804965:2048] [node 1] ICS01 socket: 595 reason# EndOfStream
2025-12-24T02:11:54.254615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241463116837568:2048] [node 1] ICP30 unregister session Session# [6:7587241467411804965:2048] VirtualId# [6:7587241467411804954:0]
2025-12-24T02:11:54.254621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241467411804965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.254664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.254684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.254970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241464022724756:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:54.258818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241464395435992:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:54.277153Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:54.277256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241468317692156:2049] [node 2] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:54.277267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241464022724757:2049] [node 2] ICP30 unregister session Session# [3:7587241468317692156:2049] VirtualId# [3:7587241468317692144:0]
2025-12-24T02:11:54.277274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241468317692156:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.277312Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.277331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.294084Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:54.294184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241468690403382:2049] [node 2] ICS01 socket: 580 reason# EndOfStream
2025-12-24T02:11:54.294194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464395435993:2049] [node 2] ICP30 unregister session Session# [5:7587241468690403382:2049] VirtualId# [5:7587241468690403372:0]
2025-12-24T02:11:54.294203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241468690403382:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.294256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.294279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.294746Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:54.294823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241466591165858:2049] [node 2] ICS01 socket: 583 reason# EndOfStream
2025-12-24T02:11:54.294829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241462296198475:2049] [node 2] ICP30 unregister session Session# [4:7587241466591165858:2049] VirtualId# [4:7587241466591165855:0]
2025-12-24T02:11:54.294835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241466591165858:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.294876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.294901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.302637Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:54.302752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241467411804931:2049] [node 2] ICS01 socket: 541 reason# EndOfStream
2025-12-24T02:11:54.302760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241463116837569:2049] [node 2] ICP30 unregister session Session# [6:7587241467411804931:2049] VirtualId# [6:7587241467411804888:0]
2025-12-24T02:11:54.302767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241467411804931:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.302820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.302859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.318426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:54.318536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241468690403384:2050] [node 3] ICS01 socket: 579 reason# EndOfStream
2025-12-24T02:11:54.318546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241464395435994:2050] [node 3] ICP30 unregister session Session# [5:7587241468690403384:2050] VirtualId# [5:7587241468690403374:0]
2025-12-24T02:11:54.318554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241468690403384:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.318603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:54.318626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:54.321313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:54.321413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241467411804960:2050] [node 3] ICS01 socket: 596 reason# EndOfStream
2025-12-24T02:11:54.321421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241463116837570:2050] [node 3] ICP30 unregister session Session# [6:7587241467411804960:2050] VirtualId# [6:7587241467411804955:0]
2025-12-24T02:11:54.321427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241467411804960:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.321476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:54.321500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:54.345809Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:54.345917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241467411804949:2059] [node 4] ICS01 socket: 535 reason# EndOfStream
2025-12-24T02:11:54.345929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241467411804878:2059] [node 4] ICP30 unregister session Session# [6:7587241467411804949:2059] VirtualId# [6:7587241467411804926:0]
2025-12-24T02:11:54.345937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241467411804949:2059] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.345990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:54.346017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:54.403037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467902500737:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:54.403049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467902500738:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:54.410882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466157752976:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.410891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466157752977:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.410896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466157752978:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.410941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241467598054869:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.410959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241467598054870:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:54.411119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241466022964894:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.411127Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241466022964895:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.422865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241466092829871:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.422874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241466092829872:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.422879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241466092829873:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.435342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241469342612433:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.435360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241469342612434:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.435365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241469342612435:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.475068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241469342612449:2060] [node 4] ICP01 ready to work
2025-12-24T02:11:54.475039Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241466092829906:2080] [node 6] ICP01 ready to work
2025-12-24T02:11:54.477075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241466092829912:2084] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.478158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054929:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.477331Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241469342612461:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:54.478639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829918:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.478383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467902500799:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:54.478556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467902500800:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:54.478897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241467902500802:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:54.479439Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241467598054931:2108] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.484138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829923:2093] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.486761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054933:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.502487Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612481:2085] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.506062Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829918:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.506244Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241469342612481:2085] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.506284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241469342612461:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.507067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466157753043:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.508930Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241467902500738:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(30012924ns) {Configure:104(30012241ns) {ProcessConfigured:135(21689925ns) {StartInitialHandshake:157(21689140ns) {DropHandshakes:470(439ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1157ns) {SwitchToState:179(842ns)}}}}}
2025-12-24T02:11:54.509296Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241467902500800:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(30682156ns) {Configure:104(30681757ns) {ProcessConfigured:135(21924268ns) {StartInitialHandshake:157(21923308ns) {DropHandshakes:470(239ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(15ns)} GenerateSessionVirtualId:988(193ns) PrepareNewSessionHandshake:477(526ns) {SwitchToState:179(206ns)}}}}}
2025-12-24T02:11:54.509133Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612513:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.507937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241466022964963:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:54.508014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964961:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.512792Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241466092829873:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22382628ns) {Configure:104(22382078ns) {ProcessConfigured:135(21949370ns) {StartInitialHandshake:157(21948910ns) {DropHandshakes:470(405ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(299ns) PrepareNewSessionHandshake:477(918ns) {SwitchToState:179(431ns)}}}}}
2025-12-24T02:11:54.511182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467902500816:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28662
2025-12-24T02:11:54.513372Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241467598054869:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(22484996ns) {Configure:104(22484554ns) {ProcessConfigured:135(22129815ns) {StartInitialHandshake:157(22129465ns) {DropHandshakes:470(452ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(1021ns) {SwitchToState:179(511ns)}}}}}
2025-12-24T02:11:54.515325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054933:2109] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.515391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241467598054869:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.515322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829923:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.515701Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241466092829871:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.516220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241467598054938:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:54.517445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241469342612513:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.517488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241469342612433:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.518760Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829908:2082] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.518781Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241466092829908:2082] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.522760Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964962:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.522850Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964961:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.522876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241466022964895:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.522982Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500807:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.522998Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467902500807:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.523960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467902500807:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.523752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612515:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.524193Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241469342612513:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.524299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241469342612433:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.524320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.524327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241469342612433:2048] [node 1] ICP22 created new session: [6:7587241469342612516:2048]
2025-12-24T02:11:54.524334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241469342612516:2048] [node 1] ICS09 handshake done sender: [6:7587241469342612513:2116] self: [6:7587241469342612514:0] peer: [1:7587241467902500806:0] socket: 556 qp: -1
2025-12-24T02:11:54.524342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241469342612516:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.524358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.526166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467902500802:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.526182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.526189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467902500802:2109] [node 6] ICP22 created new session: [1:7587241467902500818:2109]
2025-12-24T02:11:54.526196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467902500818:2109] [node 6] ICS09 handshake done sender: [1:7587241467902500807:2111] self: [1:7587241467902500806:0] peer: [6:7587241469342612514:0] socket: 553 qp: -1
2025-12-24T02:11:54.526203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467902500818:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:54.526222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.527356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753051:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.527439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753052:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.531267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964962:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.531136Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753052:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.531137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753051:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.534779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829950:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.534843Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829951:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.534724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500810:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.534744Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467902500810:2112] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.534817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500804:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.534823Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467902500804:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.534874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241466022964960:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:54.535222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241466022964967:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:54.535487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500816:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.534760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466157753050:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:54.535563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241467598054941:2112] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.539111Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241467598054931:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(59601260ns) {Configure:104(59600679ns) {ProcessConfigured:135(59179729ns) {StartInitialHandshake:157(59179355ns) {DropHandshakes:470(369ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(93ns) PrepareNewSessionHandshake:477(927ns) {SwitchToState:179(490ns)}}}}}
2025-12-24T02:11:54.539148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054929:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.540283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241467598054938:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(23844929ns) {Configure:104(23844680ns) {ProcessConfigured:135(23514749ns) {StartInitialHandshake:157(23514556ns) {DropHandshakes:470(235ns) {DropIncomingHandshake:442(26ns) DropOutgoingHandshake:457(32ns)} GenerateSessionVirtualId:988(567ns) PrepareNewSessionHandshake:477(624ns) {SwitchToState:179(268ns)}}}}}
2025-12-24T02:11:54.542724Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753055:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.540165Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467902500822:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28662
2025-12-24T02:11:54.540183Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241467902500823:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28662
2025-12-24T02:11:54.538912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964974:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.538974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964972:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.539008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964973:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.539949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241466022964894:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23957876ns) {Configure:104(23957281ns) {ProcessConfigured:135(23650528ns) {StartInitialHandshake:157(23650398ns) {DropHandshakes:470(503ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(758ns) {SwitchToState:179(246ns)}}}}}
2025-12-24T02:11:54.542875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964973:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.542777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829951:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.542822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241466092829873:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.542923Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964974:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.542951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964972:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.543281Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
2025-12-24T02:11:54.546822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054935:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.546835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241467598054935:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.546785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964965:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.546798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241466022964965:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.547014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054952:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.547157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500822:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.547850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241466022964894:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.548498Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241466092829872:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56594748ns) {Configure:104(56593649ns) {ProcessConfigured:135(56282836ns) {StartInitialHandshake:157(56282641ns) {DropHandshakes:470(439ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(918ns) {SwitchToState:179(25ns)}}}}}
2025-12-24T02:11:54.549203Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467902500810:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.549275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467902500737:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.549293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.549301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467902500737:2048] [node 2] ICP22 created new session: [1:7587241467902500826:2048]
2025-12-24T02:11:54.549308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467902500826:2048] [node 2] ICS09 handshake done sender: [1:7587241467902500810:2112] self: [1:7587241467902500809:0] peer: [2:7587241467598054943:0] socket: 554 qp: -1
2025-12-24T02:11:54.549315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467902500826:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:54.549332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.550923Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241467598054869:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241467598054933:2109] held: no
2025-12-24T02:11:54.551383Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753047:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.551395Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466157753047:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.552098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466157753047:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.552198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466157753043:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.552219Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.552225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466157753043:2109] [node 5] ICP22 created new session: [4:7587241466157753057:2109]
2025-12-24T02:11:54.552231Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466157753057:2109] [node 5] ICS09 handshake done sender: [4:7587241466157753047:2112] self: [4:7587241466157753046:0] peer: [5:7587241466092829954:0] socket: 567 qp: -1
2025-12-24T02:11:54.552237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466157753057:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:54.552245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.551162Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829950:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.551211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241466092829912:2084] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.553471Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241466092829912:2084] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16284) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241466092829950:2115] held: no
2025-12-24T02:11:54.556841Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467902500804:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.558724Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500814:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.558739Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467902500814:2113] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.560079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467902500799:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.560092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.560100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467902500799:2107] [node 5] ICP22 created new session: [1:7587241467902500830:2107]
2025-12-24T02:11:54.560107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467902500830:2107] [node 5] ICS09 handshake done sender: [1:7587241467902500804:2110] self: [1:7587241467902500803:0] peer: [5:7587241466092829949:0] socket: 551 qp: -1
2025-12-24T02:11:54.560115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467902500830:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:54.560127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.561301Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753061:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.562923Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241466022964965:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.562039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241466092829908:2082] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.562730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241466092829906:2080] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.562751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.562759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241466092829906:2080] [node 6] ICP22 created new session: [5:7587241466092829957:2080]
2025-12-24T02:11:54.562765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241466092829957:2080] [node 6] ICS09 handshake done sender: [5:7587241466092829908:2082] self: [5:7587241466092829907:0] peer: [6:7587241469342612512:0] socket: 540 qp: -1
2025-12-24T02:11:54.562772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241466092829957:2080] [node 6] ICS10 traffic start
2025-12-24T02:11:54.562781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.566859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241467598054935:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.567198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241467598054870:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.567217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.567223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241467598054870:2049] [node 3] ICP22 created new session: [2:7587241467598054955:2049]
2025-12-24T02:11:54.567230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241467598054955:2049] [node 3] ICS09 handshake done sender: [2:7587241467598054935:2110] self: [2:7587241467598054934:0] peer: [3:7587241466022964969:0] socket: 557 qp: -1
2025-12-24T02:11:54.567236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241467598054955:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:54.567252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.566840Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241466022964960:2108] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(31869767ns) {Configure:104(31868921ns) {ProcessConfigured:135(31513006ns) {StartInitialHandshake:157(31512540ns) {DropHandshakes:470(510ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(130ns)} GenerateSessionVirtualId:988(192ns) PrepareNewSessionHandshake:477(676ns) {SwitchToState:179(78ns)}}}}}
2025-12-24T02:11:54.566865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241466022964963:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.566881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.566888Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241466022964963:2109] [node 5] ICP22 created new session: [3:7587241466022964981:2109]
2025-12-24T02:11:54.566893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241466022964981:2109] [node 5] ICS09 handshake done sender: [3:7587241466022964965:2113] self: [3:7587241466022964964:0] peer: [5:7587241466092829952:0] socket: 569 qp: -1
2025-12-24T02:11:54.566900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241466022964981:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:54.566936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.567286Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241466022964967:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31931073ns) {Configure:104(31930771ns) {ProcessConfigured:135(31690080ns) {StartInitialHandshake:157(31669581ns) {DropHandshakes:470(153ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(760ns) PrepareNewSessionHandshake:477(1120ns) {SwitchToState:179(254ns)}} IncomingHandshake:261(13614ns) {DropIncomingHandshake:442(88ns) PrepareNewSessionHandshake:477(260ns) {SwitchToState:179(216ns)} GenerateSessionVirtualId:988(200ns) IssueIncomingHandshakeReply:193(225ns)}}}}
2025-12-24T02:11:54.567732Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500823:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.569951Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241467902500815:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.569961Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241467902500815:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.567859Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241466157753050:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(32956323ns) {Configure:104(32955868ns) {ProcessConfigured:135(32610952ns) {StartInitialHandshake:157(32600882ns) {DropHandshakes:470(723ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(229ns)} GenerateSessionVirtualId:988(387ns) PrepareNewSessionHandshake:477(653ns) {SwitchToState:179(49ns)}} IncomingHandshake:261(3983ns) {DropIncomingHandshake:442(448ns) PrepareNewSessionHandshake:477(202ns) {SwitchToState:179(154ns)} GenerateSessionVirtualId:988(313ns) IssueIncomingHandshakeReply:193(214ns)}}}}
2025-12-24T02:11:54.567896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753061:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.567953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466157752976:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.570802Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467902500815:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.570897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467902500800:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.570913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.570918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467902500800:2108] [node 4] ICP22 created new session: [1:7587241467902500834:2108]
2025-12-24T02:11:54.570924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467902500834:2108] [node 4] ICS09 handshake done sender: [1:7587241467902500815:2114] self: [1:7587241467902500813:0] peer: [4:7587241466157753064:0] socket: 575 qp: -1
2025-12-24T02:11:54.570929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467902500834:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:54.570936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.572646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241467902500814:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.572749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241467902500738:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.572768Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.572773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241467902500738:2049] [node 3] ICP22 created new session: [1:7587241467902500837:2049]
2025-12-24T02:11:54.572779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241467902500837:2049] [node 3] ICS09 handshake done sender: [1:7587241467902500814:2113] self: [1:7587241467902500812:0] peer: [3:7587241466022964977:0] socket: 580 qp: -1
2025-12-24T02:11:54.572785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241467902500837:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:54.572792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.573319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241466092829873:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62691) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241466092829951:2116] held: no
2025-12-24T02:11:54.574795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241466022964894:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241466022964974:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.575233Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054959:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:54.575236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
2025-12-24T02:11:54.575305Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241467598054961:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3410
2025-12-24T02:11:54.575512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054961:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.575526Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241467598054933:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.575631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241467598054869:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.575644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.575648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241467598054869:2048] [node 1] ICP22 created new session: [2:7587241467598054963:2048]
2025-12-24T02:11:54.575654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241467598054963:2048] [node 1] ICS09 handshake done sender: [2:7587241467598054933:2109] self: [2:7587241467598054943:0] peer: [1:7587241467902500809:0] socket: 555 qp: -1
2025-12-24T02:11:54.575659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241467598054963:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.575665Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.577380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241466022964895:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241466022964961:2111] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.573700Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241466157752976:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63087) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241466157753061:2117] held: no
2025-12-24T02:11:54.575325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753055:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.578481Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612519:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.578515Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612520:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.578579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612521:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.579282Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241469342612461:2066] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19362) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241469342612481:2085] held: no
2025-12-24T02:11:54.579426Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241469342612481:2085] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.578669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964988:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.579065Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241466022964961:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.580535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241469342612520:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.580594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241469342612435:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.580713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241469342612461:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.580725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.580732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241469342612461:2066] [node 5] ICP22 created new session: [6:7587241469342612523:2066]
2025-12-24T02:11:54.580739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241469342612523:2066] [node 5] ICS09 handshake done sender: [6:7587241469342612481:2085] self: [6:7587241469342612512:0] peer: [5:7587241466092829907:0] socket: 548 qp: -1
2025-12-24T02:11:54.580746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241469342612523:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:54.580769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.581376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241469342612521:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.581407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241469342612434:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.581511Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054948:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.581524Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241467598054948:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.582336Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612526:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.582308Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241467598054948:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.582453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241467598054941:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.582472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.582477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241467598054941:2112] [node 6] ICP22 created new session: [2:7587241467598054966:2112]
2025-12-24T02:11:54.582482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241467598054966:2112] [node 6] ICS09 handshake done sender: [2:7587241467598054948:2115] self: [2:7587241467598054947:0] peer: [6:7587241469342612525:0] socket: 600 qp: -1
2025-12-24T02:11:54.582491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241467598054966:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:54.582498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.582966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054960:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.582507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241469342612521:2120] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.587175Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753065:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.590741Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753066:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.594796Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466157753061:2117] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.594901Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829959:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.595382Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829960:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.598891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241466022964989:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58356
2025-12-24T02:11:54.599338Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241466092829906:2080] [node 6] ICP35 event processing took too much time PendingConnection:279(36501234ns) {HandleHandshakeStatus:329(36499979ns) {DropIncomingHandshake:442(5809ns) DropOutgoingHandshake:457(117ns) SwitchToState:179(103ns) ProcessPendingSessionEvents:495(25910117ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 25885691ns max# 25814275ns}}}
2025-12-24T02:11:54.599601Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241466092829923:2093] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.599661Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241466092829950:2115] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.602979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829961:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.603177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241466092829912:2084] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.603198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.603207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241466092829912:2084] [node 4] ICP22 created new session: [5:7587241466092829962:2084]
2025-12-24T02:11:54.603215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241466092829962:2084] [node 4] ICS09 handshake done sender: [5:7587241466092829950:2115] self: [5:7587241466092829954:0] peer: [4:7587241466157753046:0] socket: 568 qp: -1
2025-12-24T02:11:54.603224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241466092829962:2084] [node 4] ICS10 traffic start
2025-12-24T02:11:54.603234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.603682Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241466022964895:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.603709Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.603716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241466022964895:2049] [node 2] ICP22 created new session: [3:7587241466022964990:2049]
2025-12-24T02:11:54.603724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241466022964990:2049] [node 2] ICS09 handshake done sender: [3:7587241466022964961:2111] self: [3:7587241466022964969:0] peer: [2:7587241467598054934:0] socket: 559 qp: -1
2025-12-24T02:11:54.603737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241466022964990:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.603746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.603487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241466092829871:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.603499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.603504Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241466092829871:2048] [node 1] ICP22 created new session: [5:7587241466092829964:2048]
2025-12-24T02:11:54.603510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241466092829964:2048] [node 1] ICS09 handshake done sender: [5:7587241466092829923:2093] self: [5:7587241466092829949:0] peer: [1:7587241467902500803:0] socket: 552 qp: -1
2025-12-24T02:11:54.603516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241466092829964:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.603521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.603998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241466092829951:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.606308Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241469342612434:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241469342612521:2120] held: no
2025-12-24T02:11:54.606545Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241469342612434:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23533230ns) {HandleHandshakeStatus:395(23532740ns) {LogHandshakeFail:472(4080ns) {UpdateErrorStateLog:545(1382ns)} DropOutgoingHandshake:457(768ns) ProcessPendingSessionEvents:495(108745ns) {unordered DropSessionEvent:512 num# 100 sum# 101531ns max# 7521ns}}}
2025-12-24T02:11:54.603067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241467598054870:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35755117ns) {HandleHandshakeStatus:329(35754556ns) {DropIncomingHandshake:442(4099ns) DropOutgoingHandshake:457(130ns) SwitchToState:179(262ns) ProcessPendingSessionEvents:495(28023431ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27995631ns max# 27900383ns}}}
2025-12-24T02:11:54.606439Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241466157752977:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(51105670ns) {Configure:104(51104672ns) {ProcessConfigured:135(50735656ns) {StartInitialHandshake:157(50735439ns) {DropHandshakes:470(615ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(661ns) PrepareNewSessionHandshake:477(615ns) {SwitchToState:179(265ns)}}}}}
2025-12-24T02:11:54.606552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466157752977:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.604000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964980:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.604010Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241466022964980:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.604877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241466022964974:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:54.610399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466157752976:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.610423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.610434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466157752976:2048] [node 1] ICP22 created new session: [4:7587241466157753068:2048]
2025-12-24T02:11:54.610441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466157753068:2048] [node 1] ICS09 handshake done sender: [4:7587241466157753061:2117] self: [4:7587241466157753064:0] peer: [1:7587241467902500813:0] socket: 590 qp: -1
2025-12-24T02:11:54.610449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466157753068:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.610466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.610963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241466092829873:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.610976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.610981Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241466092829873:2050] [node 3] ICP22 created new session: [5:7587241466092829966:2050]
2025-12-24T02:11:54.610986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241466092829966:2050] [node 3] ICS09 handshake done sender: [5:7587241466092829951:2116] self: [5:7587241466092829952:0] peer: [3:7587241466022964964:0] socket: 576 qp: -1
2025-12-24T02:11:54.610990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241466092829966:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.610996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.612001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241466022964894:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.612015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:54.612020Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241466022964894:2048] [node 1] ICP22 created new session: [3:7587241466022964993:2048]
2025-12-24T02:11:54.612025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241466022964993:2048] [node 1] ICS09 handshake done sender: [3:7587241466022964974:2116] self: [3:7587241466022964977:0] peer: [1:7587241467902500812:0] socket: 583 qp: -1
2025-12-24T02:11:54.612029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241466022964993:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:54.612035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:54.612620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241466022964980:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.612705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241466022964960:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.612716Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.612722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241466022964960:2108] [node 6] ICP22 created new session: [3:7587241466022964995:2108]
2025-12-24T02:11:54.612727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241466022964995:2108] [node 6] ICS09 handshake done sender: [3:7587241466022964980:2118] self: [3:7587241466022964970:0] peer: [6:7587241469342612522:0] socket: 599 qp: -1
2025-12-24T02:11:54.612731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241466022964995:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:54.612736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:54.615073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241469342612434:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.615093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.615101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241469342612434:2049] [node 2] ICP22 created new session: [6:7587241469342612527:2049]
2025-12-24T02:11:54.615107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241469342612527:2049] [node 2] ICS09 handshake done sender: [6:7587241469342612521:2120] self: [6:7587241469342612525:0] peer: [2:7587241467598054947:0] socket: 586 qp: -1
2025-12-24T02:11:54.615116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241469342612527:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.615135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:54.615372Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241469342612435:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:62691) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241469342612520:2119] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.616942Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829968:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.614880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753066:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.614930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466157752978:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.618855Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054946:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.618874Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241467598054946:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.619293Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241466022964985:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.619305Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241466022964985:2119] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.619561Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241466092829968:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.619613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241466092829872:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.620137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241467598054951:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.620144Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241467598054951:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.620965Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241467598054946:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.621078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241467598054951:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.621098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241467598054931:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.621114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.621120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241467598054931:2108] [node 4] ICP22 created new session: [2:7587241467598054971:2108]
2025-12-24T02:11:54.621126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241467598054971:2108] [node 4] ICS09 handshake done sender: [2:7587241467598054946:2114] self: [2:7587241467598054932:0] peer: [4:7587241466157753067:0] socket: 584 qp: -1
2025-12-24T02:11:54.621131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241467598054971:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:54.621148Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.621669Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241466092829970:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41547
2025-12-24T02:11:54.621823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241467598054938:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.621838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.621843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241467598054938:2111] [node 5] ICP22 created new session: [2:7587241467598054973:2111]
2025-12-24T02:11:54.621849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241467598054973:2111] [node 5] ICS09 handshake done sender: [2:7587241467598054951:2116] self: [2:7587241467598054945:0] peer: [5:7587241466092829969:0] socket: 573 qp: -1
2025-12-24T02:11:54.621854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241467598054973:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:54.621861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.622326Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241466092829968:2121] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.622453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241466092829872:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.622468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.622472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241466092829872:2049] [node 2] ICP22 created new session: [5:7587241466092829971:2049]
2025-12-24T02:11:54.622476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241466092829971:2049] [node 2] ICS09 handshake done sender: [5:7587241466092829968:2121] self: [5:7587241466092829969:0] peer: [2:7587241467598054945:0] socket: 606 qp: -1
2025-12-24T02:11:54.622481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241466092829971:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.622486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.623411Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753072:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.624216Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241466157752977:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:21005) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241466157753055:2115] held: no
2025-12-24T02:11:54.624609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466157753055:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:54.624683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466157752977:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.624693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:54.624699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466157752977:2049] [node 2] ICP22 created new session: [4:7587241466157753073:2049]
2025-12-24T02:11:54.624705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466157753073:2049] [node 2] ICS09 handshake done sender: [4:7587241466157753055:2115] self: [4:7587241466157753067:0] peer: [2:7587241467598054932:0] socket: 587 qp: -1
2025-12-24T02:11:54.624711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466157753073:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:54.624727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.627509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612529:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.628362Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612530:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.629033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241469342612520:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.629157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241469342612435:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.629172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.629179Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241469342612435:2050] [node 3] ICP22 created new session: [6:7587241469342612531:2050]
2025-12-24T02:11:54.629185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241469342612531:2050] [node 3] ICS09 handshake done sender: [6:7587241469342612520:2119] self: [6:7587241469342612522:0] peer: [3:7587241466022964970:0] socket: 572 qp: -1
2025-12-24T02:11:54.629191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241469342612531:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.629200Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.630849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241466022964985:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.630913Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466157753075:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:39999
2025-12-24T02:11:54.631160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466157753066:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:54.631106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241466022964967:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.631126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.631138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241466022964967:2110] [node 4] ICP22 created new session: [3:7587241466022964998:2110]
2025-12-24T02:11:54.631144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241466022964998:2110] [node 4] ICS09 handshake done sender: [3:7587241466022964985:2119] self: [3:7587241466022964971:0] peer: [4:7587241466157753070:0] socket: 561 qp: -1
2025-12-24T02:11:54.631149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241466022964998:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:54.631161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.631274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466157752978:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.631292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:54.631297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466157752978:2050] [node 3] ICP22 created new session: [4:7587241466157753076:2050]
2025-12-24T02:11:54.631303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466157753076:2050] [node 3] ICS09 handshake done sender: [4:7587241466157753066:2120] self: [4:7587241466157753070:0] peer: [3:7587241466022964971:0] socket: 544 qp: -1
2025-12-24T02:11:54.631308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466157753076:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:54.631316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:54.634927Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241469342612530:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.635019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241469342612449:2060] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.638901Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466157753062:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.638912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466157753062:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.639912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466157753062:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:54.639997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466157753050:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.640012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:54.640019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466157753050:2110] [node 6] ICP22 created new session: [4:7587241466157753079:2110]
2025-12-24T02:11:54.640024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241466157753079:2110] [node 6] ICS09 handshake done sender: [4:7587241466157753062:2118] self: [4:7587241466157753054:0] peer: [6:7587241469342612533:0] socket: 566 qp: -1
2025-12-24T02:11:54.640029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241466157753079:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:54.640037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.643784Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241469342612449:2060] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16284) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241469342612530:2123] held: no
2025-12-24T02:11:54.643972Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241469342612534:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58452
2025-12-24T02:11:54.644292Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241469342612530:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.644407Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241469342612449:2060] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:54.644421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.644428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241469342612449:2060] [node 4] ICP22 created new session: [6:7587241469342612535:2060]
2025-12-24T02:11:54.644433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241469342612535:2060] [node 4] ICS09 handshake done sender: [6:7587241469342612530:2123] self: [6:7587241469342612533:0] peer: [4:7587241466157753054:0] socket: 541 qp: -1
2025-12-24T02:11:54.644441Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241469342612535:2060] [node 4] ICS10 traffic start
2025-12-24T02:11:54.644466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.660992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:54.661089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241466022964993:2048] [node 1] ICS01 socket: 583 reason# EndOfStream
2025-12-24T02:11:54.661101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241466022964894:2048] [node 1] ICP30 unregister session Session# [3:7587241466022964993:2048] VirtualId# [3:7587241466022964977:0]
2025-12-24T02:11:54.661109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241466022964993:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.661143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.661163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.662803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:54.662896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241467598054963:2048] [node 1] ICS01 socket: 555 reason# EndOfStream
2025-12-24T02:11:54.662905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241467598054869:2048] [node 1] ICP30 unregister session Session# [2:7587241467598054963:2048] VirtualId# [2:7587241467598054943:0]
2025-12-24T02:11:54.662912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241467598054963:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.662965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.662989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.666757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:54.666851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241466092829964:2048] [node 1] ICS01 socket: 552 reason# EndOfStream
2025-12-24T02:11:54.666862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241466092829871:2048] [node 1] ICP30 unregister session Session# [5:7587241466092829964:2048] VirtualId# [5:7587241466092829949:0]
2025-12-24T02:11:54.666869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241466092829964:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.666922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.666949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.670740Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:54.670825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241466157753068:2048] [node 1] ICS01 socket: 590 reason# EndOfStream
2025-12-24T02:11:54.670833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241466157752976:2048] [node 1] ICP30 unregister session Session# [4:7587241466157753068:2048] VirtualId# [4:7587241466157753064:0]
2025-12-24T02:11:54.670838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241466157753068:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.670879Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.670905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.674772Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:54.674868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241469342612516:2048] [node 1] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:54.674878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241469342612433:2048] [node 1] ICP30 unregister session Session# [6:7587241469342612516:2048] VirtualId# [6:7587241469342612514:0]
2025-12-24T02:11:54.674885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241469342612516:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.678426Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:54.678504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241466022964990:2049] [node 2] ICS01 socket: 559 reason# EndOfStream
2025-12-24T02:11:54.678512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241466022964895:2049] [node 2] ICP30 unregister session Session# [3:7587241466022964990:2049] VirtualId# [3:7587241466022964969:0]
2025-12-24T02:11:54.678519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241466022964990:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.678558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.678581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.674929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:54.674962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:54.694804Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:54.694918Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241466157753073:2049] [node 2] ICS01 socket: 587 reason# EndOfStream
2025-12-24T02:11:54.694931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241466157752977:2049] [node 2] ICP30 unregister session Session# [4:7587241466157753073:2049] VirtualId# [4:7587241466157753067:0]
2025-12-24T02:11:54.694941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241466157753073:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.694999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.695025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.697317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:54.697395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241466092829971:2049] [node 2] ICS01 socket: 606 reason# EndOfStream
2025-12-24T02:11:54.697401Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241466092829872:2049] [node 2] ICP30 unregister session Session# [5:7587241466092829971:2049] VirtualId# [5:7587241466092829969:0]
2025-12-24T02:11:54.697406Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241466092829971:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.697447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.697468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.706911Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:54.707033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241469342612527:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-12-24T02:11:54.707046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241469342612434:2049] [node 2] ICP30 unregister session Session# [6:7587241469342612527:2049] VirtualId# [6:7587241469342612525:0]
2025-12-24T02:11:54.707053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241469342612527:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.707117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:54.707145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:54.722949Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:54.723046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241466092829966:2050] [node 3] ICS01 socket: 576 reason# EndOfStream
2025-12-24T02:11:54.723057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241466092829873:2050] [node 3] ICP30 unregister session Session# [5:7587241466092829966:2050] VirtualId# [5:7587241466092829952:0]
2025-12-24T02:11:54.723066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241466092829966:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.723136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:54.723163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:54.722919Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:54.723046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241469342612531:2050] [node 3] ICS01 socket: 572 reason# EndOfStream
2025-12-24T02:11:54.723057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241469342612435:2050] [node 3] ICP30 unregister session Session# [6:7587241469342612531:2050] VirtualId# [6:7587241469342612522:0]
2025-12-24T02:11:54.723067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241469342612531:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.723136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:54.723163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:54.750767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:54.750897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241469342612535:2060] [node 4] ICS01 socket: 541 reason# EndOfStream
2025-12-24T02:11:54.750907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241469342612449:2060] [node 4] ICP30 unregister session Session# [6:7587241469342612535:2060] VirtualId# [6:7587241469342612533:0]
2025-12-24T02:11:54.750917Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241469342612535:2060] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:54.750987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:54.751015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:54.838067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466188972605:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.838098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466188972606:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:54.878901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241469106237639:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:54.878926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241469106237640:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:54.886774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241467367272938:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.886791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241467367272939:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.890604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466161488239:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.890631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466161488240:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.890211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241466611561192:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.890638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466161488241:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.890245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241466611561193:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.890251Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241466611561194:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.898715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465915290681:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:54.898747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465915290682:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:54.898753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465915290683:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:54.918168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241467367272956:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:54.919122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241466611561207:2059] [node 5] ICP01 ready to work
2025-12-24T02:11:54.918913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466161488265:2066] [node 6] ICP01 ready to work
2025-12-24T02:11:54.919028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241466161488269:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:54.921451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241467367272967:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:54.921585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241467367272968:2071] [node 6] ICP01 ready to work
2025-12-24T02:11:54.918961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466188972642:2083] [node 4] ICP01 ready to work
2025-12-24T02:11:54.919675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466188972650:2085] [node 5] ICP01 ready to work
2025-12-24T02:11:54.919771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241466188972651:2087] [node 6] ICP01 ready to work
2025-12-24T02:11:54.922907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241469106237702:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:54.923006Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241469106237703:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.927348Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466161488290:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.941860Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241466611561194:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(22203173ns) {Configure:104(22202675ns) {ProcessConfigured:135(21978133ns) {StartInitialHandshake:157(21977536ns) {DropHandshakes:470(275ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(14ns)} GenerateSessionVirtualId:988(206ns) PrepareNewSessionHandshake:477(1559ns) {SwitchToState:179(490ns)}}}}}
2025-12-24T02:11:54.942357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241466611561260:2068] [node 4] ICP01 ready to work
2025-12-24T02:11:54.943440Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241466611561258:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:54.943536Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241466611561259:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:54.950847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241469106237640:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(27754474ns) {Configure:104(27753911ns) {ProcessConfigured:135(27335872ns) {StartInitialHandshake:157(27335303ns) {DropHandshakes:470(457ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(1902ns) {SwitchToState:179(1252ns)}}}}}
2025-12-24T02:11:54.951152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241469106237706:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:54.953115Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241469106237713:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41844
2025-12-24T02:11:54.953132Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241469106237714:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41844
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.967038Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241467367272967:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(45371933ns) {Configure:104(45371233ns) {ProcessConfigured:135(44987075ns) {StartInitialHandshake:157(44986277ns) {DropHandshakes:470(443ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(610ns) PrepareNewSessionHandshake:477(2887ns) {SwitchToState:179(965ns)}}}}}
2025-12-24T02:11:54.968767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241467367273015:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:54.968781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241467367273016:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:54.969668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466161488318:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:54.970802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241465915290681:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(48007577ns) {Configure:104(48007039ns) {ProcessConfigured:135(47263940ns) {StartInitialHandshake:157(47263621ns) {DropHandshakes:470(440ns) {DropIncomingHandshake:442(63ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(351ns) PrepareNewSessionHandshake:477(1250ns) {SwitchToState:179(888ns)}}}}}
2025-12-24T02:11:54.971546Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465915290748:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:54.978772Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241466188972605:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(25772869ns) {Configure:104(25771583ns) {ProcessConfigured:135(25274170ns) {StartInitialHandshake:157(25273565ns) {DropHandshakes:470(628ns) {DropIncomingHandshake:442(86ns) DropOutgoingHandshake:457(105ns)} GenerateSessionVirtualId:988(235ns) PrepareNewSessionHandshake:477(1659ns) {SwitchToState:179(230ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.982911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241465915290753:2069] [node 6] ICP01 ready to work
2025-12-24T02:11:54.983842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290749:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:54.983915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290750:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:54.983965Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290751:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:54.983976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290752:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.987388Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466161488319:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:54.987420Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466161488320:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:54.990018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488290:2088] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.990064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466161488240:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.987530Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465915290750:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.987559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465915290749:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.989032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465915290682:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:54.989054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465915290748:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.994207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241469106237720:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41844
2025-12-24T02:11:54.987186Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466188972684:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51775
2025-12-24T02:11:54.987232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241466188972685:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51775
2025-12-24T02:11:54.990881Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972649:2089] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.990888Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466188972649:2089] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.991584Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466188972649:2089] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:54.991683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466188972642:2083] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.991698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:54.991710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466188972642:2083] [node 4] ICP22 created new session: [2:7587241466188972687:2083]
2025-12-24T02:11:54.991757Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466188972687:2083] [node 4] ICS09 handshake done sender: [2:7587241466188972649:2089] self: [2:7587241466188972648:0] peer: [4:7587241466161488321:0] socket: 546 qp: -1
2025-12-24T02:11:54.991778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466188972687:2083] [node 4] ICS10 traffic start
2025-12-24T02:11:54.991790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:54.994842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241466611561258:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.994895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241466611561259:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.996220Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241467367273019:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.996298Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241467367273020:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:54.995226Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237713:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.997337Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237720:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.998238Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237708:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.998244Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241469106237708:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.996557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972653:2090] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.996569Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466188972653:2090] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.997508Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466188972653:2090] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:54.997621Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972684:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.997631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466188972650:2085] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:54.997645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:54.997654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466188972650:2085] [node 5] ICP22 created new session: [2:7587241466188972691:2085]
2025-12-24T02:11:54.997660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241466188972691:2085] [node 5] ICS09 handshake done sender: [2:7587241466188972653:2090] self: [2:7587241466188972652:0] peer: [5:7587241465915290763:0] socket: 550 qp: -1
2025-12-24T02:11:54.997668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241466188972691:2085] [node 5] ICS10 traffic start
2025-12-24T02:11:54.997677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:54.997997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241466188972605:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:54.999232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273019:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.999341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273016:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.999419Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241467367272938:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.002486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465915290752:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.002557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290766:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:55.002580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465915290683:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.002595Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241465915290765:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:54.998998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488274:2078] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:54.999016Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466161488274:2078] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:54.999581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241466161488322:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:54.998950Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241469106237723:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41844
2025-12-24T02:11:54.999107Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241469106237708:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.002849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465915290750:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.005702Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465915290682:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.005723Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.005730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465915290682:2049] [node 2] ICP22 created new session: [5:7587241470210258064:2049]
2025-12-24T02:11:55.005737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241470210258064:2049] [node 2] ICS09 handshake done sender: [5:7587241465915290750:2113] self: [5:7587241465915290763:0] peer: [2:7587241466188972652:0] socket: 575 qp: -1
2025-12-24T02:11:55.005745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241470210258064:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.005753Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.005388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241466611561260:2068] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.005413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241466611561193:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.007574Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528570:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.006875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273007:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.006887Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241467367273007:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.010096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972685:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.010148Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972655:2091] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.010150Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466188972655:2091] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.007621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241469106237639:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.007642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.007650Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241469106237639:2048] [node 2] ICP22 created new session: [1:7587241473401205021:2048]
2025-12-24T02:11:55.007658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473401205021:2048] [node 2] ICS09 handshake done sender: [1:7587241469106237708:2111] self: [1:7587241469106237707:0] peer: [2:7587241466188972694:0] socket: 563 qp: -1
2025-12-24T02:11:55.007664Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473401205021:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:55.007673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.009089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488320:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.007941Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237705:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.007945Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241469106237705:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.009139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488272:2077] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.009142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241466161488272:2077] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.009757Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466161488290:2088] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.010869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241467367273007:2111] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.012186Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273015:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.012279Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273020:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.012289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241467367272967:2066] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.012302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.012309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241467367272967:2066] [node 5] ICP22 created new session: [3:7587241471662240321:2066]
2025-12-24T02:11:55.012314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241471662240321:2066] [node 5] ICS09 handshake done sender: [3:7587241467367273007:2111] self: [3:7587241467367272970:0] peer: [5:7587241465915290767:0] socket: 571 qp: -1
2025-12-24T02:11:55.012319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241471662240321:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:55.012327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.012962Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241471662240320:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:55.012994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241467367272939:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.013364Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241469106237705:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.011088Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488319:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.011119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241466161488318:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.011128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466161488240:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.011141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.011148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466161488240:2049] [node 2] ICP22 created new session: [4:7587241470456455621:2049]
2025-12-24T02:11:55.011154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470456455621:2049] [node 2] ICS09 handshake done sender: [4:7587241466161488290:2088] self: [4:7587241466161488321:0] peer: [2:7587241466188972648:0] socket: 552 qp: -1
2025-12-24T02:11:55.011161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470456455621:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.011167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.011830Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466161488274:2078] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.012771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470483939991:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51775
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.018745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528571:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.018784Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528572:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.018801Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528573:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.019664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466161488269:2070] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.019685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.019695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466161488269:2070] [node 5] ICP22 created new session: [4:7587241470456455624:2070]
2025-12-24T02:11:55.019701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470456455624:2070] [node 5] ICS09 handshake done sender: [4:7587241466161488274:2078] self: [4:7587241466161488273:0] peer: [5:7587241465915290764:0] socket: 548 qp: -1
2025-12-24T02:11:55.019707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470456455624:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:55.019715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.019922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466161488241:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.019995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241466161488239:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.020529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466161488272:2077] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.020687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241466161488265:2066] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.020698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.020703Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466161488265:2066] [node 6] ICP22 created new session: [4:7587241470456455629:2066]
2025-12-24T02:11:55.020708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470456455629:2066] [node 6] ICS09 handshake done sender: [4:7587241466161488272:2077] self: [4:7587241466161488271:0] peer: [6:7587241470906528568:0] socket: 547 qp: -1
2025-12-24T02:11:55.020713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470456455629:2066] [node 6] ICS10 traffic start
2025-12-24T02:11:55.020720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.026373Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241466611561258:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.027181Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466188972655:2091] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.030089Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367272966:2073] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.030107Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241467367272966:2073] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.030917Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241467367273016:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.032471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241467367272938:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.032489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.032496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241467367272938:2048] [node 1] ICP22 created new session: [3:7587241471662240328:2048]
2025-12-24T02:11:55.032502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241471662240328:2048] [node 1] ICS09 handshake done sender: [3:7587241467367273016:2116] self: [3:7587241467367273022:0] peer: [1:7587241469106237704:0] socket: 569 qp: -1
2025-12-24T02:11:55.032509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241471662240328:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.032516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.033675Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241467367272966:2073] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.033786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241467367272956:2064] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.033797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.033803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241467367272956:2064] [node 4] ICP22 created new session: [3:7587241471662240330:2064]
2025-12-24T02:11:55.033809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241471662240330:2064] [node 4] ICS09 handshake done sender: [3:7587241467367272966:2073] self: [3:7587241467367272960:0] peer: [4:7587241470456455626:0] socket: 570 qp: -1
2025-12-24T02:11:55.033814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241471662240330:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:55.033822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.034088Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241471662240326:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:55.030992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241466188972658:2092] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.031002Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241466188972658:2092] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.031699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466188972651:2087] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.031713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.031721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466188972651:2087] [node 6] ICP22 created new session: [2:7587241470483939994:2087]
2025-12-24T02:11:55.031728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470483939994:2087] [node 6] ICS09 handshake done sender: [2:7587241466188972655:2091] self: [2:7587241466188972654:0] peer: [6:7587241470906528569:0] socket: 551 qp: -1
2025-12-24T02:11:55.031737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470483939994:2087] [node 6] ICS10 traffic start
2025-12-24T02:11:55.031751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.032053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466188972684:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.035008Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470456455631:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:55.034768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241466611561260:2068] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.034791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.034797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241466611561260:2068] [node 4] ICP22 created new session: [6:7587241470906528574:2068]
2025-12-24T02:11:55.034803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470906528574:2068] [node 4] ICS09 handshake done sender: [6:7587241466611561258:2110] self: [6:7587241470906528568:0] peer: [4:7587241466161488271:0] socket: 555 qp: -1
2025-12-24T02:11:55.034809Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470906528574:2068] [node 4] ICS10 traffic start
2025-12-24T02:11:55.034816Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.035394Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241465915290683:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241465915290752:2115] held: no
2025-12-24T02:11:55.035512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465915290749:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.035637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241469106237640:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.035657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.035664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241469106237640:2049] [node 3] ICP22 created new session: [1:7587241473401205026:2049]
2025-12-24T02:11:55.035670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473401205026:2049] [node 3] ICS09 handshake done sender: [1:7587241469106237705:2110] self: [1:7587241469106237704:0] peer: [3:7587241467367273022:0] socket: 562 qp: -1
2025-12-24T02:11:55.035676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473401205026:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.035684Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.036204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470906528570:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.036372Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241466611561259:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.039670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241466611561193:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.039690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.039698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241466611561193:2049] [node 2] ICP22 created new session: [6:7587241470906528578:2049]
2025-12-24T02:11:55.039705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470906528578:2049] [node 2] ICS09 handshake done sender: [6:7587241466611561259:2111] self: [6:7587241470906528569:0] peer: [2:7587241466188972654:0] socket: 556 qp: -1
2025-12-24T02:11:55.039711Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470906528578:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.039718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.040491Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528576:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.040517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241466611561207:2059] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.040897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241471662240326:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.040923Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241467367272956:2064] [node 4] ICP18 handshake (actor [3:7587241471662240326:2120]) is too old
2025-12-24T02:11:55.042909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470906528576:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.042935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241466611561194:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.044549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241465915290748:2063] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12084) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241465915290749:2112] held: no
2025-12-24T02:11:55.044660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465915290748:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.044670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.044676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465915290748:2063] [node 4] ICP22 created new session: [5:7587241470210258067:2063]
2025-12-24T02:11:55.044682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241470210258067:2063] [node 4] ICS09 handshake done sender: [5:7587241465915290749:2112] self: [5:7587241465915290764:0] peer: [4:7587241466161488273:0] socket: 574 qp: -1
2025-12-24T02:11:55.044688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241470210258067:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:55.044696Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.044955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465915290751:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.044962Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237714:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.045290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241467367273013:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.045302Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241467367273013:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.042959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241466188972605:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.042980Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.042988Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466188972605:2048] [node 1] ICP22 created new session: [2:7587241470483939997:2048]
2025-12-24T02:11:55.042994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470483939997:2048] [node 1] ICS09 handshake done sender: [2:7587241466188972684:2115] self: [2:7587241466188972694:0] peer: [1:7587241469106237707:0] socket: 579 qp: -1
2025-12-24T02:11:55.043000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470483939997:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.043006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.046368Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241471662240334:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3582
2025-12-24T02:11:55.046474Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241467367273013:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.046540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241467367272968:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.046553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.046566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241467367272968:2071] [node 6] ICP22 created new session: [3:7587241471662240335:2071]
2025-12-24T02:11:55.046573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241471662240335:2071] [node 6] ICS09 handshake done sender: [3:7587241467367273013:2114] self: [3:7587241467367273012:0] peer: [6:7587241470906528581:0] socket: 587 qp: -1
2025-12-24T02:11:55.046578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241471662240335:2071] [node 6] ICS10 traffic start
2025-12-24T02:11:55.046584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.046852Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237710:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.046867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241469106237710:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.047286Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237723:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.049059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241469106237639:2048] [node 2] ICP18 handshake (actor [1:7587241469106237723:2118]) is too old
2025-12-24T02:11:55.049087Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473401205025:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:41844
2025-12-24T02:11:55.047092Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241466188972658:2092] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.051068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241466188972606:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.051086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.051091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241466188972606:2049] [node 3] ICP22 created new session: [2:7587241470483940000:2049]
2025-12-24T02:11:55.051097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470483940000:2049] [node 3] ICS09 handshake done sender: [2:7587241466188972658:2092] self: [2:7587241466188972657:0] peer: [3:7587241471662240325:0] socket: 589 qp: -1
2025-12-24T02:11:55.051101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470483940000:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.051108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.055861Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241466161488241:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241466161488319:2117] held: no
2025-12-24T02:11:55.056156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466161488319:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.063103Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241469106237710:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.063778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241469106237702:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.063800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.063809Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241469106237702:2107] [node 4] ICP22 created new session: [1:7587241473401205031:2107]
2025-12-24T02:11:55.063816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473401205031:2107] [node 4] ICS09 handshake done sender: [1:7587241469106237710:2112] self: [1:7587241469106237709:0] peer: [4:7587241470456455627:0] socket: 564 qp: -1
2025-12-24T02:11:55.063824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473401205031:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:55.063838Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.064152Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241469106237640:2049] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.062734Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470483939999:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51775
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.066921Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470906528571:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.066967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241466611561192:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.067039Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241466611561194:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:65037) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470906528576:2121] held: no
2025-12-24T02:11:55.067335Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470456455632:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13359
2025-12-24T02:11:55.068357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466161488241:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.068375Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.068392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466161488241:2050] [node 3] ICP22 created new session: [4:7587241470456455633:2050]
2025-12-24T02:11:55.068399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470456455633:2050] [node 3] ICS09 handshake done sender: [4:7587241466161488319:2117] self: [4:7587241470456455626:0] peer: [3:7587241467367272960:0] socket: 578 qp: -1
2025-12-24T02:11:55.068408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470456455633:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.068416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.068951Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241467367272939:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241467367273020:2118] held: no
2025-12-24T02:11:55.070838Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237712:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.070852Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241469106237712:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.076056Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241469106237712:2113] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.074843Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241465915290759:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.074860Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241465915290759:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.075961Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241470210258069:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.078850Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241467367273020:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.078867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241470210258070:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:55.078976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465915290759:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.078436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528582:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.078975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241469106237703:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.078999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.079012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241469106237703:2108] [node 6] ICP22 created new session: [1:7587241473401205034:2108]
2025-12-24T02:11:55.079019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473401205034:2108] [node 6] ICS09 handshake done sender: [1:7587241469106237712:2113] self: [1:7587241469106237711:0] peer: [6:7587241470906528583:0] socket: 604 qp: -1
2025-12-24T02:11:55.079025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473401205034:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:55.079035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.079088Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241465915290753:2069] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.079104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.079111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465915290753:2069] [node 6] ICP22 created new session: [5:7587241470210258073:2069]
2025-12-24T02:11:55.079118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241470210258073:2069] [node 6] ICS09 handshake done sender: [5:7587241465915290759:2118] self: [5:7587241465915290758:0] peer: [6:7587241470906528580:0] socket: 600 qp: -1
2025-12-24T02:11:55.079124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241470210258073:2069] [node 6] ICS10 traffic start
2025-12-24T02:11:55.079133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.079485Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241465915290752:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.079486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241467367272939:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.079502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.079507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241467367272939:2049] [node 2] ICP22 created new session: [3:7587241471662240337:2049]
2025-12-24T02:11:55.079512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241471662240337:2049] [node 2] ICS09 handshake done sender: [3:7587241467367273020:2118] self: [3:7587241471662240325:0] peer: [2:7587241466188972657:0] socket: 591 qp: -1
2025-12-24T02:11:55.079517Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241471662240337:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.079525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.079837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465915290683:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.079852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.079858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465915290683:2050] [node 3] ICP22 created new session: [5:7587241470210258075:2050]
2025-12-24T02:11:55.079863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241470210258075:2050] [node 3] ICS09 handshake done sender: [5:7587241465915290752:2115] self: [5:7587241465915290767:0] peer: [3:7587241467367272970:0] socket: 577 qp: -1
2025-12-24T02:11:55.079869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241470210258075:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.079877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.081187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241466161488239:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241466161488318:2116] held: no
2025-12-24T02:11:55.081358Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241466161488318:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.081473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241466161488239:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.081487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.081493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241466161488239:2048] [node 1] ICP22 created new session: [4:7587241470456455635:2048]
2025-12-24T02:11:55.081498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470456455635:2048] [node 1] ICS09 handshake done sender: [4:7587241466161488318:2116] self: [4:7587241470456455627:0] peer: [1:7587241469106237709:0] socket: 567 qp: -1
2025-12-24T02:11:55.081503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470456455635:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.081510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.083197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470906528576:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.084246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241466611561192:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29859) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470906528571:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.085249Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528585:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.085408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470906528584:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63049
2025-12-24T02:11:55.085988Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241470210258070:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.086056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241465915290681:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.085537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241466611561207:2059] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:8566) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470906528570:2117] held: no
2025-12-24T02:11:55.085647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241466611561194:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.085657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.085664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241466611561194:2050] [node 3] ICP22 created new session: [6:7587241470906528586:2050]
2025-12-24T02:11:55.085671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470906528586:2050] [node 3] ICS09 handshake done sender: [6:7587241470906528576:2121] self: [6:7587241470906528581:0] peer: [3:7587241467367273012:0] socket: 613 qp: -1
2025-12-24T02:11:55.085679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470906528586:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.085688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.086058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470906528571:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.086200Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241469106237716:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.086212Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241469106237716:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.087248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241470210258078:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30241
2025-12-24T02:11:55.087342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241469106237716:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.087507Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241470210258070:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.087573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241465915290681:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.087588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.087593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241465915290681:2048] [node 1] ICP22 created new session: [5:7587241470210258079:2048]
2025-12-24T02:11:55.087599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241470210258079:2048] [node 1] ICS09 handshake done sender: [5:7587241470210258070:2122] self: [5:7587241470210258077:0] peer: [1:7587241469106237715:0] socket: 568 qp: -1
2025-12-24T02:11:55.087604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241470210258079:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.087611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.088671Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241469106237706:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.088687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.088692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241469106237706:2109] [node 5] ICP22 created new session: [1:7587241473401205037:2109]
2025-12-24T02:11:55.088698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473401205037:2109] [node 5] ICS09 handshake done sender: [1:7587241469106237716:2116] self: [1:7587241469106237715:0] peer: [5:7587241470210258077:0] socket: 619 qp: -1
2025-12-24T02:11:55.088703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473401205037:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:55.088711Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.089306Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241466611561192:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.089321Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.089326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241466611561192:2048] [node 1] ICP22 created new session: [6:7587241470906528588:2048]
2025-12-24T02:11:55.089331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470906528588:2048] [node 1] ICS09 handshake done sender: [6:7587241470906528571:2118] self: [6:7587241470906528583:0] peer: [1:7587241469106237711:0] socket: 608 qp: -1
2025-12-24T02:11:55.089336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470906528588:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.089343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.089616Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470906528570:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.090979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241466611561207:2059] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.090994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.091000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241466611561207:2059] [node 5] ICP22 created new session: [6:7587241470906528590:2059]
2025-12-24T02:11:55.091005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470906528590:2059] [node 5] ICS09 handshake done sender: [6:7587241470906528570:2117] self: [6:7587241470906528580:0] peer: [5:7587241465915290758:0] socket: 602 qp: -1
2025-12-24T02:11:55.091012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470906528590:2059] [node 5] ICS10 traffic start
2025-12-24T02:11:55.091019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.104056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:55.104135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241471662240328:2048] [node 1] ICS01 socket: 569 reason# EndOfStream
2025-12-24T02:11:55.104141Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241467367272938:2048] [node 1] ICP30 unregister session Session# [3:7587241471662240328:2048] VirtualId# [3:7587241467367273022:0]
2025-12-24T02:11:55.104146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241471662240328:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.104175Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.104193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.103338Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:55.103444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241470483939997:2048] [node 1] ICS01 socket: 579 reason# EndOfStream
2025-12-24T02:11:55.103454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241466188972605:2048] [node 1] ICP30 unregister session Session# [2:7587241470483939997:2048] VirtualId# [2:7587241466188972694:0]
2025-12-24T02:11:55.103461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241470483939997:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.103497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.103522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.110526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:55.110641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470456455635:2048] [node 1] ICS01 socket: 567 reason# EndOfStream
2025-12-24T02:11:55.110649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241466161488239:2048] [node 1] ICP30 unregister session Session# [4:7587241470456455635:2048] VirtualId# [4:7587241470456455627:0]
2025-12-24T02:11:55.110657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470456455635:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.110789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.110819Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.110907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:55.110990Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470906528588:2048] [node 1] ICS01 socket: 608 reason# EndOfStream
2025-12-24T02:11:55.110998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241466611561192:2048] [node 1] ICP30 unregister session Session# [6:7587241470906528588:2048] VirtualId# [6:7587241470906528583:0]
2025-12-24T02:11:55.111004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470906528588:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.111075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.111104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.111147Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:55.111221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241470210258079:2048] [node 1] ICS01 socket: 568 reason# EndOfStream
2025-12-24T02:11:55.111228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465915290681:2048] [node 1] ICP30 unregister session Session# [5:7587241470210258079:2048] VirtualId# [5:7587241470210258077:0]
2025-12-24T02:11:55.111234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241470210258079:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.111267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.111291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.119364Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:55.119459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241471662240337:2049] [node 2] ICS01 socket: 591 reason# EndOfStream
2025-12-24T02:11:55.119468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241467367272939:2049] [node 2] ICP30 unregister session Session# [3:7587241471662240337:2049] VirtualId# [3:7587241471662240325:0]
2025-12-24T02:11:55.119475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241471662240337:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.119516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.119543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.129906Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:55.130004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241470210258064:2049] [node 2] ICS01 socket: 575 reason# EndOfStream
2025-12-24T02:11:55.130015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465915290682:2049] [node 2] ICP30 unregister session Session# [5:7587241470210258064:2049] VirtualId# [5:7587241465915290763:0]
2025-12-24T02:11:55.130023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241470210258064:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.130069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.130092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.130860Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:55.130944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470456455621:2049] [node 2] ICS01 socket: 552 reason# EndOfStream
2025-12-24T02:11:55.130951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241466161488240:2049] [node 2] ICP30 unregister session Session# [4:7587241470456455621:2049] VirtualId# [4:7587241466161488321:0]
2025-12-24T02:11:55.130956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470456455621:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.131003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.131029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.130994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:55.131073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470906528578:2049] [node 2] ICS01 socket: 556 reason# EndOfStream
2025-12-24T02:11:55.131080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241466611561193:2049] [node 2] ICP30 unregister session Session# [6:7587241470906528578:2049] VirtualId# [6:7587241470906528569:0]
2025-12-24T02:11:55.131086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470906528578:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.131133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.131155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.142738Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:55.142860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241470210258075:2050] [node 3] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:55.142869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465915290683:2050] [node 3] ICP30 unregister session Session# [5:7587241470210258075:2050] VirtualId# [5:7587241465915290767:0]
2025-12-24T02:11:55.142876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241470210258075:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.142920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.142944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.142764Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:55.142860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470906528586:2050] [node 3] ICS01 socket: 613 reason# EndOfStream
2025-12-24T02:11:55.142871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241466611561194:2050] [node 3] ICP30 unregister session Session# [6:7587241470906528586:2050] VirtualId# [6:7587241470906528581:0]
2025-12-24T02:11:55.142878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470906528586:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.142950Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.142977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.155025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:55.155125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241470210258067:2063] [node 4] ICS01 socket: 574 reason# EndOfStream
2025-12-24T02:11:55.155146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241465915290748:2063] [node 4] ICP30 unregister session Session# [5:7587241470210258067:2063] VirtualId# [5:7587241465915290764:0]
2025-12-24T02:11:55.155155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241470210258067:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.155199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:55.155220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:55.156742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:55.156816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470906528574:2068] [node 4] ICS01 socket: 555 reason# EndOfStream
2025-12-24T02:11:55.156823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241466611561260:2068] [node 4] ICP30 unregister session Session# [6:7587241470906528574:2068] VirtualId# [6:7587241470906528568:0]
2025-12-24T02:11:55.156829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470906528574:2068] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.156883Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:55.156907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:55.198889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241473242142202:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:55.198905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241473242142203:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:55.223030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241470370847732:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.223054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241470370847733:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:55.231647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241474005422617:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.231662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241474005422618:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.231667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241474005422619:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.233117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470051774851:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.233133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470051774852:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.233139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470051774853:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.231053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473964044758:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.231064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473964044759:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.262765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241474135557453:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.262800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241474135557454:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.262813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241474135557455:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.278883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241474135557474:2065] [node 5] ICP01 ready to work
2025-12-24T02:11:55.278987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241474135557478:2069] [node 4] ICP01 ready to work
2025-12-24T02:11:55.277622Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473964044811:2099] [node 6] ICP01 ready to work
2025-12-24T02:11:55.279488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241474005422654:2079] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.279056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470051774898:2092] [node 5] ICP01 ready to work
2025-12-24T02:11:55.279772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241473242142266:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:55.284333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241473242142274:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:55.286726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847793:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.287005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241474005422661:2085] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.291686Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557489:2078] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.291709Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557491:2080] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.293586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847792:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.293753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241470370847797:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:55.293867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241470370847798:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:55.293492Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473964044818:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.299370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422684:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.300132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241473242142278:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:55.300712Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473242142279:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54324
2025-12-24T02:11:55.300737Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473242142280:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54324
2025-12-24T02:11:55.306785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142280:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.315339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473964044826:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:55.317042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044837:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
2025-12-24T02:11:55.317056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044838:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
2025-12-24T02:11:55.317101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044839:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.317557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422694:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.317483Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241470370847797:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23522949ns) {Configure:104(23521977ns) {ProcessConfigured:135(23213352ns) {StartInitialHandshake:157(23213055ns) {DropHandshakes:470(333ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(54ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(816ns)}}}}}
2025-12-24T02:11:55.318218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241470370847806:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:55.318963Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241474135557455:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(39926959ns) {Configure:104(39926368ns) {ProcessConfigured:135(38204332ns) {StartInitialHandshake:157(38204007ns) {DropHandshakes:470(134ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(30ns)} GenerateSessionVirtualId:988(53ns) PrepareNewSessionHandshake:477(865ns) {SwitchToState:179(324ns)}}}}}
2025-12-24T02:11:55.320309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044838:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.330878Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241470051774898:2092] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(51736467ns) {Configure:104(51735793ns) {ProcessConfigured:135(50018148ns) {StartInitialHandshake:157(50017256ns) {DropHandshakes:470(363ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(187ns) PrepareNewSessionHandshake:477(2040ns) {SwitchToState:179(1207ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.336694Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241470051774851:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(34327789ns) {Configure:104(34327058ns) {ProcessConfigured:135(33912656ns) {StartInitialHandshake:157(33912048ns) {DropHandshakes:470(398ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(189ns) PrepareNewSessionHandshake:477(910ns) {SwitchToState:179(283ns)}}}}}
2025-12-24T02:11:55.336772Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774921:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.336815Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774925:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.336828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774922:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.336838Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774926:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.336463Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473242142284:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54324
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.346935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044837:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.346976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241473964044758:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.344239Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241474135557491:2080] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.344294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557533:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.344324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241474135557474:2065] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.347806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241474135557489:2078] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.347852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241474135557455:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.346783Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241470370847733:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(59800781ns) {Configure:104(59799872ns) {ProcessConfigured:135(59053884ns) {StartInitialHandshake:157(59053097ns) {DropHandshakes:470(444ns) {DropIncomingHandshake:442(68ns) DropOutgoingHandshake:457(85ns)} GenerateSessionVirtualId:988(287ns) PrepareNewSessionHandshake:477(923ns) {SwitchToState:179(538ns)}}}}}
2025-12-24T02:11:55.348034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847793:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.348127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241474135557533:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.348143Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241474135557454:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.348102Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470051774921:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.348274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470051774851:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.348053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241470370847732:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.349753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422656:2083] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.349773Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241474005422656:2083] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.358846Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044839:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.358452Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422695:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.359128Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847803:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.359142Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241470370847803:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.359319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422684:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.359018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142263:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.359093Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044813:2100] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.359104Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473964044813:2100] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.359034Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241473242142263:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.362853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470051774925:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.362897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470051774922:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.362932Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470051774853:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.363103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470051774932:2117] [node 6] ICP01 ready to work
2025-12-24T02:11:55.363571Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470051774932:2117] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.363876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422695:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.363909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241474005422661:2085] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.363693Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142270:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.363708Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241473242142270:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.364341Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142268:2109] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.364345Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241473242142268:2109] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.366920Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422694:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.366954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241474005422619:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.365661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847792:2107] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.365921Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847815:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.370605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241474135557493:2081] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.370623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241474135557493:2081] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.370545Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241473242142270:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.368197Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470051774924:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.368215Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241470051774924:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.371313Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241473242142268:2109] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.375167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044830:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.375182Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473964044830:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.376259Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044828:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.376275Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473964044828:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.378888Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473964044813:2100] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.382950Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473964044830:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.383055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473964044811:2099] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.383077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.383087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473964044811:2099] [node 6] ICP22 created new session: [3:7587241473964044845:2099]
2025-12-24T02:11:55.383094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473964044845:2099] [node 6] ICS09 handshake done sender: [3:7587241473964044813:2100] self: [3:7587241473964044812:0] peer: [6:7587241474135557535:0] socket: 552 qp: -1
2025-12-24T02:11:55.383104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473964044845:2099] [node 6] ICS10 traffic start
2025-12-24T02:11:55.383116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.382828Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241470370847793:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.383953Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241474005422656:2083] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.384094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241474005422654:2079] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.384113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.384119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241474005422654:2079] [node 6] ICP22 created new session: [5:7587241474005422700:2079]
2025-12-24T02:11:55.384125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241474005422700:2079] [node 6] ICS09 handshake done sender: [5:7587241474005422656:2083] self: [5:7587241474005422655:0] peer: [6:7587241474135557534:0] socket: 556 qp: -1
2025-12-24T02:11:55.384130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241474005422700:2079] [node 6] ICS10 traffic start
2025-12-24T02:11:55.384137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.383711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241470370847732:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.383728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.383737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241470370847732:2048] [node 1] ICP22 created new session: [2:7587241470370847816:2048]
2025-12-24T02:11:55.383743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470370847816:2048] [node 1] ICS09 handshake done sender: [2:7587241470370847793:2108] self: [2:7587241470370847813:0] peer: [1:7587241473242142269:0] socket: 563 qp: -1
2025-12-24T02:11:55.383749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470370847816:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.383757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.388623Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470051774926:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.388822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470051774852:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.390560Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847801:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.390572Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241470370847801:2112] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.390893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470051774924:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.391554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241470051774898:2092] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.391566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.391575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470051774898:2092] [node 5] ICP22 created new session: [4:7587241470051774938:2092]
2025-12-24T02:11:55.391583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470051774938:2092] [node 5] ICS09 handshake done sender: [4:7587241470051774924:2113] self: [4:7587241470051774911:0] peer: [5:7587241474005422697:0] socket: 592 qp: -1
2025-12-24T02:11:55.391589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470051774938:2092] [node 5] ICS10 traffic start
2025-12-24T02:11:55.391599Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.391929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774936:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.392120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774937:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.392136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470051774921:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.393499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470051774851:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.393513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.393521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470051774851:2048] [node 1] ICP22 created new session: [4:7587241470051774941:2048]
2025-12-24T02:11:55.393526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470051774941:2048] [node 1] ICS09 handshake done sender: [4:7587241470051774921:2111] self: [4:7587241470051774929:0] peer: [1:7587241473242142267:0] socket: 565 qp: -1
2025-12-24T02:11:55.393532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470051774941:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.393538Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.395070Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241474135557493:2081] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.399852Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241470370847803:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.399979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241470370847798:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.399991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.399997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241470370847798:2110] [node 6] ICP22 created new session: [2:7587241470370847820:2110]
2025-12-24T02:11:55.400002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470370847820:2110] [node 6] ICS09 handshake done sender: [2:7587241470370847803:2113] self: [2:7587241470370847802:0] peer: [6:7587241474135557536:0] socket: 584 qp: -1
2025-12-24T02:11:55.400007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470370847820:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:55.400014Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.399557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473964044818:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.399582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.399588Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473964044818:2103] [node 5] ICP22 created new session: [3:7587241473964044849:2103]
2025-12-24T02:11:55.399596Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473964044849:2103] [node 5] ICS09 handshake done sender: [3:7587241473964044830:2112] self: [3:7587241473964044829:0] peer: [5:7587241474005422698:0] socket: 579 qp: -1
2025-12-24T02:11:55.399602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473964044849:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:55.399610Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.400378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422702:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.400537Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422703:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.400581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422704:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.401218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241474005422694:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.401275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241474005422619:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.401286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.401292Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241474005422619:2050] [node 3] ICP22 created new session: [5:7587241474005422708:2050]
2025-12-24T02:11:55.401297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241474005422708:2050] [node 3] ICS09 handshake done sender: [5:7587241474005422694:2115] self: [5:7587241474005422698:0] peer: [3:7587241473964044829:0] socket: 582 qp: -1
2025-12-24T02:11:55.401302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241474005422708:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.401308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.401614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241474005422695:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.402436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241474005422661:2085] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.402446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.402450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241474005422661:2085] [node 4] ICP22 created new session: [5:7587241474005422710:2085]
2025-12-24T02:11:55.402456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241474005422710:2085] [node 4] ICS09 handshake done sender: [5:7587241474005422695:2116] self: [5:7587241474005422697:0] peer: [4:7587241470051774911:0] socket: 593 qp: -1
2025-12-24T02:11:55.402460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241474005422710:2085] [node 4] ICS10 traffic start
2025-12-24T02:11:55.402465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.404469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473964044828:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.404802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473964044826:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.404817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.404823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473964044826:2110] [node 4] ICP22 created new session: [3:7587241473964044852:2110]
2025-12-24T02:11:55.404829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473964044852:2110] [node 4] ICS09 handshake done sender: [3:7587241473964044828:2111] self: [3:7587241473964044827:0] peer: [4:7587241470051774931:0] socket: 578 qp: -1
2025-12-24T02:11:55.404833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473964044852:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:55.404839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.406796Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142279:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.406823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241473242142202:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.406835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.406843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241473242142202:2048] [node 2] ICP22 created new session: [1:7587241473242142287:2048]
2025-12-24T02:11:55.406850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473242142287:2048] [node 2] ICS09 handshake done sender: [1:7587241473242142270:2110] self: [1:7587241473242142269:0] peer: [2:7587241470370847813:0] socket: 560 qp: -1
2025-12-24T02:11:55.406855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473242142287:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:55.406863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.407024Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241473242142202:2048] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.407230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241473242142266:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.407237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.407242Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241473242142266:2108] [node 4] ICP22 created new session: [1:7587241473242142290:2108]
2025-12-24T02:11:55.407247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473242142290:2108] [node 4] ICS09 handshake done sender: [1:7587241473242142268:2109] self: [1:7587241473242142267:0] peer: [4:7587241470051774929:0] socket: 559 qp: -1
2025-12-24T02:11:55.407252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473242142290:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:55.407257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.407876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473242142292:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54324
2025-12-24T02:11:55.408066Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422713:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.410928Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241473964044811:2099] [node 6] ICP35 event processing took too much time PendingConnection:279(27775459ns) {HandleHandshakeStatus:329(27774319ns) {DropIncomingHandshake:442(230ns) DropOutgoingHandshake:457(356ns) SwitchToState:179(210ns) ProcessPendingSessionEvents:495(27655958ns) {unordered ForwardSessionEventToSession:612 num# 70 sum# 27639521ns max# 27574339ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.411149Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241474005422654:2079] [node 6] ICP35 event processing took too much time PendingConnection:279(26957985ns) {HandleHandshakeStatus:329(26957236ns) {DropIncomingHandshake:442(4052ns) DropOutgoingHandshake:457(110ns) SwitchToState:179(126ns) ProcessPendingSessionEvents:495(26864729ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26843268ns max# 26762776ns}}}
2025-12-24T02:11:55.414895Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422703:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.414970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241474005422618:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.414720Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044854:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
2025-12-24T02:11:55.412256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847822:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.412297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847823:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.417522Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241470370847801:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.417707Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241470370847798:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(17622009ns) {HandleHandshakeStatus:329(17621245ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(156ns) SwitchToState:179(65ns) ProcessPendingSessionEvents:495(17545755ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 17528212ns max# 17448644ns}}}
2025-12-24T02:11:55.417842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847822:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.417880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847808:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.417883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241470370847808:2115] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.421248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241473242142293:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54324
2025-12-24T02:11:55.421331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241473242142266:2108] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.419912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241474135557478:2069] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.421383Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241473242142263:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.421538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241470370847808:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.421615Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241470370847797:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.421632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.421639Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241470370847797:2109] [node 4] ICP22 created new session: [2:7587241470370847828:2109]
2025-12-24T02:11:55.421645Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470370847828:2109] [node 4] ICS09 handshake done sender: [2:7587241470370847801:2112] self: [2:7587241470370847799:0] peer: [4:7587241470051774935:0] socket: 583 qp: -1
2025-12-24T02:11:55.421649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470370847828:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:55.421656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.421864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241470370847806:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.421872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.421875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241470370847806:2111] [node 5] ICP22 created new session: [2:7587241470370847830:2111]
2025-12-24T02:11:55.421879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470370847830:2111] [node 5] ICS09 handshake done sender: [2:7587241470370847808:2115] self: [2:7587241470370847807:0] peer: [5:7587241474005422714:0] socket: 585 qp: -1
2025-12-24T02:11:55.421883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470370847830:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:55.421887Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.419944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.419953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241474135557478:2069] [node 4] ICP22 created new session: [6:7587241474135557540:2069]
2025-12-24T02:11:55.419960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241474135557540:2069] [node 4] ICS09 handshake done sender: [6:7587241474135557493:2081] self: [6:7587241474135557492:0] peer: [4:7587241470051774933:0] socket: 570 qp: -1
2025-12-24T02:11:55.419967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241474135557540:2069] [node 4] ICS10 traffic start
2025-12-24T02:11:55.419976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.421546Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241474135557474:2065] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31403) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241474135557491:2080] held: no
2025-12-24T02:11:55.421726Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557538:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.422909Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422715:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.423091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241474005422703:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.423151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241474005422618:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.423165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.423170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241474005422618:2049] [node 2] ICP22 created new session: [5:7587241474005422716:2049]
2025-12-24T02:11:55.423176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241474005422716:2049] [node 2] ICS09 handshake done sender: [5:7587241474005422703:2118] self: [5:7587241474005422714:0] peer: [2:7587241470370847807:0] socket: 608 qp: -1
2025-12-24T02:11:55.423183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241474005422716:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.423190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.422909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241473242142203:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.422928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.422936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241473242142203:2049] [node 3] ICP22 created new session: [1:7587241473242142297:2049]
2025-12-24T02:11:55.422941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473242142297:2049] [node 3] ICS09 handshake done sender: [1:7587241473242142263:2107] self: [1:7587241473242142262:0] peer: [3:7587241473964044841:0] socket: 554 qp: -1
2025-12-24T02:11:55.422946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473242142297:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.422954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.424740Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470051774922:2112] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.430165Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241470051774853:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26141) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241470051774925:2114] held: no
2025-12-24T02:11:55.430323Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774943:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.428184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044855:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
2025-12-24T02:11:55.430817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142293:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.430891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241473242142203:2049] [node 3] ICP18 handshake (actor [1:7587241473242142293:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.434311Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473964044855:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.434437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241473964044759:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.434608Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473964044837:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.434658Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557539:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.434777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241474005422713:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.434848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241474005422617:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.434890Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847823:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.435074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241473964044758:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.435090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.435096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473964044758:2048] [node 1] ICP22 created new session: [3:7587241473964044857:2048]
2025-12-24T02:11:55.435102Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473964044857:2048] [node 1] ICS09 handshake done sender: [3:7587241473964044837:2115] self: [3:7587241473964044841:0] peer: [1:7587241473242142262:0] socket: 571 qp: -1
2025-12-24T02:11:55.435108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473964044857:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.435115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.438827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142277:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.438840Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241473242142277:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.442864Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241470370847812:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.442878Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241470370847812:2116] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.442929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557543:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.443069Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241474135557489:2078] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.444165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241473242142277:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.446857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241474135557491:2080] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.447238Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557542:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.447459Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241470370847833:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:53495
2025-12-24T02:11:55.449381Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241474135557455:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.449420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.449428Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241474135557455:2050] [node 3] ICP22 created new session: [6:7587241474135557545:2050]
2025-12-24T02:11:55.449435Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241474135557545:2050] [node 3] ICS09 handshake done sender: [6:7587241474135557489:2078] self: [6:7587241474135557535:0] peer: [3:7587241473964044812:0] socket: 566 qp: -1
2025-12-24T02:11:55.449442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241474135557545:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.449450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.449806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241474135557543:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.449814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241474135557474:2065] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.449820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.449830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241474135557474:2065] [node 5] ICP22 created new session: [6:7587241474135557548:2065]
2025-12-24T02:11:55.449835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241474135557548:2065] [node 5] ICS09 handshake done sender: [6:7587241474135557491:2080] self: [6:7587241474135557534:0] peer: [5:7587241474005422655:0] socket: 567 qp: -1
2025-12-24T02:11:55.449840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241474135557548:2065] [node 5] ICS10 traffic start
2025-12-24T02:11:55.449845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.450319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241474135557453:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.446965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241473242142274:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.446991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.446999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241473242142274:2111] [node 5] ICP22 created new session: [1:7587241473242142300:2111]
2025-12-24T02:11:55.447005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473242142300:2111] [node 5] ICS09 handshake done sender: [1:7587241473242142277:2113] self: [1:7587241473242142275:0] peer: [5:7587241474005422718:0] socket: 614 qp: -1
2025-12-24T02:11:55.447012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473242142300:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:55.447022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.450856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241474135557533:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.452315Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241474005422617:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241474005422713:2121] held: no
2025-12-24T02:11:55.452473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241474005422719:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43898
2025-12-24T02:11:55.453955Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241474005422713:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.453224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241474135557454:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.453242Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.453248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241474135557454:2049] [node 2] ICP22 created new session: [6:7587241474135557551:2049]
2025-12-24T02:11:55.453254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241474135557551:2049] [node 2] ICS09 handshake done sender: [6:7587241474135557533:2117] self: [6:7587241474135557536:0] peer: [2:7587241470370847802:0] socket: 586 qp: -1
2025-12-24T02:11:55.453260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241474135557551:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.453266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.453763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241473242142282:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.453774Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241473242142282:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.454892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473964044859:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7526
2025-12-24T02:11:55.454949Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241470370847812:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.455126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473964044855:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.455069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241470370847733:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.455084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.455090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241470370847733:2049] [node 3] ICP22 created new session: [2:7587241470370847835:2049]
2025-12-24T02:11:55.455094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241470370847835:2049] [node 3] ICS09 handshake done sender: [2:7587241470370847812:2116] self: [2:7587241470370847794:0] peer: [3:7587241473964044856:0] socket: 620 qp: -1
2025-12-24T02:11:55.455098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241470370847835:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.455109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.455225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241473964044759:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.455239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.455245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473964044759:2049] [node 2] ICP22 created new session: [3:7587241473964044860:2049]
2025-12-24T02:11:55.455252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473964044860:2049] [node 2] ICS09 handshake done sender: [3:7587241473964044855:2119] self: [3:7587241473964044856:0] peer: [2:7587241470370847794:0] socket: 623 qp: -1
2025-12-24T02:11:55.455259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473964044860:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.455266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: 2025-12-24T02:11:55.455827Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241473242142282:2116] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:55.455982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241473242142278:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.456007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.456013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241473242142278:2112] [node 6] ICP22 created new session: [1:7587241473242142303:2112]
2025-12-24T02:11:55.456018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241473242142303:2112] [node 6] ICS09 handshake done sender: [1:7587241473242142282:2116] self: [1:7587241473242142281:0] peer: [6:7587241474135557550:0] socket: 615 qp: -1
2025-12-24T02:11:55.455019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470051774932:2117] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.455038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.455046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470051774932:2117] [node 6] ICP22 created new session: [4:7587241470051774946:2117]
2025-12-24T02:11:55.455057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470051774946:2117] [node 6] ICS09 handshake done sender: [4:7587241470051774922:2112] self: [4:7587241470051774933:0] peer: [6:7587241474135557492:0] socket: 577 qp: -1
2025-12-24T02:11:55.455064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470051774946:2117] [node 6] ICS10 traffic start
2025-12-24T02:11:55.455073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.455555Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774945:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
2025-12-24T02:11:55.455886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470051774944:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6747
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.458787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241474005422617:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.458804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.458812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241474005422617:2048] [node 1] ICP22 created new session: [5:7587241474005422720:2048]
2025-12-24T02:11:55.458818Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241474005422720:2048] [node 1] ICS09 handshake done sender: [5:7587241474005422713:2121] self: [5:7587241474005422718:0] peer: [1:7587241473242142275:0] socket: 616 qp: -1
2025-12-24T02:11:55.458825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241474005422720:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.458831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.456024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241473242142303:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:55.456036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.459479Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241474135557553:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:8477
2025-12-24T02:11:55.466210Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241474135557543:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.466320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241474135557453:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.466336Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.466345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241474135557453:2048] [node 1] ICP22 created new session: [6:7587241474135557554:2048]
2025-12-24T02:11:55.466352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241474135557554:2048] [node 1] ICS09 handshake done sender: [6:7587241474135557543:2121] self: [6:7587241474135557550:0] peer: [1:7587241473242142281:0] socket: 622 qp: -1
2025-12-24T02:11:55.466359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241474135557554:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.466370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.470909Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241470370847798:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.478810Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241470051774852:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63440) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241470051774926:2115] held: no
2025-12-24T02:11:55.478976Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470051774925:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.479640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470051774853:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.479654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.479663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470051774853:2050] [node 3] ICP22 created new session: [4:7587241470051774948:2050]
2025-12-24T02:11:55.479670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470051774948:2050] [node 3] ICS09 handshake done sender: [4:7587241470051774925:2114] self: [4:7587241470051774931:0] peer: [3:7587241473964044827:0] socket: 589 qp: -1
2025-12-24T02:11:55.479678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470051774948:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.479686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.480182Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470051774926:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.480234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470051774852:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.480243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.480247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470051774852:2049] [node 2] ICP22 created new session: [4:7587241470051774950:2049]
2025-12-24T02:11:55.480251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470051774950:2049] [node 2] ICS09 handshake done sender: [4:7587241470051774926:2115] self: [4:7587241470051774935:0] peer: [2:7587241470370847799:0] socket: 590 qp: -1
2025-12-24T02:11:55.480256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470051774950:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.480262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.498773Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:55.498894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241470370847816:2048] [node 1] ICS01 socket: 563 reason# EndOfStream
2025-12-24T02:11:55.498909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241470370847732:2048] [node 1] ICP30 unregister session Session# [2:7587241470370847816:2048] VirtualId# [2:7587241470370847813:0]
2025-12-24T02:11:55.498919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241470370847816:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.498969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.498996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.502850Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:55.502953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241473964044857:2048] [node 1] ICS07 socket disconnect 571 reason# EndOfStream
2025-12-24T02:11:55.502960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241473964044857:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.503011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.503034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241473964044857:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.503457Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241473964044758:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.503469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241473964044857:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.503474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241473964044758:2048] [node 1] ICP30 unregister session Session# [3:7587241473964044857:2048] VirtualId# [3:7587241473964044841:0]
2025-12-24T02:11:55.503531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.503537Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241473964044758:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.503543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.506767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:55.506867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241474005422720:2048] [node 1] ICS01 socket: 616 reason# EndOfStream
2025-12-24T02:11:55.506876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241474005422617:2048] [node 1] ICP30 unregister session Session# [5:7587241474005422720:2048] VirtualId# [5:7587241474005422718:0]
2025-12-24T02:11:55.506883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241474005422720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.506926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.506957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.506767Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:55.506874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241470051774941:2048] [node 1] ICS07 socket disconnect 565 reason# EndOfStream
2025-12-24T02:11:55.506882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470051774941:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.506935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.506954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241470051774941:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.507318Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241470051774851:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.507329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470051774941:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.507334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241470051774851:2048] [node 1] ICP30 unregister session Session# [4:7587241470051774941:2048] VirtualId# [4:7587241470051774929:0]
2025-12-24T02:11:55.507371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.507377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241470051774851:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.507379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.514813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241473964044758:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:55.515145Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:55.515243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241474135557554:2048] [node 1] ICS07 socket disconnect 622 reason# EndOfStream
2025-12-24T02:11:55.515249Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241474135557554:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.515298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.515322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241474135557554:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.515718Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241474135557453:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.515730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241474135557554:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.515735Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241474135557453:2048] [node 1] ICP30 unregister session Session# [6:7587241474135557554:2048] VirtualId# [6:7587241474135557550:0]
2025-12-24T02:11:55.515790Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.515797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241474135557453:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.515800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13524) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.521272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241470051774851:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:55.522742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:55.522859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241473964044860:2049] [node 2] ICS07 socket disconnect 623 reason# EndOfStream
2025-12-24T02:11:55.522867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241473964044860:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.522928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.522952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241473964044860:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:55.525107Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:55.525196Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241474135557551:2049] [node 2] ICS01 socket: 586 reason# EndOfStream
2025-12-24T02:11:55.525205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241474135557454:2049] [node 2] ICP30 unregister session Session# [6:7587241474135557551:2049] VirtualId# [6:7587241474135557536:0]
2025-12-24T02:11:55.525211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241474135557551:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.525254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.525289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.526762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:55.526768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:55.526861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241474005422716:2049] [node 2] ICS01 socket: 608 reason# EndOfStream
2025-12-24T02:11:55.526855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470051774950:2049] [node 2] ICS01 socket: 590 reason# EndOfStream
2025-12-24T02:11:55.526867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241474005422618:2049] [node 2] ICP30 unregister session Session# [5:7587241474005422716:2049] VirtualId# [5:7587241474005422714:0]
2025-12-24T02:11:55.526863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241470051774852:2049] [node 2] ICP30 unregister session Session# [4:7587241470051774950:2049] VirtualId# [4:7587241470051774935:0]
2025-12-24T02:11:55.526871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470051774950:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.526873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241474005422716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.526920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.526920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.526956Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.526944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.530718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241474135557453:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:55.562763Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:55.562866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241474005422708:2050] [node 3] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:55.562878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241474005422619:2050] [node 3] ICP30 unregister session Session# [5:7587241474005422708:2050] VirtualId# [5:7587241474005422698:0]
2025-12-24T02:11:55.562887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241474005422708:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.562945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.562971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.570988Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:55.571088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241474135557545:2050] [node 3] ICS01 socket: 566 reason# EndOfStream
2025-12-24T02:11:55.571098Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241474135557455:2050] [node 3] ICP30 unregister session Session# [6:7587241474135557545:2050] VirtualId# [6:7587241474135557535:0]
2025-12-24T02:11:55.571105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241474135557545:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.571158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.571186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.574762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:55.574842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241474005422710:2085] [node 4] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:55.574850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241474005422661:2085] [node 4] ICP30 unregister session Session# [5:7587241474005422710:2085] VirtualId# [5:7587241474005422697:0]
2025-12-24T02:11:55.574856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241474005422710:2085] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.574903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:55.574936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:55.575650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:55.575723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241474135557540:2069] [node 4] ICS01 socket: 570 reason# EndOfStream
2025-12-24T02:11:55.575729Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241474135557478:2069] [node 4] ICP30 unregister session Session# [6:7587241474135557540:2069] VirtualId# [6:7587241474135557492:0]
2025-12-24T02:11:55.575735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241474135557540:2069] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.575770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:55.575800Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:55.674879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241471364030720:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:55.674891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241471364030721:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:55.673621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241471269246877:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.673637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241471269246878:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:55.678586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473961386552:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.678596Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473961386553:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.686856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241471791496135:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.686865Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241471791496136:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.686870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241471791496137:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.702748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470582916291:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.702786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470582916292:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.702797Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470582916293:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.711076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241470501558633:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:55.711109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241470501558634:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:55.711115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241470501558635:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:55.742662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241471364030782:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:55.743139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473961386587:2075] [node 4] ICP01 ready to work
2025-12-24T02:11:55.743017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241471269246922:2088] [node 4] ICP01 ready to work
2025-12-24T02:11:55.743117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241471269246923:2090] [node 5] ICP01 ready to work
2025-12-24T02:11:55.743132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470582916328:2076] [node 5] ICP01 ready to work
2025-12-24T02:11:55.743964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473961386591:2071] [node 6] ICP01 ready to work
2025-12-24T02:11:55.744317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241471269246933:2093] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.745375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473961386596:2088] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26227
2025-12-24T02:11:55.743092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241471791496153:2064] [node 6] ICP01 ready to work
2025-12-24T02:11:55.743345Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241471364030786:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:55.743753Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241471364030788:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.750626Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241471364030793:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23080
2025-12-24T02:11:55.746861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241470501558662:2070] [node 5] ICP01 ready to work
2025-12-24T02:11:55.754054Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030793:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.754512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241471364030721:2049] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.750846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246942:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.751376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241471791496174:2073] [node 4] ICP01 ready to work
2025-12-24T02:11:55.753959Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241473961386619:2099] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.763384Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496197:2099] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.763445Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496198:2100] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.763457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496199:2101] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.764894Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386595:2087] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.764911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473961386595:2087] [node 1] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.762776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241471364030786:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(19373356ns) {Configure:104(19372987ns) {ProcessConfigured:135(19122281ns) {StartInitialHandshake:157(19122073ns) {DropHandshakes:470(227ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(58ns) PrepareNewSessionHandshake:477(923ns) {SwitchToState:179(76ns)}}}}}
2025-12-24T02:11:55.767466Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246942:2102] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.768343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241471364030798:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23080
2025-12-24T02:11:55.768319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241471791496135:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21787218ns) {Configure:104(21786765ns) {ProcessConfigured:135(21495717ns) {StartInitialHandshake:157(21495226ns) {DropHandshakes:470(395ns) {DropIncomingHandshake:442(49ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(199ns) PrepareNewSessionHandshake:477(607ns) {SwitchToState:179(137ns)}}}}}
2025-12-24T02:11:55.768540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496197:2099] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.768564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241471791496135:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.766791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473961386595:2087] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.766942Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473961386552:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.766958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.766965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473961386552:2048] [node 1] ICP22 created new session: [3:7587241473961386631:2048]
2025-12-24T02:11:55.766971Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473961386631:2048] [node 1] ICS09 handshake done sender: [3:7587241473961386595:2087] self: [3:7587241473961386594:0] peer: [1:7587241471364030794:0] socket: 569 qp: -1
2025-12-24T02:11:55.766979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473961386631:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.766999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.770733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241470501558671:2078] [node 4] ICP01 ready to work
2025-12-24T02:11:55.771518Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241470582916348:2095] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.772872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473961386634:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26227
2025-12-24T02:11:55.773584Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386596:2088] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.773294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558710:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.773341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558711:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.773369Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558712:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.775423Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241471364030720:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20446126ns) {Configure:104(20445591ns) {ProcessConfigured:135(20119202ns) {StartInitialHandshake:157(20119004ns) {DropHandshakes:470(513ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(210ns)} GenerateSessionVirtualId:988(256ns) PrepareNewSessionHandshake:477(693ns) {SwitchToState:179(450ns)}}}}}
2025-12-24T02:11:55.775572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241471364030793:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.776947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241471364030721:2049] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.776965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.776972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241471364030721:2049] [node 3] ICP22 created new session: [1:7587241471364030801:2049]
2025-12-24T02:11:55.776978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241471364030801:2049] [node 3] ICS09 handshake done sender: [1:7587241471364030793:2112] self: [1:7587241471364030794:0] peer: [3:7587241473961386594:0] socket: 572 qp: -1
2025-12-24T02:11:55.776986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241471364030801:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.776993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.777379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241471364030800:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23080
2025-12-24T02:11:55.774865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470501558711:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.776134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241470501558635:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.774956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916352:2102] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.775124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916367:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.775154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916368:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.775251Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241470582916328:2076] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32020048ns) {Configure:104(32019489ns) {ProcessConfigured:135(31561274ns) {StartInitialHandshake:157(31560682ns) {DropHandshakes:470(399ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(217ns) PrepareNewSessionHandshake:477(590ns) {SwitchToState:179(205ns)}}}}}
2025-12-24T02:11:55.779770Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030790:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.779787Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241471364030790:2111] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.778847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916367:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.778884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916352:2102] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.779920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470582916292:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.779948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470582916291:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.783113Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496198:2100] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.783153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241471791496136:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.785496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470501558710:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.785528Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470501558712:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.785564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241470501558662:2070] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.785575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241470501558633:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.782655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386598:2089] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.782669Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473961386598:2089] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.784174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386634:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.784306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473961386638:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26227
2025-12-24T02:11:55.784344Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473961386639:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26227
2025-12-24T02:11:55.784359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241473961386553:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.784519Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246932:2097] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.784532Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241471269246932:2097] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.785057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246958:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.785069Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246959:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.785107Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246960:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.786656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916368:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.786683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241470582916293:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.785746Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496218:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.787029Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496199:2101] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.786786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246928:2095] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.786795Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241471269246928:2095] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.787241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241471269246932:2097] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.787513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241471269246923:2090] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.787530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.787536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241471269246923:2090] [node 5] ICP22 created new session: [2:7587241471269246962:2090]
2025-12-24T02:11:55.787543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241471269246962:2090] [node 5] ICS09 handshake done sender: [2:7587241471269246932:2097] self: [2:7587241471269246931:0] peer: [5:7587241471791496217:0] socket: 565 qp: -1
2025-12-24T02:11:55.787549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241471269246962:2090] [node 5] ICS10 traffic start
2025-12-24T02:11:55.787567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.788058Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241471791496198:2100] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.788145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241471791496136:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.788160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.788166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241471791496136:2049] [node 2] ICP22 created new session: [5:7587241471791496220:2049]
2025-12-24T02:11:55.788172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241471791496220:2049] [node 2] ICS09 handshake done sender: [5:7587241471791496198:2100] self: [5:7587241471791496217:0] peer: [2:7587241471269246931:0] socket: 585 qp: -1
2025-12-24T02:11:55.788178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241471791496220:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.788194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.787249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386638:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.787285Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386602:2090] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.787288Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473961386602:2090] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.791184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030785:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.791199Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241471364030785:2110] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.790738Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473961386602:2090] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.790910Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473961386587:2075] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.790921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.790930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473961386587:2075] [node 4] ICP22 created new session: [3:7587241473961386643:2075]
2025-12-24T02:11:55.790936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473961386643:2075] [node 4] ICS09 handshake done sender: [3:7587241473961386602:2090] self: [3:7587241473961386601:0] peer: [4:7587241470582916374:0] socket: 580 qp: -1
2025-12-24T02:11:55.790942Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473961386643:2075] [node 4] ICS10 traffic start
2025-12-24T02:11:55.790951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.794714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246930:2096] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.794733Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241471269246930:2096] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.794777Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496155:2065] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.794789Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241471791496155:2065] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.796053Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030796:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.796063Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241471364030796:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.797028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496223:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.799746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473961386598:2089] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.799966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473961386591:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.799985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.799993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473961386591:2071] [node 6] ICP22 created new session: [3:7587241473961386647:2071]
2025-12-24T02:11:55.800000Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473961386647:2071] [node 6] ICS09 handshake done sender: [3:7587241473961386598:2089] self: [3:7587241473961386597:0] peer: [6:7587241470501558713:0] socket: 570 qp: -1
2025-12-24T02:11:55.800006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473961386647:2071] [node 6] ICS10 traffic start
2025-12-24T02:11:55.800015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.805473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241470501558635:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470501558711:2115] held: no
2025-12-24T02:11:55.806595Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241471364030790:2111] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.806735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241471364030796:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.807577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246959:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.808007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241471269246928:2095] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.815930Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241471791496197:2099] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.816149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241471791496135:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.816173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.816181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241471791496135:2048] [node 1] ICP22 created new session: [5:7587241471791496225:2048]
2025-12-24T02:11:55.816189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241471791496225:2048] [node 1] ICS09 handshake done sender: [5:7587241471791496197:2099] self: [5:7587241471791496202:0] peer: [1:7587241471364030789:0] socket: 575 qp: -1
2025-12-24T02:11:55.816198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241471791496225:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.816208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.816377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241471791496155:2065] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.819226Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241473961386591:2071] [node 6] ICP35 event processing took too much time PendingConnection:279(19136348ns) {HandleHandshakeStatus:329(19135615ns) {DropIncomingHandshake:442(4233ns) DropOutgoingHandshake:457(205ns) SwitchToState:179(91ns) ProcessPendingSessionEvents:495(19048016ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19027300ns max# 18934909ns}}}
2025-12-24T02:11:55.819456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241471791496153:2064] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.819480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.819486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241471791496153:2064] [node 6] ICP22 created new session: [5:7587241471791496228:2064]
2025-12-24T02:11:55.819493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241471791496228:2064] [node 6] ICS09 handshake done sender: [5:7587241471791496155:2065] self: [5:7587241471791496154:0] peer: [6:7587241470501558714:0] socket: 558 qp: -1
2025-12-24T02:11:55.819499Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241471791496228:2064] [node 6] ICS10 traffic start
2025-12-24T02:11:55.819506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.824168Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558718:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.823951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916375:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.826441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558719:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.824281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241471269246878:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.826730Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558720:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.824305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.824318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241471269246878:2049] [node 3] ICP22 created new session: [2:7587241471269246968:2049]
2025-12-24T02:11:55.824326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241471269246968:2049] [node 3] ICS09 handshake done sender: [2:7587241471269246928:2095] self: [2:7587241471269246927:0] peer: [3:7587241473961386640:0] socket: 563 qp: -1
2025-12-24T02:11:55.824332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241471269246968:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:55.824342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.824609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246958:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.826335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241471269246878:2049] [node 3] ICP18 handshake (actor [2:7587241471269246958:2116]) is too old
2025-12-24T02:11:55.826400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241471269246930:2096] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.826761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558721:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.826824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241471364030786:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.826851Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.826861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241471364030786:2109] [node 6] ICP22 created new session: [1:7587241471364030805:2109]
2025-12-24T02:11:55.826867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241471364030805:2109] [node 6] ICS09 handshake done sender: [1:7587241471364030796:2113] self: [1:7587241471364030787:0] peer: [6:7587241470501558715:0] socket: 584 qp: -1
2025-12-24T02:11:55.827092Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916376:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.827141Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916377:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.826929Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496227:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.833935Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241470501558662:2070] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14449) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470501558710:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.835112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916376:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.835143Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470582916368:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.834549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241473961386649:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:26227
2025-12-24T02:11:55.826873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241471364030805:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:55.826882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.827140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241471364030788:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.827149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.827154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241471364030788:2108] [node 5] ICP22 created new session: [1:7587241471364030807:2108]
2025-12-24T02:11:55.827159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241471364030807:2108] [node 5] ICS09 handshake done sender: [1:7587241471364030790:2111] self: [1:7587241471364030789:0] peer: [5:7587241471791496202:0] socket: 562 qp: -1
2025-12-24T02:11:55.827164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241471364030807:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:55.827169Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.830855Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241471364030785:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.830936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241471364030782:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.830946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.830952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241471364030782:2107] [node 4] ICP22 created new session: [1:7587241471364030811:2107]
2025-12-24T02:11:55.830958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241471364030811:2107] [node 4] ICS09 handshake done sender: [1:7587241471364030785:2110] self: [1:7587241471364030784:0] peer: [4:7587241470582916371:0] socket: 561 qp: -1
2025-12-24T02:11:55.830964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241471364030811:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:55.830971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.834207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241471364030813:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23080
2025-12-24T02:11:55.834269Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241471364030814:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23080
2025-12-24T02:11:55.830818Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496227:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.830911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241471791496137:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.838871Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241473961386629:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.838879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241473961386629:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.840172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241473961386553:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241473961386634:2114] held: no
2025-12-24T02:11:55.840275Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473961386634:2114] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.842189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241473961386553:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.842201Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.842207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473961386553:2049] [node 2] ICP22 created new session: [3:7587241473961386651:2049]
2025-12-24T02:11:55.842211Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473961386651:2049] [node 2] ICS09 handshake done sender: [3:7587241473961386634:2114] self: [3:7587241473961386640:0] peer: [2:7587241471269246927:0] socket: 568 qp: -1
2025-12-24T02:11:55.842216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473961386651:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.842221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.840334Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470501558710:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.840481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241470501558662:2070] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.840492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.840498Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241470501558662:2070] [node 5] ICP22 created new session: [6:7587241470501558722:2070]
2025-12-24T02:11:55.840504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470501558722:2070] [node 5] ICS09 handshake done sender: [6:7587241470501558710:2114] self: [6:7587241470501558714:0] peer: [5:7587241471791496154:0] socket: 581 qp: -1
2025-12-24T02:11:55.840510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470501558722:2070] [node 5] ICS10 traffic start
2025-12-24T02:11:55.840517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.840722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470501558711:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.840889Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470501558712:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.840943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241470501558633:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.840951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.840956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241470501558633:2048] [node 1] ICP22 created new session: [6:7587241470501558724:2048]
2025-12-24T02:11:55.840961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470501558724:2048] [node 1] ICS09 handshake done sender: [6:7587241470501558712:2116] self: [6:7587241470501558715:0] peer: [1:7587241471364030787:0] socket: 593 qp: -1
2025-12-24T02:11:55.840966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470501558724:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.840970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:55.842990Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241473961386629:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.840503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470582916293:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.840520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.840526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470582916293:2050] [node 3] ICP22 created new session: [4:7587241470582916379:2050]
2025-12-24T02:11:55.840532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470582916379:2050] [node 3] ICS09 handshake done sender: [4:7587241470582916368:2114] self: [4:7587241470582916374:0] peer: [3:7587241473961386601:0] socket: 595 qp: -1
2025-12-24T02:11:55.840538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470582916379:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.840545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.840998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916378:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
2025-12-24T02:11:55.843086Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241471269246923:2090] [node 5] ICP35 event processing took too much time PendingConnection:279(55454573ns) {HandleHandshakeStatus:329(55453696ns) {DropIncomingHandshake:442(3393ns) DropOutgoingHandshake:457(344ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(48044623ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 40386645ns max# 40300514ns}}}
2025-12-24T02:11:55.843112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241471269246922:2088] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.843120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.843126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241471269246922:2088] [node 4] ICP22 created new session: [2:7587241471269246971:2088]
2025-12-24T02:11:55.843131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241471269246971:2088] [node 4] ICS09 handshake done sender: [2:7587241471269246930:2096] self: [2:7587241471269246929:0] peer: [4:7587241470582916370:0] socket: 564 qp: -1
2025-12-24T02:11:55.843138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241471269246971:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:55.843144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.843207Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241471791496135:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(26945757ns) {HandleHandshakeStatus:329(26945178ns) {DropIncomingHandshake:442(287ns) DropOutgoingHandshake:457(2094ns) SwitchToState:179(4356ns) ProcessPendingSessionEvents:495(26833540ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26814731ns max# 26736727ns}}}
2025-12-24T02:11:55.845282Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246960:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.843301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241473961386619:2099] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.843318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.843322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241473961386619:2099] [node 5] ICP22 created new session: [3:7587241473961386653:2099]
2025-12-24T02:11:55.843327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241473961386653:2099] [node 5] ICS09 handshake done sender: [3:7587241473961386629:2113] self: [3:7587241473961386628:0] peer: [5:7587241471791496230:0] socket: 612 qp: -1
2025-12-24T02:11:55.843331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241473961386653:2099] [node 5] ICS10 traffic start
2025-12-24T02:11:55.844414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241470501558635:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.843335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:55.844428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.844433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241470501558635:2050] [node 3] ICP22 created new session: [6:7587241470501558726:2050]
2025-12-24T02:11:55.844438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470501558726:2050] [node 3] ICS09 handshake done sender: [6:7587241470501558711:2115] self: [6:7587241470501558713:0] peer: [3:7587241473961386597:0] socket: 592 qp: -1
2025-12-24T02:11:55.844443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470501558726:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.844447Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.844611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470501558720:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.844875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241470501558634:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.843648Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030813:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.843694Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030800:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.844112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241471364030786:2109] [node 6] ICP18 handshake (actor [1:7587241471364030813:2117]) is too old
2025-12-24T02:11:55.844123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241471364030782:2107] [node 4] ICP18 handshake (actor [1:7587241471364030800:2116]) is too old
2025-12-24T02:11:55.847155Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241471791496137:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29542) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241471791496227:2120] held: no
2025-12-24T02:11:55.847747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246935:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.847757Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241471269246935:2098] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.848532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241471269246935:2098] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.848612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241471269246933:2093] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.848628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.848634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241471269246933:2093] [node 6] ICP22 created new session: [2:7587241471269246974:2093]
2025-12-24T02:11:55.848639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241471269246974:2093] [node 6] ICS09 handshake done sender: [2:7587241471269246935:2098] self: [2:7587241471269246934:0] peer: [6:7587241470501558729:0] socket: 616 qp: -1
2025-12-24T02:11:55.848644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241471269246974:2093] [node 6] ICS10 traffic start
2025-12-24T02:11:55.848651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.851912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030814:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.852262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241471364030799:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.852269Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241471364030799:2115] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.853135Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241471364030799:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.853386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241471364030720:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.853403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.853409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241471364030720:2048] [node 2] ICP22 created new session: [1:7587241471364030817:2048]
2025-12-24T02:11:55.853415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241471364030817:2048] [node 2] ICS09 handshake done sender: [1:7587241471364030799:2115] self: [1:7587241471364030795:0] peer: [2:7587241471269246977:0] socket: 625 qp: -1
2025-12-24T02:11:55.853420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241471364030817:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:55.853428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.851169Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246976:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.852154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241471269246976:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.852176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241471269246877:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.853054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241471269246978:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44559
2025-12-24T02:11:55.853289Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241471269246976:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.853859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241471269246877:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.853872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.853877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241471269246877:2048] [node 1] ICP22 created new session: [2:7587241471269246979:2048]
2025-12-24T02:11:55.853882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241471269246979:2048] [node 1] ICS09 handshake done sender: [2:7587241471269246976:2119] self: [2:7587241471269246977:0] peer: [1:7587241471364030795:0] socket: 591 qp: -1
2025-12-24T02:11:55.853886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241471269246979:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.853896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.854795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496231:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.855536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241471269246922:2088] [node 4] ICP18 handshake (actor [2:7587241471269246960:2118]) is too old
2025-12-24T02:11:55.856433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241471791496227:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:55.856540Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241473961386587:2075] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.856617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241471791496137:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.856630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:55.856637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241471791496137:2050] [node 3] ICP22 created new session: [5:7587241471791496232:2050]
2025-12-24T02:11:55.856643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241471791496232:2050] [node 3] ICS09 handshake done sender: [5:7587241471791496227:2120] self: [5:7587241471791496230:0] peer: [3:7587241473961386628:0] socket: 613 qp: -1
2025-12-24T02:11:55.856648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241471791496232:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:55.856655Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:55.854875Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241470501558634:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:4014) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470501558720:2120] held: no
2025-12-24T02:11:55.854980Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558730:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.862515Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470582916352:2102] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:55.863126Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470501558720:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.866882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241470501558634:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.866905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.866916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241470501558634:2049] [node 2] ICP22 created new session: [6:7587241470501558731:2049]
2025-12-24T02:11:55.866923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470501558731:2049] [node 2] ICS09 handshake done sender: [6:7587241470501558720:2120] self: [6:7587241470501558729:0] peer: [2:7587241471269246934:0] socket: 622 qp: -1
2025-12-24T02:11:55.866932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470501558731:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.866944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.867306Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470582916367:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:55.867690Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241473961386619:2099] [node 5] ICP35 event processing took too much time PendingConnection:279(24267662ns) {HandleHandshakeStatus:329(24267379ns) {DropIncomingHandshake:442(4063ns) DropOutgoingHandshake:457(66ns) SwitchToState:179(139ns) ProcessPendingSessionEvents:495(24207352ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24185674ns max# 24105998ns}}}
2025-12-24T02:11:55.868172Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241470582916291:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241470582916352:2102] held: no
2025-12-24T02:11:55.868267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470582916291:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.868278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:55.868284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470582916291:2048] [node 1] ICP22 created new session: [4:7587241470582916383:2048]
2025-12-24T02:11:55.868290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470582916383:2048] [node 1] ICS09 handshake done sender: [4:7587241470582916352:2102] self: [4:7587241470582916371:0] peer: [1:7587241471364030784:0] socket: 574 qp: -1
2025-12-24T02:11:55.868297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470582916383:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:55.868302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.869592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241470582916292:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.869607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:55.869611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470582916292:2049] [node 2] ICP22 created new session: [4:7587241470582916385:2049]
2025-12-24T02:11:55.869616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470582916385:2049] [node 2] ICS09 handshake done sender: [4:7587241470582916367:2113] self: [4:7587241470582916370:0] peer: [2:7587241471269246929:0] socket: 582 qp: -1
2025-12-24T02:11:55.869621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470582916385:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:55.869627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:55.869764Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241470582916292:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:55.869974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241470582916382:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21515
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.870587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558733:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.871012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496234:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.870668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916382:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.875105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241470501558733:2123] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.875149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241470501558671:2078] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:55.878754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241471791496234:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.878791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241471791496174:2073] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.881187Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241471791496236:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:28984
2025-12-24T02:11:55.881567Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241471791496234:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.878876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916373:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.878886Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241470582916373:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.879992Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241470582916369:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:55.879996Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241470582916369:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:55.881674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241471791496174:2073] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.880528Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470582916373:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:55.881689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.881725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241471791496174:2073] [node 4] ICP22 created new session: [5:7587241471791496237:2073]
2025-12-24T02:11:55.881731Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241471791496237:2073] [node 4] ICS09 handshake done sender: [5:7587241471791496234:2122] self: [5:7587241471791496235:0] peer: [4:7587241470582916332:0] socket: 597 qp: -1
2025-12-24T02:11:55.881736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241471791496237:2073] [node 4] ICS10 traffic start
2025-12-24T02:11:55.881742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.880609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241470582916348:2095] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.880626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:55.880634Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470582916348:2095] [node 6] ICP22 created new session: [4:7587241470582916392:2095]
2025-12-24T02:11:55.880641Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470582916392:2095] [node 6] ICS09 handshake done sender: [4:7587241470582916373:2117] self: [4:7587241470582916366:0] peer: [6:7587241470501558734:0] socket: 577 qp: -1
2025-12-24T02:11:55.880648Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470582916392:2095] [node 6] ICS10 traffic start
2025-12-24T02:11:55.880659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:55.881431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241470582916369:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:55.882946Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241470501558671:2078] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:2900) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241470501558733:2123] held: no
2025-12-24T02:11:55.883813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241470582916328:2076] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:55.883830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:55.883836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241470582916328:2076] [node 5] ICP22 created new session: [4:7587241470582916394:2076]
2025-12-24T02:11:55.883842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241470582916394:2076] [node 5] ICS09 handshake done sender: [4:7587241470582916369:2115] self: [4:7587241470582916332:0] peer: [5:7587241471791496235:0] socket: 576 qp: -1
2025-12-24T02:11:55.883847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241470582916394:2076] [node 5] ICS10 traffic start
2025-12-24T02:11:55.883855Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:55.890861Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241470501558735:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36399
2025-12-24T02:11:55.891049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241470501558733:2123] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:55.891122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241470501558671:2078] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:55.891135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:55.891139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241470501558671:2078] [node 4] ICP22 created new session: [6:7587241470501558736:2078]
2025-12-24T02:11:55.891145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241470501558736:2078] [node 4] ICS09 handshake done sender: [6:7587241470501558733:2123] self: [6:7587241470501558734:0] peer: [4:7587241470582916366:0] socket: 594 qp: -1
2025-12-24T02:11:55.891150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241470501558736:2078] [node 4] ICS10 traffic start
2025-12-24T02:11:55.891155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:55.909691Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:55.909800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241471269246979:2048] [node 1] ICS07 socket disconnect 591 reason# EndOfStream
2025-12-24T02:11:55.909807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241471269246979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.909860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.909877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241471269246979:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.910794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:55.910865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241473961386631:2048] [node 1] ICS07 socket disconnect 569 reason# EndOfStream
2025-12-24T02:11:55.910871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241473961386631:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.910910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.910926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241473961386631:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.914230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241473961386552:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:55.914246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241473961386631:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.914511Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241473961386552:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.914520Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241473961386631:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.914527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241473961386552:2048] [node 1] ICP30 unregister session Session# [3:7587241473961386631:2048] VirtualId# [3:7587241473961386594:0]
2025-12-24T02:11:55.914557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.914564Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241473961386552:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.914567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.915460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241471269246877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:55.915474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241471269246979:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.915835Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241471269246877:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.915842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241471269246979:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.915847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241471269246877:2048] [node 1] ICP30 unregister session Session# [2:7587241471269246979:2048] VirtualId# [2:7587241471269246977:0]
2025-12-24T02:11:55.915898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.915903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241471269246877:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.915905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.926859Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:55.926975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470501558724:2048] [node 1] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:55.926984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241470501558633:2048] [node 1] ICP30 unregister session Session# [6:7587241470501558724:2048] VirtualId# [6:7587241470501558715:0]
2025-12-24T02:11:55.926993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470501558724:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.927042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.927068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.926989Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:55.927066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241471791496225:2048] [node 1] ICS07 socket disconnect 575 reason# EndOfStream
2025-12-24T02:11:55.927071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241471791496225:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.927108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.927128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241471791496225:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:55.927763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241471791496135:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:55.927773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241471791496225:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:55.927778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241471791496135:2048] [node 1] ICP30 unregister session Session# [5:7587241471791496225:2048] VirtualId# [5:7587241471791496202:0]
2025-12-24T02:11:55.927820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.927826Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241471791496135:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.927830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10330) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:55.927414Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:55.927479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470582916383:2048] [node 1] ICS01 socket: 574 reason# EndOfStream
2025-12-24T02:11:55.927486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241470582916291:2048] [node 1] ICP30 unregister session Session# [4:7587241470582916383:2048] VirtualId# [4:7587241470582916371:0]
2025-12-24T02:11:55.927491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470582916383:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.927527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:55.927550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:55.936814Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:55.936911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241473961386651:2049] [node 2] ICS01 socket: 568 reason# EndOfStream
2025-12-24T02:11:55.936922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241473961386553:2049] [node 2] ICP30 unregister session Session# [3:7587241473961386651:2049] VirtualId# [3:7587241473961386640:0]
2025-12-24T02:11:55.936930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241473961386651:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.936965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.936984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.937205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241473961386552:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:55.938780Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:55.938858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241470582916385:2049] [node 2] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:55.938866Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241470582916292:2049] [node 2] ICP30 unregister session Session# [4:7587241470582916385:2049] VirtualId# [4:7587241470582916370:0]
2025-12-24T02:11:55.938871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241470582916385:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.938910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.938934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.938768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241471791496135:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:55.938796Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:55.938862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241471791496220:2049] [node 2] ICS01 socket: 585 reason# EndOfStream
2025-12-24T02:11:55.938868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241471791496136:2049] [node 2] ICP30 unregister session Session# [5:7587241471791496220:2049] VirtualId# [5:7587241471791496217:0]
2025-12-24T02:11:55.938873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241471791496220:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.938907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.938929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.939036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:55.939108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470501558731:2049] [node 2] ICS01 socket: 622 reason# EndOfStream
2025-12-24T02:11:55.939112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241470501558634:2049] [node 2] ICP30 unregister session Session# [6:7587241470501558731:2049] VirtualId# [6:7587241470501558729:0]
2025-12-24T02:11:55.939117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470501558731:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.939143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:55.939161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:55.950915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:55.951013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470501558726:2050] [node 3] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:55.951024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241470501558635:2050] [node 3] ICP30 unregister session Session# [6:7587241470501558726:2050] VirtualId# [6:7587241470501558713:0]
2025-12-24T02:11:55.951031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470501558726:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.951082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.951106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.954756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:55.954844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241471791496232:2050] [node 3] ICS01 socket: 613 reason# EndOfStream
2025-12-24T02:11:55.954853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241471791496137:2050] [node 3] ICP30 unregister session Session# [5:7587241471791496232:2050] VirtualId# [5:7587241471791496230:0]
2025-12-24T02:11:55.954859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241471791496232:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.954922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:55.954948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:55.963981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:55.964094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241470501558736:2078] [node 4] ICS01 socket: 594 reason# EndOfStream
2025-12-24T02:11:55.964106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241470501558671:2078] [node 4] ICP30 unregister session Session# [6:7587241470501558736:2078] VirtualId# [6:7587241470501558734:0]
2025-12-24T02:11:55.964115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241470501558736:2078] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:55.964171Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:55.964197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:56.030490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477075682706:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.030506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477075682707:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.034880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475736963059:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:56.034896Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475736963060:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.042733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241474665902634:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.042752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241474665902635:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.042758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241474665902636:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.046733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474601682784:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.046760Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474601682785:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.050785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241475293979524:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.050801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241475293979525:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.050806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241475293979526:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.050781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475166270376:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.050792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475166270377:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.050801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475166270378:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.066535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475166270393:2061] [node 5] ICP01 ready to work
2025-12-24T02:11:56.067015Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475736963107:2092] [node 6] ICP01 ready to work
2025-12-24T02:11:56.079361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241475293979584:2082] [node 4] ICP01 ready to work
2025-12-24T02:11:56.079802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477075682770:2103] [node 6] ICP01 ready to work
2025-12-24T02:11:56.079897Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477075682771:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:56.082768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477075682773:2102] [node 4] ICP01 ready to work
2025-12-24T02:11:56.082847Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241475166270393:2061] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(16251574ns) {Configure:104(16251198ns) {ProcessConfigured:135(15981507ns) {StartInitialHandshake:157(15980885ns) {DropHandshakes:470(426ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(141ns) PrepareNewSessionHandshake:477(1116ns) {SwitchToState:179(562ns)}}}}}
2025-12-24T02:11:56.082933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475166270427:2066] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.083298Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241474601682785:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16387070ns) {Configure:104(16385858ns) {ProcessConfigured:135(15891537ns) {StartInitialHandshake:157(15891262ns) {DropHandshakes:470(461ns) {DropIncomingHandshake:442(88ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(218ns) PrepareNewSessionHandshake:477(956ns) {SwitchToState:179(481ns)}}}}}
2025-12-24T02:11:56.083729Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682836:2098] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.083184Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241477075682707:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16361517ns) {Configure:104(16360479ns) {ProcessConfigured:135(15934609ns) {StartInitialHandshake:157(15934352ns) {DropHandshakes:470(497ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(84ns)} GenerateSessionVirtualId:988(203ns) PrepareNewSessionHandshake:477(590ns) {SwitchToState:179(22ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.084036Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241475736963060:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16968903ns) {Configure:104(16967732ns) {ProcessConfigured:135(16209532ns) {StartInitialHandshake:157(16209210ns) {DropHandshakes:470(472ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(109ns) PrepareNewSessionHandshake:477(962ns) {SwitchToState:179(391ns)}}}}}
2025-12-24T02:11:56.088124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474601682850:2096] [node 5] ICP01 ready to work
2025-12-24T02:11:56.090713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475736963106:2088] [node 4] ICP01 ready to work
2025-12-24T02:11:56.087041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241474665902673:2083] [node 6] ICP01 ready to work
2025-12-24T02:11:56.091679Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241475293979526:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16333226ns) {Configure:104(16332750ns) {ProcessConfigured:135(15929366ns) {StartInitialHandshake:157(15928819ns) {DropHandshakes:470(432ns) {DropIncomingHandshake:442(30ns) DropOutgoingHandshake:457(34ns)} GenerateSessionVirtualId:988(474ns) PrepareNewSessionHandshake:477(549ns) {SwitchToState:179(86ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.096837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475736963123:2096] [node 5] ICP01 ready to work
2025-12-24T02:11:56.096935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474601682849:2100] [node 4] ICP01 ready to work
2025-12-24T02:11:56.095781Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979594:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.096114Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241475293979584:2082] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(16659283ns) {Configure:104(16658647ns) {ProcessConfigured:135(16379179ns) {StartInitialHandshake:157(16378799ns) {DropHandshakes:470(514ns) {DropIncomingHandshake:442(41ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(792ns) {SwitchToState:179(392ns)}}}}}
2025-12-24T02:11:56.098758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475736963129:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22639
2025-12-24T02:11:56.098948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474601682848:2091] [node 6] ICP01 ready to work
2025-12-24T02:11:56.099015Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682836:2098] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.099399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241474665902690:2089] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.107315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477075682782:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54650
2025-12-24T02:11:56.107352Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477075682780:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54650
2025-12-24T02:11:56.107378Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477075682781:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54650
2025-12-24T02:11:56.107655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902712:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.107688Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902710:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.107700Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902711:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.107906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241474665902636:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20617469ns) {Configure:104(20616857ns) {ProcessConfigured:135(20299540ns) {StartInitialHandshake:157(20299188ns) {DropHandshakes:470(489ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(42ns)} GenerateSessionVirtualId:988(454ns) PrepareNewSessionHandshake:477(602ns) {SwitchToState:179(178ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.114647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682865:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.114906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241477075682771:2107] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(32079348ns) {Configure:104(32078685ns) {ProcessConfigured:135(31747643ns) {StartInitialHandshake:157(31747388ns) {DropHandshakes:470(567ns) {DropIncomingHandshake:442(95ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(548ns) PrepareNewSessionHandshake:477(848ns) {SwitchToState:179(509ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.115917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902710:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.115968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241474665902635:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.115039Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241475166270377:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(24034209ns) {Configure:104(24033234ns) {ProcessConfigured:135(23664021ns) {StartInitialHandshake:157(23663745ns) {DropHandshakes:470(278ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(196ns) PrepareNewSessionHandshake:477(911ns) {SwitchToState:179(30ns)}}}}}
2025-12-24T02:11:56.115294Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682864:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.119121Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682780:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.119193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682777:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.119197Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477075682777:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.120096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477075682777:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.120195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477075682773:2102] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.120212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.120222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477075682773:2102] [node 4] ICP22 created new session: [2:7587241477075682789:2102]
2025-12-24T02:11:56.120230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477075682789:2102] [node 4] ICS09 handshake done sender: [2:7587241477075682777:2111] self: [2:7587241477075682776:0] peer: [4:7587241474665902714:0] socket: 568 qp: -1
2025-12-24T02:11:56.120238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477075682789:2102] [node 4] ICS10 traffic start
2025-12-24T02:11:56.120248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.118771Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241475736963106:2088] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(27939388ns) {Configure:104(27938816ns) {ProcessConfigured:135(27373721ns) {StartInitialHandshake:157(27373338ns) {DropHandshakes:470(350ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(191ns) PrepareNewSessionHandshake:477(1526ns) {SwitchToState:179(1129ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.122844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902715:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.123318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241475293979588:2089] [node 6] ICP01 ready to work
2025-12-24T02:11:56.123384Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979594:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.123568Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241474665902710:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.122802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241475166270376:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(31591383ns) {Configure:104(31590371ns) {ProcessConfigured:135(31312345ns) {StartInitialHandshake:157(31311966ns) {DropHandshakes:470(385ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(212ns) PrepareNewSessionHandshake:477(886ns) {SwitchToState:179(536ns)}}}}}
2025-12-24T02:11:56.126829Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682863:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.127076Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682863:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.127109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241474601682785:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.128380Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963129:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.128547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475736963137:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22639
2025-12-24T02:11:56.128587Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475736963136:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22639
2025-12-24T02:11:56.130042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682865:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.131013Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963137:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.131163Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241474601682850:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(34108172ns) {Configure:104(34106751ns) {ProcessConfigured:135(33720897ns) {StartInitialHandshake:157(33720348ns) {DropHandshakes:470(590ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(190ns)} GenerateSessionVirtualId:988(241ns) PrepareNewSessionHandshake:477(895ns) {SwitchToState:179(381ns)}}}}}
2025-12-24T02:11:56.131283Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241475736963107:2092] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(61375531ns) {Configure:104(61372388ns) {ProcessConfigured:135(61049227ns) {StartInitialHandshake:157(61048848ns) {DropHandshakes:470(427ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(45ns)} GenerateSessionVirtualId:988(254ns) PrepareNewSessionHandshake:477(423ns) {SwitchToState:179(97ns)}}}}}
2025-12-24T02:11:56.133701Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241474601682849:2100] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(34647844ns) {Configure:104(34647043ns) {ProcessConfigured:135(34290951ns) {StartInitialHandshake:157(34290532ns) {DropHandshakes:470(403ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(430ns) PrepareNewSessionHandshake:477(976ns) {SwitchToState:179(458ns)}}}}}
2025-12-24T02:11:56.136840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241474665902635:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.136864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.136874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241474665902635:2049] [node 2] ICP22 created new session: [4:7587241474665902716:2049]
2025-12-24T02:11:56.136881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241474665902716:2049] [node 2] ICS09 handshake done sender: [4:7587241474665902710:2113] self: [4:7587241474665902714:0] peer: [2:7587241477075682776:0] socket: 577 qp: -1
2025-12-24T02:11:56.136890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241474665902716:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.136909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.138799Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241474665902690:2089] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(39274758ns) {Configure:104(39274412ns) {ProcessConfigured:135(38984928ns) {StartInitialHandshake:157(38984594ns) {DropHandshakes:470(165ns) {DropIncomingHandshake:442(25ns) DropOutgoingHandshake:457(20ns)} GenerateSessionVirtualId:988(188ns) PrepareNewSessionHandshake:477(903ns) {SwitchToState:179(166ns)}}}}}
2025-12-24T02:11:56.138912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241475736963123:2096] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(19983647ns) {Configure:104(19983140ns) {ProcessConfigured:135(19599931ns) {StartInitialHandshake:157(19599630ns) {DropHandshakes:470(331ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(60ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(864ns) {SwitchToState:179(317ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.139150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682781:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.139150Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682775:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.139159Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477075682775:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.139181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241477075682706:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.140297Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682864:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.140332Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241474601682784:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.142969Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979603:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.143110Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902711:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.143187Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902712:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.143523Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963124:2107] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.143533Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475736963124:2107] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.146586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902718:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.146883Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902718:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.146909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241474665902634:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.143475Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241477075682770:2103] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(63519416ns) {Configure:104(63519094ns) {ProcessConfigured:135(63183358ns) {StartInitialHandshake:157(63183034ns) {DropHandshakes:470(348ns) {DropIncomingHandshake:442(16ns) DropOutgoingHandshake:457(29ns)} GenerateSessionVirtualId:988(186ns) PrepareNewSessionHandshake:477(631ns) {SwitchToState:179(285ns)}}}}}
2025-12-24T02:11:56.147175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963126:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.147188Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475736963126:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.147796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241475293979525:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(56701415ns) {Configure:104(56700799ns) {ProcessConfigured:135(56235611ns) {StartInitialHandshake:157(56235277ns) {DropHandshakes:470(349ns) {DropIncomingHandshake:442(48ns) DropOutgoingHandshake:457(49ns)} GenerateSessionVirtualId:988(190ns) PrepareNewSessionHandshake:477(1093ns) {SwitchToState:179(797ns)}}}}}
2025-12-24T02:11:56.149396Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979603:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.149443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241475293979525:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.150471Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682786:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.150481Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477075682786:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.151084Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475736963126:2108] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.155805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270454:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.155856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270455:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.155886Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270456:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.158842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477075682794:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54650
2025-12-24T02:11:56.156100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963134:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.158771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475166270454:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.156114Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475736963134:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.158804Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475166270455:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.156570Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963136:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.158840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475166270378:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.158964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477075682775:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.159080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475736963059:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.159103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.159112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475736963059:2048] [node 2] ICP22 created new session: [1:7587241475736963145:2048]
2025-12-24T02:11:56.159119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475736963145:2048] [node 2] ICS09 handshake done sender: [1:7587241475736963126:2108] self: [1:7587241475736963125:0] peer: [2:7587241477075682792:0] socket: 572 qp: -1
2025-12-24T02:11:56.159128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475736963145:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:56.159149Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.159127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475166270456:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.159161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475166270393:2061] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.163793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682860:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.163807Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474601682860:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.164840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.164854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.165493Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474601682860:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.165668Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682871:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.165677Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474601682848:2091] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.165694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.165700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474601682848:2091] [node 6] ICP22 created new session: [3:7587241474601682874:2091]
2025-12-24T02:11:56.165707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474601682874:2091] [node 6] ICS09 handshake done sender: [3:7587241474601682860:2113] self: [3:7587241474601682859:0] peer: [6:7587241475166270458:0] socket: 597 qp: -1
2025-12-24T02:11:56.165713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474601682874:2091] [node 6] ICS10 traffic start
2025-12-24T02:11:56.165734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.166965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475166270378:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15425) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241475166270455:2116] held: no
2025-12-24T02:11:56.167179Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979598:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.167191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241475293979598:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.167032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475736963124:2107] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.167056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477075682707:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.167077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.167083Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477075682707:2049] [node 3] ICP22 created new session: [2:7587241477075682796:2049]
2025-12-24T02:11:56.167089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477075682796:2049] [node 3] ICS09 handshake done sender: [2:7587241477075682775:2110] self: [2:7587241477075682761:0] peer: [3:7587241474601682866:0] socket: 567 qp: -1
2025-12-24T02:11:56.167096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477075682796:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.167103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.170791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474601682863:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.170931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902705:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.170946Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241474665902705:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.171117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475736963060:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.171136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.171144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475736963060:2049] [node 3] ICP22 created new session: [1:7587241475736963149:2049]
2025-12-24T02:11:56.171150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475736963149:2049] [node 3] ICS09 handshake done sender: [1:7587241475736963124:2107] self: [1:7587241475736963108:0] peer: [3:7587241474601682870:0] socket: 571 qp: -1
2025-12-24T02:11:56.171155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475736963149:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.171187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.174784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477075682781:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.176318Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241477075682706:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.176335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.176342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477075682706:2048] [node 1] ICP22 created new session: [2:7587241477075682799:2048]
2025-12-24T02:11:56.176349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477075682799:2048] [node 1] ICS09 handshake done sender: [2:7587241477075682781:2113] self: [2:7587241477075682792:0] peer: [1:7587241475736963125:0] socket: 592 qp: -1
2025-12-24T02:11:56.176355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477075682799:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.176361Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.176524Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241477075682706:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:56.176815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477075682786:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.176894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477075682771:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.176904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.176911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477075682771:2107] [node 5] ICP22 created new session: [2:7587241477075682802:2107]
2025-12-24T02:11:56.176916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477075682802:2107] [node 5] ICS09 handshake done sender: [2:7587241477075682786:2116] self: [2:7587241477075682774:0] peer: [5:7587241475293979605:0] socket: 601 qp: -1
2025-12-24T02:11:56.176921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477075682802:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:56.176926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.178473Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979606:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.178494Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979607:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.178557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475166270427:2066] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.178598Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241475166270427:2066] [node 4] ICP35 event processing took too much time PendingConnection:279(19749265ns) {IncomingHandshake:261(19748917ns) {DropIncomingHandshake:442(16ns) PrepareNewSessionHandshake:477(418ns) {SwitchToState:179(241ns)} GenerateSessionVirtualId:988(447ns) IssueIncomingHandshakeReply:193(19742532ns) {SwitchToState:179(170ns)}}}
2025-12-24T02:11:56.183389Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241475293979603:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.187150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241475293979598:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.187178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241475293979525:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.187193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.187202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241475293979525:2049] [node 2] ICP22 created new session: [5:7587241475293979609:2049]
2025-12-24T02:11:56.187210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241475293979609:2049] [node 2] ICS09 handshake done sender: [5:7587241475293979603:2115] self: [5:7587241475293979605:0] peer: [2:7587241477075682774:0] socket: 603 qp: -1
2025-12-24T02:11:56.187217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241475293979609:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.187237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.187479Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475736963134:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.187590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475736963106:2088] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.187606Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.187611Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475736963106:2088] [node 4] ICP22 created new session: [1:7587241475736963153:2088]
2025-12-24T02:11:56.187617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475736963153:2088] [node 4] ICS09 handshake done sender: [1:7587241475736963134:2113] self: [1:7587241475736963133:0] peer: [4:7587241474665902723:0] socket: 606 qp: -1
2025-12-24T02:11:56.187622Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475736963153:2088] [node 4] ICS10 traffic start
2025-12-24T02:11:56.187630Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.187583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241475293979588:2089] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.187604Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.187610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241475293979588:2089] [node 6] ICP22 created new session: [5:7587241475293979612:2089]
2025-12-24T02:11:56.187616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241475293979612:2089] [node 6] ICS09 handshake done sender: [5:7587241475293979598:2113] self: [5:7587241475293979597:0] peer: [6:7587241475166270459:0] socket: 604 qp: -1
2025-12-24T02:11:56.187621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241475293979612:2089] [node 6] ICS10 traffic start
2025-12-24T02:11:56.187627Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.189464Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979611:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.193403Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241474665902634:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241474665902718:2118] held: no
2025-12-24T02:11:56.191140Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475166270393:2061] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:5560) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241475166270456:2117] held: no
2025-12-24T02:11:56.192916Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241474601682785:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.192945Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.192957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474601682785:2049] [node 2] ICP22 created new session: [3:7587241474601682878:2049]
2025-12-24T02:11:56.192965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474601682878:2049] [node 2] ICS09 handshake done sender: [3:7587241474601682863:2114] self: [3:7587241474601682866:0] peer: [2:7587241477075682761:0] socket: 578 qp: -1
2025-12-24T02:11:56.192972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474601682878:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.192982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.193279Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475736963155:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22639
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.194867Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477075682804:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54650
2025-12-24T02:11:56.195142Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241474601682784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241474601682864:2115] held: no
2025-12-24T02:11:56.195779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241474665902705:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.195874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241474665902673:2083] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.195185Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241475736963059:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(36000523ns) {HandleHandshakeStatus:329(35999754ns) {DropIncomingHandshake:442(4242ns) DropOutgoingHandshake:457(336ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(24231402ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 24214058ns max# 24140220ns}}}
2025-12-24T02:11:56.199276Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979611:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.199324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241475293979524:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.195892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.195898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241474665902673:2083] [node 6] ICP22 created new session: [4:7587241474665902725:2083]
2025-12-24T02:11:56.195904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241474665902725:2083] [node 6] ICS09 handshake done sender: [4:7587241474665902705:2112] self: [4:7587241474665902704:0] peer: [6:7587241475166270457:0] socket: 586 qp: -1
2025-12-24T02:11:56.195909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241474665902725:2083] [node 6] ICS10 traffic start
2025-12-24T02:11:56.195916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.199912Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682804:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.199369Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241474601682785:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:56.199492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963155:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.202826Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979606:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.202887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241475293979526:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.203017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241477075682707:2049] [node 3] ICP35 event processing took too much time PendingConnection:279(35853807ns) {HandleHandshakeStatus:329(35853092ns) {DropIncomingHandshake:442(2317ns) DropOutgoingHandshake:457(118ns) SwitchToState:179(233ns) ProcessPendingSessionEvents:495(24225673ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 24201932ns max# 24124390ns}}}
2025-12-24T02:11:56.203809Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241475293979525:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16571400ns) {HandleHandshakeStatus:329(16570611ns) {DropIncomingHandshake:442(365ns) DropOutgoingHandshake:457(1800ns) SwitchToState:179(336ns) ProcessPendingSessionEvents:495(16465403ns) {unordered ForwardSessionEventToSession:612 num# 50 sum# 16453004ns max# 16408297ns}}}
2025-12-24T02:11:56.203146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682867:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.203161Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474601682867:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.206903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979616:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.207092Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475166270427:2066] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241475166270454:2115] held: no
2025-12-24T02:11:56.206915Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474601682880:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16700
2025-12-24T02:11:56.207094Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474601682864:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.207124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241474601682784:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.207138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.207144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474601682784:2048] [node 1] ICP22 created new session: [3:7587241474601682882:2048]
2025-12-24T02:11:56.207150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474601682882:2048] [node 1] ICS09 handshake done sender: [3:7587241474601682864:2115] self: [3:7587241474601682870:0] peer: [1:7587241475736963108:0] socket: 598 qp: -1
2025-12-24T02:11:56.207156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474601682882:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.207163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.207640Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.206970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963142:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.206983Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475736963142:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.207855Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241475736963106:2088] [node 4] ICP35 event processing took too much time PendingConnection:279(20156558ns) {HandleHandshakeStatus:329(20155944ns) {DropIncomingHandshake:442(4197ns) DropOutgoingHandshake:457(102ns) SwitchToState:179(109ns) ProcessPendingSessionEvents:495(8582139ns) {unordered ForwardSessionEventToSession:612 num# 90 sum# 8559490ns max# 8491770ns}}}
2025-12-24T02:11:56.208018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475736963142:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.208272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475736963123:2096] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.208291Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.208297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475736963123:2096] [node 5] ICP22 created new session: [1:7587241475736963158:2096]
2025-12-24T02:11:56.208302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475736963158:2096] [node 5] ICS09 handshake done sender: [1:7587241475736963142:2117] self: [1:7587241475736963135:0] peer: [5:7587241475293979615:0] socket: 620 qp: -1
2025-12-24T02:11:56.208308Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475736963158:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:56.208315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.208419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241475293979611:2119] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.210890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270462:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.211012Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902728:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.211524Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474601682867:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.211656Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474601682850:2096] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.211677Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.211684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474601682850:2096] [node 5] ICP22 created new session: [3:7587241474601682885:2096]
2025-12-24T02:11:56.211689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474601682885:2096] [node 5] ICS09 handshake done sender: [3:7587241474601682867:2117] self: [3:7587241474601682857:0] peer: [5:7587241475293979617:0] socket: 565 qp: -1
2025-12-24T02:11:56.211695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474601682885:2096] [node 5] ICS10 traffic start
2025-12-24T02:11:56.211702Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.208793Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241475293979524:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.208807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.208813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241475293979524:2048] [node 1] ICP22 created new session: [5:7587241475293979619:2048]
2025-12-24T02:11:56.208819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241475293979619:2048] [node 1] ICS09 handshake done sender: [5:7587241475293979611:2119] self: [5:7587241475293979615:0] peer: [1:7587241475736963135:0] socket: 622 qp: -1
2025-12-24T02:11:56.208824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241475293979619:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.208834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.208961Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241475293979524:2048] [node 1] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.214881Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241474665902718:2118] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.216001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270465:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.216848Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270466:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.216289Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902729:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.217096Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270463:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.217140Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475166270455:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.217248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902729:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.217265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241474665902634:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.217362Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475166270456:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.217277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.217284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241474665902634:2048] [node 1] ICP22 created new session: [4:7587241474665902730:2048]
2025-12-24T02:11:56.217290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241474665902730:2048] [node 1] ICS09 handshake done sender: [4:7587241474665902718:2118] self: [4:7587241474665902723:0] peer: [1:7587241475736963133:0] socket: 607 qp: -1
2025-12-24T02:11:56.217296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241474665902730:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.217303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.217517Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241474665902636:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.217502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475166270463:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.217516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475166270378:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.217526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.217531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475166270378:2050] [node 3] ICP22 created new session: [6:7587241475166270467:2050]
2025-12-24T02:11:56.217536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475166270467:2050] [node 3] ICS09 handshake done sender: [6:7587241475166270455:2116] self: [6:7587241475166270458:0] peer: [3:7587241474601682859:0] socket: 612 qp: -1
2025-12-24T02:11:56.217541Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475166270467:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.217547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.217691Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475166270454:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.216126Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475736963160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22639
2025-12-24T02:11:56.218306Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963160:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.218334Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475736963141:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.218338Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475736963141:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.217965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475166270393:2061] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.217975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.217979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475166270393:2061] [node 5] ICP22 created new session: [6:7587241475166270469:2061]
2025-12-24T02:11:56.217984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475166270469:2061] [node 5] ICS09 handshake done sender: [6:7587241475166270456:2117] self: [6:7587241475166270459:0] peer: [5:7587241475293979597:0] socket: 613 qp: -1
2025-12-24T02:11:56.217989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475166270469:2061] [node 5] ICS10 traffic start
2025-12-24T02:11:56.217993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.218155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475166270376:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.219265Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475736963141:2116] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.219310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475166270427:2066] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.219323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.219329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475166270427:2066] [node 4] ICP22 created new session: [6:7587241475166270473:2066]
2025-12-24T02:11:56.219334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475166270473:2066] [node 4] ICS09 handshake done sender: [6:7587241475166270454:2115] self: [6:7587241475166270457:0] peer: [4:7587241474665902704:0] socket: 589 qp: -1
2025-12-24T02:11:56.219383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475736963107:2092] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.219399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.219404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475736963107:2092] [node 6] ICP22 created new session: [1:7587241475736963163:2092]
2025-12-24T02:11:56.219410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475736963163:2092] [node 6] ICS09 handshake done sender: [1:7587241475736963141:2116] self: [1:7587241475736963109:0] peer: [6:7587241475166270471:0] socket: 617 qp: -1
2025-12-24T02:11:56.219415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475736963163:2092] [node 6] ICS10 traffic start
2025-12-24T02:11:56.219420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.219338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475166270473:2066] [node 4] ICS10 traffic start
2025-12-24T02:11:56.219344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.219562Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270472:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.220689Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475166270376:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241475166270463:2119] held: no
2025-12-24T02:11:56.221052Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475166270463:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.221133Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475166270376:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.221144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.221150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475166270376:2048] [node 1] ICP22 created new session: [6:7587241475166270475:2048]
2025-12-24T02:11:56.221155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475166270475:2048] [node 1] ICS09 handshake done sender: [6:7587241475166270463:2119] self: [6:7587241475166270471:0] peer: [1:7587241475736963109:0] socket: 582 qp: -1
2025-12-24T02:11:56.221160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475166270475:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.221166Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.221887Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241474665902719:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.221895Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241474665902719:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.222649Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241474665902719:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.222725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241474665902690:2089] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.222739Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.222748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241474665902690:2089] [node 5] ICP22 created new session: [4:7587241474665902734:2089]
2025-12-24T02:11:56.222754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241474665902734:2089] [node 5] ICS09 handshake done sender: [4:7587241474665902719:2119] self: [4:7587241474665902709:0] peer: [5:7587241475293979621:0] socket: 625 qp: -1
2025-12-24T02:11:56.222758Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241474665902734:2089] [node 5] ICS10 traffic start
2025-12-24T02:11:56.222763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.221743Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241475293979616:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.221783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241475293979584:2082] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.223680Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241475293979584:2082] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22125) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241475293979616:2120] held: no
2025-12-24T02:11:56.226974Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979622:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.227184Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241475293979623:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47125
2025-12-24T02:11:56.230621Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270464:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.231061Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241475293979526:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15425) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241475293979606:2117] held: no
2025-12-24T02:11:56.231183Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241475293979616:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.235031Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475166270464:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.235081Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475166270377:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.238252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477075682793:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.238272Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477075682793:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.235165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474601682869:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.235186Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474601682869:2118] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.238460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474601682869:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.238566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474601682849:2100] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.238583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.238591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474601682849:2100] [node 4] ICP22 created new session: [3:7587241474601682888:2100]
2025-12-24T02:11:56.238598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474601682888:2100] [node 4] ICS09 handshake done sender: [3:7587241474601682869:2118] self: [3:7587241474601682858:0] peer: [4:7587241474665902732:0] socket: 626 qp: -1
2025-12-24T02:11:56.238605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474601682888:2100] [node 4] ICS10 traffic start
2025-12-24T02:11:56.238613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.238968Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241474601682850:2096] [node 5] ICP35 event processing took too much time PendingConnection:279(27208382ns) {HandleHandshakeStatus:329(27207797ns) {DropIncomingHandshake:442(3704ns) DropOutgoingHandshake:457(127ns) SwitchToState:179(286ns) ProcessPendingSessionEvents:495(27123639ns) {unordered ForwardSessionEventToSession:612 num# 80 sum# 27099607ns max# 27033260ns}}}
2025-12-24T02:11:56.243542Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477075682793:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.243645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477075682770:2103] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.243662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.243670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477075682770:2103] [node 6] ICP22 created new session: [2:7587241477075682807:2103]
2025-12-24T02:11:56.243676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477075682807:2103] [node 6] ICS09 handshake done sender: [2:7587241477075682793:2117] self: [2:7587241477075682772:0] peer: [6:7587241475166270477:0] socket: 575 qp: -1
2025-12-24T02:11:56.243681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477075682807:2103] [node 6] ICS10 traffic start
2025-12-24T02:11:56.243688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.242988Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241475293979606:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.247436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241475293979584:2082] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.247455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.247463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241475293979584:2082] [node 4] ICP22 created new session: [5:7587241475293979624:2082]
2025-12-24T02:11:56.247471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241475293979624:2082] [node 4] ICS09 handshake done sender: [5:7587241475293979616:2120] self: [5:7587241475293979621:0] peer: [4:7587241474665902709:0] socket: 627 qp: -1
2025-12-24T02:11:56.247479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241475293979624:2082] [node 4] ICS10 traffic start
2025-12-24T02:11:56.247488Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.247773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241475293979526:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.247780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.247786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241475293979526:2050] [node 3] ICP22 created new session: [5:7587241475293979626:2050]
2025-12-24T02:11:56.247790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241475293979626:2050] [node 3] ICS09 handshake done sender: [5:7587241475293979606:2117] self: [5:7587241475293979617:0] peer: [3:7587241474601682857:0] socket: 605 qp: -1
2025-12-24T02:11:56.247795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241475293979626:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.247802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.248383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241474665902736:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27990
2025-12-24T02:11:56.250924Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241474665902729:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.251064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241474665902636:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.251083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.251091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241474665902636:2050] [node 3] ICP22 created new session: [4:7587241474665902737:2050]
2025-12-24T02:11:56.251113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241474665902737:2050] [node 3] ICS09 handshake done sender: [4:7587241474665902729:2121] self: [4:7587241474665902732:0] peer: [3:7587241474601682858:0] socket: 630 qp: -1
2025-12-24T02:11:56.251119Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241474665902737:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.251127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.254523Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475166270377:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241475166270464:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.258891Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241474665902636:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:56.258085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475166270478:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:27887
2025-12-24T02:11:56.260987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475166270464:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.261072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475166270377:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.261086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.261096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475166270377:2049] [node 2] ICP22 created new session: [6:7587241475166270479:2049]
2025-12-24T02:11:56.261104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475166270479:2049] [node 2] ICS09 handshake done sender: [6:7587241475166270464:2120] self: [6:7587241475166270477:0] peer: [2:7587241477075682772:0] socket: 584 qp: -1
2025-12-24T02:11:56.261111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475166270479:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.261134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.261943Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241477075682770:2103] [node 6] ICP35 event processing took too much time PendingConnection:279(18177188ns) {HandleHandshakeStatus:329(18176562ns) {DropIncomingHandshake:442(3658ns) DropOutgoingHandshake:457(124ns) SwitchToState:179(77ns) ProcessPendingSessionEvents:495(18105897ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 18077106ns max# 17990734ns}}}
2025-12-24T02:11:56.285434Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:56.285528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241477075682799:2048] [node 1] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:56.285538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241477075682706:2048] [node 1] ICP30 unregister session Session# [2:7587241477075682799:2048] VirtualId# [2:7587241477075682792:0]
2025-12-24T02:11:56.285546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241477075682799:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.285582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.285603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.286071Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:56.286133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241474601682882:2048] [node 1] ICS07 socket disconnect 598 reason# EndOfStream
2025-12-24T02:11:56.286138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241474601682882:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.286161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.286176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241474601682882:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:56.288447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241474601682784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Socket error# Connection reset by peer state# ReceiveResponse processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:56.288461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241474601682882:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:56.288727Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241474601682784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.288739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241474601682882:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.288748Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241474601682784:2048] [node 1] ICP30 unregister session Session# [3:7587241474601682882:2048] VirtualId# [3:7587241474601682870:0]
2025-12-24T02:11:56.288774Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.288780Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241474601682784:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.288782Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.297915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:56.298008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241474665902730:2048] [node 1] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:56.298017Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241474665902634:2048] [node 1] ICP30 unregister session Session# [4:7587241474665902730:2048] VirtualId# [4:7587241474665902723:0]
2025-12-24T02:11:56.298025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241474665902730:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.298075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.298100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.298755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:56.298868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475166270475:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:56.298876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475166270376:2048] [node 1] ICP30 unregister session Session# [6:7587241475166270475:2048] VirtualId# [6:7587241475166270471:0]
2025-12-24T02:11:56.298883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475166270475:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.298994Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:56.299068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241475293979619:2048] [node 1] ICS07 socket disconnect 622 reason# EndOfStream
2025-12-24T02:11:56.299074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241475293979619:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.299124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.299147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241475293979619:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:56.299569Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241475293979524:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.299581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241475293979619:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.299586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241475293979524:2048] [node 1] ICP30 unregister session Session# [5:7587241475293979619:2048] VirtualId# [5:7587241475293979615:0]
2025-12-24T02:11:56.299607Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.299613Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241475293979524:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.299617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:28504) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.300454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241474601682784:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:56.298935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.298964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.310067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241475293979524:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:56.311317Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:56.311411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241474601682878:2049] [node 2] ICS01 socket: 578 reason# EndOfStream
2025-12-24T02:11:56.311420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241474601682785:2049] [node 2] ICP30 unregister session Session# [3:7587241474601682878:2049] VirtualId# [3:7587241474601682866:0]
2025-12-24T02:11:56.311427Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241474601682878:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.311460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.311479Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.324325Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:56.324432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241474665902716:2049] [node 2] ICS07 socket disconnect 577 reason# EndOfStream
2025-12-24T02:11:56.324440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241474665902716:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.324494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.324519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241474665902716:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:56.324753Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:56.324828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475166270479:2049] [node 2] ICS01 socket: 584 reason# EndOfStream
2025-12-24T02:11:56.324834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475166270377:2049] [node 2] ICP30 unregister session Session# [6:7587241475166270479:2049] VirtualId# [6:7587241475166270477:0]
2025-12-24T02:11:56.324841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475166270479:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.324871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.324896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.325423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241474665902635:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.325436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241474665902716:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.325440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241474665902635:2049] [node 2] ICP30 unregister session Session# [4:7587241474665902716:2049] VirtualId# [4:7587241474665902714:0]
2025-12-24T02:11:56.325471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.325477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241474665902635:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.325480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.326813Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:56.326903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241475293979609:2049] [node 2] ICS07 socket disconnect 603 reason# EndOfStream
2025-12-24T02:11:56.326910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241475293979609:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.326963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.326985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241475293979609:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:56.327348Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241475293979525:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.327359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241475293979609:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.327365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241475293979525:2049] [node 2] ICP30 unregister session Session# [5:7587241475293979609:2049] VirtualId# [5:7587241475293979605:0]
2025-12-24T02:11:56.327416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.327421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241475293979525:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.327423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31445) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.334762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241474665902635:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:56.339444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241475293979525:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:56.342761Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:56.342866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475166270467:2050] [node 3] ICS01 socket: 612 reason# EndOfStream
2025-12-24T02:11:56.342876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475166270378:2050] [node 3] ICP30 unregister session Session# [6:7587241475166270467:2050] VirtualId# [6:7587241475166270458:0]
2025-12-24T02:11:56.342884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475166270467:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.342935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:56.342959Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:56.342992Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:56.343060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241475293979626:2050] [node 3] ICS01 socket: 605 reason# EndOfStream
2025-12-24T02:11:56.343065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241475293979526:2050] [node 3] ICP30 unregister session Session# [5:7587241475293979626:2050] VirtualId# [5:7587241475293979617:0]
2025-12-24T02:11:56.343070Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241475293979626:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.343096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:56.343117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:56.366783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:56.366895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241475293979624:2082] [node 4] ICS01 socket: 627 reason# EndOfStream
2025-12-24T02:11:56.366906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241475293979584:2082] [node 4] ICP30 unregister session Session# [5:7587241475293979624:2082] VirtualId# [5:7587241475293979621:0]
2025-12-24T02:11:56.366915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241475293979624:2082] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.366968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:56.366995Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:56.367033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:56.367116Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475166270473:2066] [node 4] ICS01 socket: 589 reason# EndOfStream
2025-12-24T02:11:56.367123Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475166270427:2066] [node 4] ICP30 unregister session Session# [6:7587241475166270473:2066] VirtualId# [6:7587241475166270457:0]
2025-12-24T02:11:56.367128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475166270473:2066] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.367153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:56.367172Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:56.430778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477536707588:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.430819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477536707589:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.434826Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475706237027:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:56.434841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475706237028:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.434810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478365005004:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.434821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478365005005:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.441776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477946734677:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.441786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477946734678:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.441792Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477946734679:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.438933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241478175855302:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.438941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241478175855303:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.438945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241478175855304:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.450813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241476084091870:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.450825Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241476084091871:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.450830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241476084091872:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.497821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475706237091:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.498307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855317:2060] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.498383Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855318:2061] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.494795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478365005065:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:56.495937Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005066:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.499626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477946734729:2098] [node 6] ICP01 ready to work
2025-12-24T02:11:56.499043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477536707650:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:56.499717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477536707655:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:56.500138Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477536707654:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42237
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.501771Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475706237098:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31236
2025-12-24T02:11:56.501897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475706237099:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31236
2025-12-24T02:11:56.498937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241478175855322:2064] [node 4] ICP01 ready to work
2025-12-24T02:11:56.499761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241478175855335:2071] [node 6] ICP01 ready to work
2025-12-24T02:11:56.510914Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477536707653:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42237
2025-12-24T02:11:56.511040Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005072:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.515082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707653:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.515158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475706237101:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:56.515701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237099:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.516029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477946734738:2095] [node 5] ICP01 ready to work
2025-12-24T02:11:56.514773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241476084091889:2063] [node 4] ICP01 ready to work
2025-12-24T02:11:56.514966Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005066:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.515226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478365005076:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:56.518946Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855317:2060] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.519003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241478175855304:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.518915Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237098:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.518824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241476084091893:2066] [node 5] ICP01 ready to work
2025-12-24T02:11:56.523438Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477946734729:2098] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(23728401ns) {Configure:104(23727999ns) {ProcessConfigured:135(23304072ns) {StartInitialHandshake:157(23303720ns) {DropHandshakes:470(318ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(62ns)} GenerateSessionVirtualId:988(80ns) PrepareNewSessionHandshake:477(1242ns) {SwitchToState:179(496ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.527392Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734746:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.527449Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734747:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.528509Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734746:2111] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.528540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734747:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.527134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005079:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.530371Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241477536707661:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:56.531160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478365005081:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:56.532519Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241475706237107:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:56.534831Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477946734738:2095] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18713456ns) {Configure:104(18713022ns) {ProcessConfigured:135(18411474ns) {StartInitialHandshake:157(18411149ns) {DropHandshakes:470(230ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(592ns) {SwitchToState:179(324ns)}}}}}
2025-12-24T02:11:56.535173Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241478365005076:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19896374ns) {Configure:104(19895493ns) {ProcessConfigured:135(19423353ns) {StartInitialHandshake:157(19423185ns) {DropHandshakes:470(334ns) {DropIncomingHandshake:442(45ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(110ns) PrepareNewSessionHandshake:477(774ns) {SwitchToState:179(24ns)}}}}}
2025-12-24T02:11:56.539559Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241476084091893:2066] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20598134ns) {Configure:104(20597618ns) {ProcessConfigured:135(20228179ns) {StartInitialHandshake:157(20227558ns) {DropHandshakes:470(450ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(226ns) PrepareNewSessionHandshake:477(1288ns) {SwitchToState:179(693ns)}}}}}
2025-12-24T02:11:56.539785Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241478365005005:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20725875ns) {Configure:104(20725460ns) {ProcessConfigured:135(20355720ns) {StartInitialHandshake:157(20355576ns) {DropHandshakes:470(645ns) {DropIncomingHandshake:442(18ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(245ns) PrepareNewSessionHandshake:477(760ns) {SwitchToState:179(54ns)}}}}}
2025-12-24T02:11:56.542819Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241477536707589:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(23408078ns) {Configure:104(23407525ns) {ProcessConfigured:135(23055892ns) {StartInitialHandshake:157(23055457ns) {DropHandshakes:470(368ns) {DropIncomingHandshake:442(20ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(202ns) PrepareNewSessionHandshake:477(987ns) {SwitchToState:179(636ns)}}}}}
2025-12-24T02:11:56.543080Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707654:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.543105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241477536707588:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.543194Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237088:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.543204Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475706237088:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.546431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855318:2061] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.546786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005068:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.546815Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478365005068:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.550828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005072:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.550939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241478365005004:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.554061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091946:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.550795Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855384:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.550859Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855385:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.550898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855386:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.554753Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855385:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.554832Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005079:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.554989Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855384:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.554752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237096:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.554763Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475706237096:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.555347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475706237096:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.555445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475706237028:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.555470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.555478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475706237028:2049] [node 3] ICP22 created new session: [1:7587241475706237112:2049]
2025-12-24T02:11:56.555485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475706237112:2049] [node 3] ICS09 handshake done sender: [1:7587241475706237096:2110] self: [1:7587241475706237095:0] peer: [3:7587241478365005086:0] socket: 581 qp: -1
2025-12-24T02:11:56.555493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475706237112:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.555502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.558463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241478365005004:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241478365005072:2111] held: no
2025-12-24T02:11:56.558970Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241476084091946:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.558854Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241478365005081:2114] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(27629986ns) {Configure:104(27629447ns) {ProcessConfigured:135(27255145ns) {StartInitialHandshake:157(27254753ns) {DropHandshakes:470(483ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(74ns)} GenerateSessionVirtualId:988(114ns) PrepareNewSessionHandshake:477(841ns) {SwitchToState:179(257ns)}}}}}
2025-12-24T02:11:56.558786Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241477536707661:2113] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(28295833ns) {Configure:104(28295349ns) {ProcessConfigured:135(27839146ns) {StartInitialHandshake:157(27820994ns) {DropHandshakes:470(445ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(470ns) PrepareNewSessionHandshake:477(902ns) {SwitchToState:179(43ns)}} IncomingHandshake:261(4870ns) {DropIncomingHandshake:442(136ns) PrepareNewSessionHandshake:477(204ns) {SwitchToState:179(170ns)} GenerateSessionVirtualId:988(210ns) IssueIncomingHandshakeReply:193(266ns)}}}}
2025-12-24T02:11:56.558845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707652:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.558851Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477536707652:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.562712Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707657:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.562731Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477536707657:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.562789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855386:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.562860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477946734678:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.562913Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477946734678:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(34311703ns) {Configure:104(34311348ns) {ProcessConfigured:135(33932687ns) {IncomingHandshake:261(33921476ns) {DropIncomingHandshake:442(66ns) PrepareNewSessionHandshake:477(451ns) {SwitchToState:179(96ns)} GenerateSessionVirtualId:988(1649ns) IssueIncomingHandshakeReply:193(33914911ns) {SwitchToState:179(423ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.566807Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475706237114:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31236
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.567338Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237114:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.566769Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734749:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.566778Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241477946734749:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.567069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477946734679:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(32232474ns) {Configure:104(32231213ns) {ProcessConfigured:135(31891614ns) {StartInitialHandshake:157(31890835ns) {DropHandshakes:470(373ns) {DropIncomingHandshake:442(44ns) DropOutgoingHandshake:457(61ns)} GenerateSessionVirtualId:988(208ns) PrepareNewSessionHandshake:477(691ns) {SwitchToState:179(513ns)}}}}}
2025-12-24T02:11:56.567161Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734755:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.568440Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734744:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.568445Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241477946734744:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.569112Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477946734744:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.569181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241477946734729:2098] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.569193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.569200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477946734729:2098] [node 6] ICP22 created new session: [4:7587241477946734758:2098]
2025-12-24T02:11:56.569207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477946734758:2098] [node 6] ICS09 handshake done sender: [4:7587241477946734744:2110] self: [4:7587241477946734730:0] peer: [6:7587241476084091948:0] socket: 596 qp: -1
2025-12-24T02:11:56.569213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477946734758:2098] [node 6] ICS10 traffic start
2025-12-24T02:11:56.569231Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.567322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241476084091889:2063] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(52419438ns) {Configure:104(52418576ns) {ProcessConfigured:135(52087912ns) {StartInitialHandshake:157(52087680ns) {DropHandshakes:470(412ns) {DropIncomingHandshake:442(71ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(934ns) {SwitchToState:179(661ns)}}}}}
2025-12-24T02:11:56.567382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241476084091889:2063] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.570810Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477946734677:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(35934890ns) {Configure:104(35934281ns) {ProcessConfigured:135(35654064ns) {StartInitialHandshake:157(35653497ns) {DropHandshakes:470(231ns) {DropIncomingHandshake:442(29ns) DropOutgoingHandshake:457(48ns)} GenerateSessionVirtualId:988(207ns) PrepareNewSessionHandshake:477(511ns) {SwitchToState:179(187ns)}}}}}
2025-12-24T02:11:56.570943Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734755:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.570970Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477946734679:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.574873Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855390:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.576096Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241478175855303:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.576138Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241478175855303:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21086264ns) {IncomingHandshake:261(21085982ns) {DropIncomingHandshake:442(180ns) PrepareNewSessionHandshake:477(501ns) {SwitchToState:179(253ns)} GenerateSessionVirtualId:988(216ns) IssueIncomingHandshakeReply:193(21081392ns) {SwitchToState:179(480ns)}}}
2025-12-24T02:11:56.575408Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477536707672:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42237
2025-12-24T02:11:56.578868Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478365005068:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.578845Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475706237116:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31236
2025-12-24T02:11:56.578742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241478175855302:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.578776Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241478175855302:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(32277132ns) {IncomingHandshake:261(32276872ns) {DropIncomingHandshake:442(49ns) PrepareNewSessionHandshake:477(396ns) {SwitchToState:179(93ns)} GenerateSessionVirtualId:988(685ns) IssueIncomingHandshakeReply:193(32271849ns) {SwitchToState:179(407ns)}}}
2025-12-24T02:11:56.578996Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475706237088:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.579054Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241478175855317:2060] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.579072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477536707652:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.579174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475706237027:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.579203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.579209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475706237027:2048] [node 2] ICP22 created new session: [1:7587241475706237118:2048]
2025-12-24T02:11:56.579215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475706237118:2048] [node 2] ICS09 handshake done sender: [1:7587241475706237088:2107] self: [1:7587241475706237087:0] peer: [2:7587241477536707668:0] socket: 574 qp: -1
2025-12-24T02:11:56.579232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475706237118:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:56.579240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.580294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241478175855304:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.580314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.580321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241478175855304:2050] [node 3] ICP22 created new session: [5:7587241478175855391:2050]
2025-12-24T02:11:56.580327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241478175855391:2050] [node 3] ICS09 handshake done sender: [5:7587241478175855317:2060] self: [5:7587241478175855380:0] peer: [3:7587241478365005067:0] socket: 579 qp: -1
2025-12-24T02:11:56.580334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241478175855391:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.580354Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.583069Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241476084091870:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(60202625ns) {Configure:104(60201940ns) {ProcessConfigured:135(59899618ns) {StartInitialHandshake:157(59898996ns) {DropHandshakes:470(344ns) {DropIncomingHandshake:442(47ns) DropOutgoingHandshake:457(51ns)} GenerateSessionVirtualId:988(265ns) PrepareNewSessionHandshake:477(1207ns) {SwitchToState:179(81ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.586897Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005092:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.590856Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477536707657:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.590822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005083:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.590845Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478365005083:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.591201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237093:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.591219Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475706237093:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.594791Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477946734749:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.594603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241478175855322:2064] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.594655Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241478175855322:2064] [node 4] ICP35 event processing took too much time PendingConnection:279(31795115ns) {IncomingHandshake:261(31794701ns) {DropIncomingHandshake:442(52ns) PrepareNewSessionHandshake:477(403ns) {SwitchToState:179(133ns)} GenerateSessionVirtualId:988(684ns) IssueIncomingHandshakeReply:193(31790202ns) {SwitchToState:179(998ns)}}}
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.606890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855393:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.607002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855394:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.607408Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241478175855386:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.607620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241478175855384:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.607679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241478175855322:2064] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.607700Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.607706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241478175855322:2064] [node 4] ICP22 created new session: [5:7587241478175855395:2064]
2025-12-24T02:11:56.607714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241478175855395:2064] [node 4] ICS09 handshake done sender: [5:7587241478175855386:2119] self: [5:7587241478175855389:0] peer: [4:7587241477946734742:0] socket: 609 qp: -1
2025-12-24T02:11:56.607724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241478175855395:2064] [node 4] ICS10 traffic start
2025-12-24T02:11:56.607734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.608843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241478175855303:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.608856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.608863Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241478175855303:2049] [node 2] ICP22 created new session: [5:7587241478175855397:2049]
2025-12-24T02:11:56.608869Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241478175855397:2049] [node 2] ICS09 handshake done sender: [5:7587241478175855384:2117] self: [5:7587241478175855388:0] peer: [2:7587241477536707651:0] socket: 594 qp: -1
2025-12-24T02:11:56.608874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241478175855397:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.608896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.612421Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241477536707588:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241477536707654:2111] held: no
2025-12-24T02:11:56.612595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707672:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.612603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477536707655:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.612611Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.612616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477536707655:2107] [node 4] ICP22 created new session: [2:7587241477536707676:2107]
2025-12-24T02:11:56.612623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477536707676:2107] [node 4] ICS09 handshake done sender: [2:7587241477536707657:2112] self: [2:7587241477536707656:0] peer: [4:7587241477946734748:0] socket: 590 qp: -1
2025-12-24T02:11:56.612628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477536707676:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:56.612644Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.614246Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734764:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.611019Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241475706237027:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(31722541ns) {HandleHandshakeStatus:329(31721928ns) {DropIncomingHandshake:442(13269ns) DropOutgoingHandshake:457(758ns) SwitchToState:179(75ns) ProcessPendingSessionEvents:495(31621808ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31603197ns max# 31519501ns}}}
2025-12-24T02:11:56.611105Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237116:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.613978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005093:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.614063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478365005072:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.612976Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241477536707675:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:42237
2025-12-24T02:11:56.614304Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734765:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.614392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477536707654:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.614752Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091950:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.614609Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477536707650:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.614633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.614643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477536707650:2108] [node 5] ICP22 created new session: [2:7587241477536707680:2108]
2025-12-24T02:11:56.614651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477536707680:2108] [node 5] ICS09 handshake done sender: [2:7587241477536707652:2109] self: [2:7587241477536707651:0] peer: [5:7587241478175855388:0] socket: 589 qp: -1
2025-12-24T02:11:56.614658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477536707680:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:56.614666Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.614609Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005094:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.614804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478365005065:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.614825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.614832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478365005065:2107] [node 5] ICP22 created new session: [3:7587241478365005095:2107]
2025-12-24T02:11:56.614839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478365005095:2107] [node 5] ICS09 handshake done sender: [3:7587241478365005068:2109] self: [3:7587241478365005067:0] peer: [5:7587241478175855380:0] socket: 577 qp: -1
2025-12-24T02:11:56.614845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478365005095:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:56.614864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.615174Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005093:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.615028Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241477536707588:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.615043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.615051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477536707588:2048] [node 1] ICP22 created new session: [2:7587241477536707682:2048]
2025-12-24T02:11:56.615056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477536707682:2048] [node 1] ICS09 handshake done sender: [2:7587241477536707654:2111] self: [2:7587241477536707668:0] peer: [1:7587241475706237087:0] socket: 588 qp: -1
2025-12-24T02:11:56.615060Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477536707682:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.615067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.617186Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241478365005004:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.617202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.617207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478365005004:2048] [node 1] ICP22 created new session: [3:7587241478365005099:2048]
2025-12-24T02:11:56.617213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478365005099:2048] [node 1] ICS09 handshake done sender: [3:7587241478365005072:2111] self: [3:7587241478365005086:0] peer: [1:7587241475706237095:0] socket: 582 qp: -1
2025-12-24T02:11:56.617220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478365005099:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.617226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.614644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477946734764:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.614673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477946734677:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.615349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477946734747:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.615455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477946734678:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.615465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.615470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477946734678:2049] [node 2] ICP22 created new session: [4:7587241477946734767:2049]
2025-12-24T02:11:56.615475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477946734767:2049] [node 2] ICS09 handshake done sender: [4:7587241477946734747:2112] self: [4:7587241477946734748:0] peer: [2:7587241477536707656:0] socket: 601 qp: -1
2025-12-24T02:11:56.615480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477946734767:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.615498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.622530Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475706237093:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.622619Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241476084091950:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.622683Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241476084091870:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.619629Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241478175855304:2050] [node 3] ICP35 event processing took too much time PendingConnection:279(39228108ns) {HandleHandshakeStatus:329(39227024ns) {DropIncomingHandshake:442(192ns) DropOutgoingHandshake:457(2139ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(28541354ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28516789ns max# 28434914ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.622771Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478365005083:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.622878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478365005076:2112] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.622901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.622912Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478365005076:2112] [node 4] ICP22 created new session: [3:7587241478365005101:2112]
2025-12-24T02:11:56.622919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478365005101:2112] [node 4] ICS09 handshake done sender: [3:7587241478365005083:2115] self: [3:7587241478365005077:0] peer: [4:7587241477946734762:0] socket: 604 qp: -1
2025-12-24T02:11:56.622925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478365005101:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:56.622931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.622804Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241478175855400:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:22370
2025-12-24T02:11:56.623237Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241478175855318:2061] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.623333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241478175855302:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.623348Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.623354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241478175855302:2048] [node 1] ICP22 created new session: [5:7587241478175855401:2048]
2025-12-24T02:11:56.623360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241478175855401:2048] [node 1] ICS09 handshake done sender: [5:7587241478175855318:2061] self: [5:7587241478175855383:0] peer: [1:7587241475706237092:0] socket: 580 qp: -1
2025-12-24T02:11:56.623366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241478175855401:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.623383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.623420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475706237091:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.623449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.623456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475706237091:2108] [node 5] ICP22 created new session: [1:7587241475706237123:2108]
2025-12-24T02:11:56.623461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475706237123:2108] [node 5] ICS09 handshake done sender: [1:7587241475706237093:2109] self: [1:7587241475706237092:0] peer: [5:7587241478175855383:0] socket: 578 qp: -1
2025-12-24T02:11:56.623466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475706237123:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:56.623474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.623581Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241477946734679:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11584) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241477946734755:2115] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.634956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091952:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.635124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091953:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.635180Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091954:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.635217Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091955:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.639240Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237109:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.639257Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475706237109:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.639407Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237103:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.639421Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241475706237103:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.639279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241477946734738:2095] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.639320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.639330Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477946734738:2095] [node 5] ICP22 created new session: [4:7587241477946734769:2095]
2025-12-24T02:11:56.639340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477946734769:2095] [node 5] ICS09 handshake done sender: [4:7587241477946734749:2113] self: [4:7587241477946734742:0] peer: [5:7587241478175855389:0] socket: 606 qp: -1
2025-12-24T02:11:56.639349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477946734769:2095] [node 5] ICS10 traffic start
2025-12-24T02:11:56.639380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.639007Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241476084091955:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.639039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241476084091954:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.639066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241476084091946:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.639136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241476084091893:2066] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.639548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241476084091953:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.639574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241476084091872:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.639646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241476084091889:2063] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.639657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.639662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241476084091889:2063] [node 4] ICP22 created new session: [6:7587241476084091959:2063]
2025-12-24T02:11:56.639667Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241476084091959:2063] [node 4] ICS09 handshake done sender: [6:7587241476084091946:2113] self: [6:7587241476084091948:0] peer: [4:7587241477946734730:0] socket: 598 qp: -1
2025-12-24T02:11:56.639673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241476084091959:2063] [node 4] ICS10 traffic start
2025-12-24T02:11:56.639705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.643154Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241477536707650:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(60259819ns) {HandleHandshakeStatus:329(60259130ns) {DropIncomingHandshake:442(589ns) DropOutgoingHandshake:457(369ns) SwitchToState:179(78ns) ProcessPendingSessionEvents:495(153160ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 129104ns max# 40744ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.646924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005094:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.646986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241478365005005:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.646824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734771:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.651114Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477946734755:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.651209Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477946734679:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.651229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.651238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477946734679:2050] [node 3] ICP22 created new session: [4:7587241477946734772:2050]
2025-12-24T02:11:56.651245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477946734772:2050] [node 3] ICS09 handshake done sender: [4:7587241477946734755:2115] self: [4:7587241477946734762:0] peer: [3:7587241478365005077:0] socket: 607 qp: -1
2025-12-24T02:11:56.651254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477946734772:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.651264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.655017Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241475706237091:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(31467436ns) {HandleHandshakeStatus:329(31466515ns) {DropIncomingHandshake:442(12169ns) DropOutgoingHandshake:457(106ns) SwitchToState:179(68ns) ProcessPendingSessionEvents:495(31372651ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 31336192ns max# 31236545ns}}}
2025-12-24T02:11:56.659110Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241478365005065:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(76168741ns) {HandleHandshakeStatus:329(76167504ns) {DropIncomingHandshake:442(6891ns) DropOutgoingHandshake:457(132ns) SwitchToState:179(41ns) ProcessPendingSessionEvents:495(19734173ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 19702742ns max# 19598521ns}}}
2025-12-24T02:11:56.662754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707669:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.662780Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477536707669:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.662824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241477536707667:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.662838Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241477536707667:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.662870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241476084091871:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.662912Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241476084091871:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23782591ns) {IncomingHandshake:261(23781379ns) {DropIncomingHandshake:442(257ns) PrepareNewSessionHandshake:477(1203ns) {SwitchToState:179(310ns)} GenerateSessionVirtualId:988(360ns) IssueIncomingHandshakeReply:193(23774509ns) {SwitchToState:179(1052ns)}}}
2025-12-24T02:11:56.662880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478365005088:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.662896Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478365005088:2117] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.662817Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241478175855337:2073] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.662839Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241478175855337:2073] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.663248Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241477946734738:2095] [node 5] ICP35 event processing took too much time PendingConnection:279(68223037ns) {HandleHandshakeStatus:329(68222196ns) {DropIncomingHandshake:442(12607ns) DropOutgoingHandshake:457(220ns) SwitchToState:179(112ns) ProcessPendingSessionEvents:495(143375ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 122999ns max# 28153ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.667005Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241478175855302:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(43552822ns) {HandleHandshakeStatus:329(43552315ns) {DropIncomingHandshake:442(148ns) DropOutgoingHandshake:457(1392ns) SwitchToState:179(95ns) ProcessPendingSessionEvents:495(27381585ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27355556ns max# 27259747ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.666911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475706237103:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.668557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241475706237126:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31236
2025-12-24T02:11:56.665260Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091962:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.668890Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241476084091889:2063] [node 4] ICP35 event processing took too much time PendingConnection:279(29111567ns) {HandleHandshakeStatus:329(29110871ns) {DropIncomingHandshake:442(106ns) DropOutgoingHandshake:457(1184ns) SwitchToState:179(50ns) ProcessPendingSessionEvents:495(8909109ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8884332ns max# 8800136ns}}}
2025-12-24T02:11:56.669307Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241476084091950:2116] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.669808Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477946734774:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16442
2025-12-24T02:11:56.669349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241476084091870:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.669370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.669377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241476084091870:2048] [node 1] ICP22 created new session: [6:7587241476084091963:2048]
2025-12-24T02:11:56.669383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241476084091963:2048] [node 1] ICS09 handshake done sender: [6:7587241476084091950:2116] self: [6:7587241476084091951:0] peer: [1:7587241475706237102:0] socket: 612 qp: -1
2025-12-24T02:11:56.669389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241476084091963:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.669394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.672465Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477536707669:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.672507Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478365005104:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:16429
2025-12-24T02:11:56.672633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241477536707667:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.672640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477536707661:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.672661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.672667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477536707661:2113] [node 6] ICP22 created new session: [2:7587241477536707686:2113]
2025-12-24T02:11:56.672675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477536707686:2113] [node 6] ICS09 handshake done sender: [2:7587241477536707669:2116] self: [2:7587241477536707663:0] peer: [6:7587241476084091956:0] socket: 586 qp: -1
2025-12-24T02:11:56.672681Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477536707686:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:56.672689Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.671811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241475706237126:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.671880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241475706237109:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.672225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475706237101:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.672239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.672247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475706237101:2113] [node 6] ICP22 created new session: [1:7587241475706237129:2113]
2025-12-24T02:11:56.672252Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475706237129:2113] [node 6] ICS09 handshake done sender: [1:7587241475706237103:2114] self: [1:7587241475706237102:0] peer: [6:7587241476084091951:0] socket: 592 qp: -1
2025-12-24T02:11:56.672258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475706237129:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:56.672266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.672791Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241475706237107:2115] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.672806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.672812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241475706237107:2115] [node 4] ICP22 created new session: [1:7587241475706237131:2115]
2025-12-24T02:11:56.672817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241475706237131:2115] [node 4] ICS09 handshake done sender: [1:7587241475706237109:2116] self: [1:7587241475706237108:0] peer: [4:7587241477946734766:0] socket: 623 qp: -1
2025-12-24T02:11:56.672822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241475706237131:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:56.672828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.672956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241477536707589:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.672972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.672978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241477536707589:2049] [node 3] ICP22 created new session: [2:7587241477536707688:2049]
2025-12-24T02:11:56.672983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241477536707688:2049] [node 3] ICS09 handshake done sender: [2:7587241477536707667:2115] self: [2:7587241477536707662:0] peer: [3:7587241478365005103:0] socket: 625 qp: -1
2025-12-24T02:11:56.673279Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478365005094:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.672988Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241477536707688:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.672996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.673919Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241478365005005:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241478365005094:2120] held: no
2025-12-24T02:11:56.674149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241478365005005:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.674162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.674167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478365005005:2049] [node 2] ICP22 created new session: [3:7587241478365005105:2049]
2025-12-24T02:11:56.674174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478365005105:2049] [node 2] ICS09 handshake done sender: [3:7587241478365005094:2120] self: [3:7587241478365005103:0] peer: [2:7587241477536707662:0] socket: 630 qp: -1
2025-12-24T02:11:56.674180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478365005105:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.674188Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.676591Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241477946734677:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241477946734764:2117] held: no
2025-12-24T02:11:56.676755Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477946734764:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.676860Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241476084091871:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64932) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241476084091955:2120] held: no
2025-12-24T02:11:56.677098Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091965:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.677407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091966:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.677422Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241476084091967:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52585
2025-12-24T02:11:56.677434Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241476084091955:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.677567Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477946734677:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.677587Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.677594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477946734677:2048] [node 1] ICP22 created new session: [4:7587241477946734775:2048]
2025-12-24T02:11:56.677600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477946734775:2048] [node 1] ICS09 handshake done sender: [4:7587241477946734764:2117] self: [4:7587241477946734766:0] peer: [1:7587241475706237108:0] socket: 626 qp: -1
2025-12-24T02:11:56.677607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477946734775:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.677617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.677490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241476084091871:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.677505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.677512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241476084091871:2049] [node 2] ICP22 created new session: [6:7587241476084091968:2049]
2025-12-24T02:11:56.677519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241476084091968:2049] [node 2] ICS09 handshake done sender: [6:7587241476084091955:2120] self: [6:7587241476084091956:0] peer: [2:7587241477536707663:0] socket: 593 qp: -1
2025-12-24T02:11:56.677524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241476084091968:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.677531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.678048Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478365005088:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.678773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478365005081:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.678802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.678808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478365005081:2114] [node 6] ICP22 created new session: [3:7587241478365005108:2114]
2025-12-24T02:11:56.678814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478365005108:2114] [node 6] ICS09 handshake done sender: [3:7587241478365005088:2117] self: [3:7587241478365005082:0] peer: [6:7587241476084091958:0] socket: 627 qp: -1
2025-12-24T02:11:56.678820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478365005108:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:56.678828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.678933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241478175855337:2073] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.679071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241476084091953:2118] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.683047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241478175855335:2071] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.683068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.683074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241478175855335:2071] [node 6] ICP22 created new session: [5:7587241478175855404:2071]
2025-12-24T02:11:56.683081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241478175855404:2071] [node 6] ICS09 handshake done sender: [5:7587241478175855337:2073] self: [5:7587241478175855336:0] peer: [6:7587241476084091957:0] socket: 633 qp: -1
2025-12-24T02:11:56.683086Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241478175855404:2071] [node 6] ICS10 traffic start
2025-12-24T02:11:56.683098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.683004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241476084091872:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.683021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.683032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241476084091872:2050] [node 3] ICP22 created new session: [6:7587241476084091970:2050]
2025-12-24T02:11:56.683041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241476084091970:2050] [node 3] ICS09 handshake done sender: [6:7587241476084091953:2118] self: [6:7587241476084091958:0] peer: [3:7587241478365005082:0] socket: 571 qp: -1
2025-12-24T02:11:56.683048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241476084091970:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.683054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.683432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241476084091954:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.685673Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241476084091893:2066] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.685687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.685693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241476084091893:2066] [node 5] ICP22 created new session: [6:7587241476084091972:2066]
2025-12-24T02:11:56.685699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241476084091972:2066] [node 5] ICS09 handshake done sender: [6:7587241476084091954:2119] self: [6:7587241476084091957:0] peer: [5:7587241478175855336:0] socket: 585 qp: -1
2025-12-24T02:11:56.685706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241476084091972:2066] [node 5] ICS10 traffic start
2025-12-24T02:11:56.685712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.708218Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:56.708326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241478365005099:2048] [node 1] ICS01 socket: 582 reason# EndOfStream
2025-12-24T02:11:56.708337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241478365005004:2048] [node 1] ICP30 unregister session Session# [3:7587241478365005099:2048] VirtualId# [3:7587241478365005086:0]
2025-12-24T02:11:56.708346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241478365005099:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.708383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.708408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.710760Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:56.710848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241477536707682:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:56.710855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241477536707588:2048] [node 1] ICP30 unregister session Session# [2:7587241477536707682:2048] VirtualId# [2:7587241477536707668:0]
2025-12-24T02:11:56.710862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241477536707682:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.710911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.710939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.722832Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:56.722938Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241477946734775:2048] [node 1] ICS01 socket: 626 reason# EndOfStream
2025-12-24T02:11:56.722948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241477946734677:2048] [node 1] ICP30 unregister session Session# [4:7587241477946734775:2048] VirtualId# [4:7587241477946734766:0]
2025-12-24T02:11:56.722956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241477946734775:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.723007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.723037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.723277Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:56.723354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241476084091963:2048] [node 1] ICS07 socket disconnect 612 reason# EndOfStream
2025-12-24T02:11:56.723360Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241476084091963:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.723389Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.723411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241476084091963:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:56.722875Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:56.722967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241478175855401:2048] [node 1] ICS07 socket disconnect 580 reason# EndOfStream
2025-12-24T02:11:56.722974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241478175855401:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.723022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:56.723043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241478175855401:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:56.724137Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241478175855302:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.724147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241478175855401:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.724151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241478175855302:2048] [node 1] ICP30 unregister session Session# [5:7587241478175855401:2048] VirtualId# [5:7587241478175855383:0]
2025-12-24T02:11:56.724196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.724200Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241478175855302:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.724202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.723884Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241476084091870:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.723896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241476084091963:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.723902Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241476084091870:2048] [node 1] ICP30 unregister session Session# [6:7587241476084091963:2048] VirtualId# [6:7587241476084091951:0]
2025-12-24T02:11:56.723966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:56.723973Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241476084091870:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.723976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:1146) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.734757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241478175855302:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:56.734818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241476084091870:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:56.754571Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:56.754677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241478365005105:2049] [node 2] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:56.754688Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241478365005005:2049] [node 2] ICP30 unregister session Session# [3:7587241478365005105:2049] VirtualId# [3:7587241478365005103:0]
2025-12-24T02:11:56.754768Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241478365005105:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.754807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.754828Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.761710Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:56.761816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241477946734767:2049] [node 2] ICS01 socket: 601 reason# EndOfStream
2025-12-24T02:11:56.761827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241477946734678:2049] [node 2] ICP30 unregister session Session# [4:7587241477946734767:2049] VirtualId# [4:7587241477946734748:0]
2025-12-24T02:11:56.761834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241477946734767:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.761897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.761923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.762735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:56.762815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241478175855397:2049] [node 2] ICS01 socket: 594 reason# EndOfStream
2025-12-24T02:11:56.762822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241478175855303:2049] [node 2] ICP30 unregister session Session# [5:7587241478175855397:2049] VirtualId# [5:7587241478175855388:0]
2025-12-24T02:11:56.762831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241478175855397:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.762874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.762898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.770749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:56.770841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241476084091968:2049] [node 2] ICS01 socket: 593 reason# EndOfStream
2025-12-24T02:11:56.770850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241476084091871:2049] [node 2] ICP30 unregister session Session# [6:7587241476084091968:2049] VirtualId# [6:7587241476084091956:0]
2025-12-24T02:11:56.770856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241476084091968:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.770911Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:56.770936Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:56.778781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:56.778894Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241478175855391:2050] [node 3] ICS01 socket: 579 reason# EndOfStream
2025-12-24T02:11:56.778904Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241478175855304:2050] [node 3] ICP30 unregister session Session# [5:7587241478175855391:2050] VirtualId# [5:7587241478175855380:0]
2025-12-24T02:11:56.778913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241478175855391:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.779030Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:56.779052Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:56.780196Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:56.780471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241477946734772:2050] [node 3] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:56.780479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241477946734679:2050] [node 3] ICP30 unregister session Session# [4:7587241477946734772:2050] VirtualId# [4:7587241477946734762:0]
2025-12-24T02:11:56.780486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241477946734772:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.780537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:56.780564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:56.781166Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:56.781301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241476084091970:2050] [node 3] ICS01 socket: 571 reason# EndOfStream
2025-12-24T02:11:56.781309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241476084091872:2050] [node 3] ICP30 unregister session Session# [6:7587241476084091970:2050] VirtualId# [6:7587241476084091958:0]
2025-12-24T02:11:56.781316Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241476084091970:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.781357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:56.781382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:56.794598Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:56.794725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241476084091959:2063] [node 4] ICS07 socket disconnect 598 reason# EndOfStream
2025-12-24T02:11:56.794733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241476084091959:2063] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.794802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:56.794837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241476084091959:2063] [node 4] ICS15 start handshake
2025-12-24T02:11:56.794744Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:56.794835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241478175855395:2064] [node 4] ICS01 socket: 609 reason# EndOfStream
2025-12-24T02:11:56.794844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241478175855322:2064] [node 4] ICP30 unregister session Session# [5:7587241478175855395:2064] VirtualId# [5:7587241478175855389:0]
2025-12-24T02:11:56.794852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241478175855395:2064] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.794903Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:56.794929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:56.795268Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241476084091889:2063] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14912) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:56.795291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241476084091959:2063] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:56.795298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241476084091889:2063] [node 4] ICP30 unregister session Session# [6:7587241476084091959:2063] VirtualId# [6:7587241476084091948:0]
2025-12-24T02:11:56.795362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:56.795377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241476084091889:2063] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:14912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.795380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14912) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:56.805348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241476084091889:2063] [node 4] ICP33 wake up from error state
2025-12-24T02:11:56.816537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:56.816670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241476084091972:2066] [node 5] ICS01 socket: 585 reason# EndOfStream
2025-12-24T02:11:56.816687Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241476084091893:2066] [node 5] ICP30 unregister session Session# [6:7587241476084091972:2066] VirtualId# [6:7587241476084091957:0]
2025-12-24T02:11:56.816697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241476084091972:2066] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:56.816763Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:56.816796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:56.843724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478010506188:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:56.843738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478010506189:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.845119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241475904316384:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.845138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241475904316385:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:56.848538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477702889322:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.848551Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477702889323:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.848556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477702889324:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.847026Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474227829267:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.847033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474227829268:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.854801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241477545169700:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.854818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241477545169701:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.854824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241477545169702:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.866717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478010506252:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:56.866890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478010506254:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:56.858853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475776146888:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:56.858864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475776146889:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:56.858871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475776146890:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:56.867063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477702889381:2102] [node 6] ICP01 ready to work
2025-12-24T02:11:56.867043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478010506255:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.868775Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316443:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.868778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241477545169761:2106] [node 6] ICP01 ready to work
2025-12-24T02:11:56.867095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474227829328:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:56.869954Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146946:2105] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.870900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475776146949:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:56.883362Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241474227829328:2107] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16156541ns) {Configure:104(16155793ns) {ProcessConfigured:135(15733080ns) {StartInitialHandshake:157(15732088ns) {DropHandshakes:470(470ns) {DropIncomingHandshake:442(54ns) DropOutgoingHandshake:457(81ns)} GenerateSessionVirtualId:988(86ns) PrepareNewSessionHandshake:477(5349ns) {SwitchToState:179(1490ns)}}}}}
2025-12-24T02:11:56.888352Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241475904316385:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16862972ns) {Configure:104(16861901ns) {ProcessConfigured:135(16547017ns) {StartInitialHandshake:157(16545978ns) {DropHandshakes:470(297ns) {DropIncomingHandshake:442(27ns) DropOutgoingHandshake:457(79ns)} GenerateSessionVirtualId:988(221ns) PrepareNewSessionHandshake:477(2388ns) {SwitchToState:179(1197ns)}}}}}
2025-12-24T02:11:56.888654Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241475904316452:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:56.888879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241475904316453:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:56.889033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241475904316454:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.891352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241477545169771:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:56.891582Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169770:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:56.894120Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241475776146889:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(17996026ns) {Configure:104(17993961ns) {ProcessConfigured:135(17567190ns) {StartInitialHandshake:157(17566458ns) {DropHandshakes:470(725ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(370ns) PrepareNewSessionHandshake:477(1156ns) {SwitchToState:179(312ns)}}}}}
2025-12-24T02:11:56.894507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241475776146958:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.898883Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241477702889324:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(31730601ns) {Configure:104(31729889ns) {ProcessConfigured:135(31248309ns) {StartInitialHandshake:157(31247496ns) {DropHandshakes:470(458ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(122ns)} GenerateSessionVirtualId:988(225ns) PrepareNewSessionHandshake:477(2181ns) {SwitchToState:179(789ns)}}}}}
2025-12-24T02:11:56.900189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241477702889393:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:56.900343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889392:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.910962Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241475776146888:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16176192ns) {Configure:104(16175397ns) {ProcessConfigured:135(15806157ns) {StartInitialHandshake:157(15805507ns) {DropHandshakes:470(534ns) {DropIncomingHandshake:442(59ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(247ns) PrepareNewSessionHandshake:477(2638ns) {SwitchToState:179(1732ns)}}}}}
2025-12-24T02:11:56.911849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475776146946:2105] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.912122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475776146949:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.912158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146966:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.914932Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241475904316454:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(24346401ns) {Configure:104(24345635ns) {ProcessConfigured:135(24095053ns) {StartInitialHandshake:157(24094503ns) {DropHandshakes:470(360ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(53ns)} GenerateSessionVirtualId:988(234ns) PrepareNewSessionHandshake:477(2197ns) {SwitchToState:179(1222ns)}}}}}
2025-12-24T02:11:56.915035Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316443:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.915360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316463:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
2025-12-24T02:11:56.915386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316464:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
2025-12-24T02:11:56.915410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241475904316384:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.921718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506250:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.921734Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478010506250:2108] [node 2] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.922181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478010506268:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36706
2025-12-24T02:11:56.922203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478010506269:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.922218Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478010506270:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.922321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478010506267:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36706
2025-12-24T02:11:56.922496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478010506250:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.922670Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478010506188:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.922713Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.922728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478010506188:2048] [node 2] ICP22 created new session: [1:7587241478010506272:2048]
2025-12-24T02:11:56.922739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478010506272:2048] [node 2] ICS09 handshake done sender: [1:7587241478010506250:2108] self: [1:7587241478010506248:0] peer: [2:7587241475904316466:0] socket: 576 qp: -1
2025-12-24T02:11:56.922746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478010506272:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:56.922757Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.923232Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889392:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.923306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889401:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
2025-12-24T02:11:56.923326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477702889322:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.924930Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316467:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
2025-12-24T02:11:56.924956Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316468:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
2025-12-24T02:11:56.926479Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316463:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.919433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474227829337:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:56.922791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169770:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.919606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241474227829338:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:56.923278Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169763:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.919761Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829333:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
2025-12-24T02:11:56.919839Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829334:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
2025-12-24T02:11:56.923289Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241477545169763:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.919874Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829335:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
2025-12-24T02:11:56.924344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241477545169763:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.919892Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829336:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
2025-12-24T02:11:56.924484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241477545169761:2106] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.924502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.924512Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241477545169761:2106] [node 6] ICP22 created new session: [5:7587241477545169781:2106]
2025-12-24T02:11:56.924519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241477545169781:2106] [node 6] ICS09 handshake done sender: [5:7587241477545169763:2108] self: [5:7587241477545169762:0] peer: [6:7587241475776146967:0] socket: 580 qp: -1
2025-12-24T02:11:56.924526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241477545169781:2106] [node 6] ICS10 traffic start
2025-12-24T02:11:56.924536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.924684Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146969:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.924710Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146970:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.926980Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475776146966:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.927071Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475776146946:2105] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.930293Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241475904316443:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.926772Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829333:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.926910Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829334:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.926972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241474227829267:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.930485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475776146949:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.930514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.930523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475776146949:2107] [node 5] ICP22 created new session: [6:7587241475776146971:2107]
2025-12-24T02:11:56.930531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475776146971:2107] [node 5] ICS09 handshake done sender: [6:7587241475776146946:2105] self: [6:7587241475776146967:0] peer: [5:7587241477545169762:0] socket: 583 qp: -1
2025-12-24T02:11:56.930539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475776146971:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:56.930553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.935835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475776146958:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.942884Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889391:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.942903Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241477702889391:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.951075Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241478010506188:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(28257873ns) {HandleHandshakeStatus:329(28256156ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(594ns) SwitchToState:179(244ns) ProcessPendingSessionEvents:495(27663878ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 27632979ns max# 27508335ns}}}
2025-12-24T02:11:56.951218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506259:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.951231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478010506259:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.951809Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506267:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.951974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506257:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.951985Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478010506257:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.952292Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316464:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.952418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241475904316384:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.952439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.952449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241475904316384:2048] [node 1] ICP22 created new session: [2:7587241475904316469:2048]
2025-12-24T02:11:56.952460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241475904316469:2048] [node 1] ICS09 handshake done sender: [2:7587241475904316443:2106] self: [2:7587241475904316466:0] peer: [1:7587241478010506248:0] socket: 577 qp: -1
2025-12-24T02:11:56.952470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241475904316469:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.952480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.953543Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889401:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:56.953614Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477702889391:2112] [node 6] ICH04 handshake succeeded
::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.953673Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146975:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.953794Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241475904316471:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38772
2025-12-24T02:11:56.953973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241477702889381:2102] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.953994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.954000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477702889381:2102] [node 6] ICP22 created new session: [4:7587241477702889404:2102]
2025-12-24T02:11:56.954005Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477702889404:2102] [node 6] ICS09 handshake done sender: [4:7587241477702889391:2112] self: [4:7587241477702889390:0] peer: [6:7587241475776146973:0] socket: 594 qp: -1
2025-12-24T02:11:56.954011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477702889404:2102] [node 6] ICS10 traffic start
2025-12-24T02:11:56.954020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.954152Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475776146966:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.954312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477702889323:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.954230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475776146958:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.954245Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.954250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475776146958:2111] [node 4] ICP22 created new session: [6:7587241475776146976:2111]
2025-12-24T02:11:56.954255Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475776146976:2111] [node 4] ICS09 handshake done sender: [6:7587241475776146966:2116] self: [6:7587241475776146973:0] peer: [4:7587241477702889390:0] socket: 596 qp: -1
2025-12-24T02:11:56.954261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475776146976:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:56.954267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.953889Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169784:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.954527Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316471:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.955874Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316456:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.955879Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241475904316456:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.957123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506268:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.957976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241478010506188:2048] [node 2] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.958300Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169785:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.955213Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829335:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.955357Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829352:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
2025-12-24T02:11:56.958899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829336:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.959032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241474227829268:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.959111Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829354:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.963115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316467:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.963164Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316451:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.963168Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241475904316451:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.964213Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241475904316456:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.964448Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506270:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.964511Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478010506277:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36706
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.964871Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889408:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
2025-12-24T02:11:56.964683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169784:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.964716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241477545169701:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.963248Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475776146969:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.963349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475776146890:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.967732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478010506259:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:56.967940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316458:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.967951Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241475904316458:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.968600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241475904316452:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.968624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.968633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241475904316452:2109] [node 4] ICP22 created new session: [2:7587241475904316479:2109]
2025-12-24T02:11:56.968640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241475904316479:2109] [node 4] ICS09 handshake done sender: [2:7587241475904316456:2113] self: [2:7587241475904316455:0] peer: [4:7587241477702889406:0] socket: 603 qp: -1
2025-12-24T02:11:56.968649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241475904316479:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:56.968661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.969536Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241475904316451:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.969813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241475904316385:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.969833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.969839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241475904316385:2049] [node 3] ICP22 created new session: [2:7587241475904316483:2049]
2025-12-24T02:11:56.969844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241475904316483:2049] [node 3] ICS09 handshake done sender: [2:7587241475904316451:2112] self: [2:7587241475904316449:0] peer: [3:7587241474227829355:0] socket: 602 qp: -1
2025-12-24T02:11:56.969849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241475904316483:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.969857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.969685Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169787:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:56.966810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829332:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.966829Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474227829332:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.967492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829352:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.967535Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241474227829356:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18453
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.970895Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146979:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.970918Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146980:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.974053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146981:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.971693Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478010506257:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.971823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478010506252:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.971835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:56.971842Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478010506252:2107] [node 4] ICP22 created new session: [1:7587241478010506281:2107]
2025-12-24T02:11:56.971850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478010506281:2107] [node 4] ICS09 handshake done sender: [1:7587241478010506259:2112] self: [1:7587241478010506258:0] peer: [4:7587241477702889402:0] socket: 582 qp: -1
2025-12-24T02:11:56.971855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478010506281:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:56.971865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:56.972108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478010506189:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.972120Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.972124Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478010506189:2049] [node 3] ICP22 created new session: [1:7587241478010506283:2049]
2025-12-24T02:11:56.972128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478010506283:2049] [node 3] ICS09 handshake done sender: [1:7587241478010506257:2111] self: [1:7587241478010506256:0] peer: [3:7587241474227829350:0] socket: 578 qp: -1
2025-12-24T02:11:56.972131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478010506283:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:56.972136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.975448Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241475904316458:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.974857Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241474227829268:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241474227829336:2114] held: no
2025-12-24T02:11:56.979855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241475904316453:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.979884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.979894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241475904316453:2110] [node 5] ICP22 created new session: [2:7587241475904316485:2110]
2025-12-24T02:11:56.979902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241475904316485:2110] [node 5] ICS09 handshake done sender: [2:7587241475904316458:2114] self: [2:7587241475904316457:0] peer: [5:7587241477545169786:0] socket: 624 qp: -1
2025-12-24T02:11:56.979912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241475904316485:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:56.979924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.980229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169785:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.980253Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241477545169784:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.980069Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241477702889323:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:29847) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241477702889401:2116] held: no
2025-12-24T02:11:56.980320Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477702889401:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.980290Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474227829332:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.980532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477702889323:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.980555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.980561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477702889323:2049] [node 2] ICP22 created new session: [4:7587241477702889410:2049]
2025-12-24T02:11:56.980568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477702889410:2049] [node 2] ICS09 handshake done sender: [4:7587241477702889401:2116] self: [4:7587241477702889406:0] peer: [2:7587241475904316455:0] socket: 604 qp: -1
2025-12-24T02:11:56.980575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477702889410:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.980584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.980880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241477545169701:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.980902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.980908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241477545169701:2049] [node 2] ICP22 created new session: [5:7587241477545169788:2049]
2025-12-24T02:11:56.980914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241477545169788:2049] [node 2] ICS09 handshake done sender: [5:7587241477545169784:2116] self: [5:7587241477545169786:0] peer: [2:7587241475904316457:0] socket: 625 qp: -1
2025-12-24T02:11:56.980920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241477545169788:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.980927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.980941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889409:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
2025-12-24T02:11:56.982635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474227829328:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.982657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.982662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474227829328:2107] [node 6] ICP22 created new session: [3:7587241474227829359:2107]
2025-12-24T02:11:56.982668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474227829359:2107] [node 6] ICS09 handshake done sender: [3:7587241474227829332:2110] self: [3:7587241474227829329:0] peer: [6:7587241475776146978:0] socket: 608 qp: -1
2025-12-24T02:11:56.982880Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506277:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.982921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241478010506252:2107] [node 4] ICP18 handshake (actor [1:7587241478010506277:2119]) is too old
2025-12-24T02:11:56.983114Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475776146980:2121] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.982674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474227829359:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:56.982682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.983398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241477545169700:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.983824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241475776146981:2122] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.983853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475776146969:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:56.983941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475776146889:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.983975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241475776146888:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.983982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475776146890:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.983992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:56.983997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475776146890:2050] [node 3] ICP22 created new session: [6:7587241475776146984:2050]
2025-12-24T02:11:56.984001Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475776146984:2050] [node 3] ICS09 handshake done sender: [6:7587241475776146969:2117] self: [6:7587241475776146978:0] peer: [3:7587241474227829329:0] socket: 618 qp: -1
2025-12-24T02:11:56.984007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475776146984:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:56.984016Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:56.985313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241475904316460:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.985326Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241475904316460:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.986262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146986:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:56.986463Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241475904316460:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.987032Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241477702889322:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25231) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241477702889392:2113] held: no
2025-12-24T02:11:56.987220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477702889392:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.987473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474227829336:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:56.987564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506261:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.987582Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478010506261:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.987800Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475776146980:2121] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.987900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241475904316454:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.987926Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.987933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241475904316454:2111] [node 6] ICP22 created new session: [2:7587241475904316488:2111]
2025-12-24T02:11:56.987939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241475904316488:2111] [node 6] ICS09 handshake done sender: [2:7587241475904316460:2115] self: [2:7587241475904316459:0] peer: [6:7587241475776146982:0] socket: 639 qp: -1
2025-12-24T02:11:56.987946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241475904316488:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:56.987970Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:56.987452Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477702889322:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.987466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.987474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477702889322:2048] [node 1] ICP22 created new session: [4:7587241477702889414:2048]
2025-12-24T02:11:56.987480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477702889414:2048] [node 1] ICS09 handshake done sender: [4:7587241477702889392:2113] self: [4:7587241477702889402:0] peer: [1:7587241478010506258:0] socket: 591 qp: -1
2025-12-24T02:11:56.987487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477702889414:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.987494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.987854Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889413:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.989768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:56.990454Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478010506264:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.990473Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478010506264:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.992618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169792:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.992713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241477545169771:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:56.992862Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169794:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:56.992995Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889397:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.993008Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241477702889397:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:56.993892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889413:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:56.994083Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478010506261:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:56.994325Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477702889397:2115] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.994545Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169795:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:56.994725Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241477702889393:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.994746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.994759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477702889393:2108] [node 5] ICP22 created new session: [4:7587241477702889418:2108]
2025-12-24T02:11:56.994766Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241477702889418:2108] [node 5] ICS09 handshake done sender: [4:7587241477702889397:2115] self: [4:7587241477702889396:0] peer: [5:7587241477545169793:0] socket: 609 qp: -1
2025-12-24T02:11:56.994774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241477702889418:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:56.994784Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.995138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241474227829268:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.995158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.995163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474227829268:2049] [node 2] ICP22 created new session: [3:7587241474227829363:2049]
2025-12-24T02:11:56.995168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474227829363:2049] [node 2] ICS09 handshake done sender: [3:7587241474227829336:2114] self: [3:7587241474227829355:0] peer: [2:7587241475904316449:0] socket: 607 qp: -1
2025-12-24T02:11:56.995174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474227829363:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.995180Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.995426Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241474227829267:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25231) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241474227829333:2111] held: no
2025-12-24T02:11:56.995441Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241477545169785:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.995547Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474227829333:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:56.995756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241477545169700:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.995770Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:56.995775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241477545169700:2048] [node 1] ICP22 created new session: [5:7587241477545169797:2048]
2025-12-24T02:11:56.995780Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241477545169797:2048] [node 1] ICS09 handshake done sender: [5:7587241477545169785:2117] self: [5:7587241477545169790:0] peer: [1:7587241478010506260:0] socket: 633 qp: -1
2025-12-24T02:11:56.995785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241477545169797:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:56.995792Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:56.997731Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478010506264:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:56.997867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478010506254:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.997892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:56.997930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478010506254:2109] [node 5] ICP22 created new session: [1:7587241478010506288:2109]
2025-12-24T02:11:56.997937Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478010506288:2109] [node 5] ICS09 handshake done sender: [1:7587241478010506261:2113] self: [1:7587241478010506260:0] peer: [5:7587241477545169790:0] socket: 632 qp: -1
2025-12-24T02:11:56.997945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478010506288:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:56.997954Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:56.998379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478010506255:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:56.998393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:56.998400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478010506255:2110] [node 6] ICP22 created new session: [1:7587241478010506290:2110]
2025-12-24T02:11:56.998404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478010506290:2110] [node 6] ICS09 handshake done sender: [1:7587241478010506264:2114] self: [1:7587241478010506263:0] peer: [6:7587241475776146983:0] socket: 613 qp: -1
2025-12-24T02:11:56.998410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478010506290:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:56.998418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.998993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241477702889420:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:56.999224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475776146889:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:56.999246Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:56.999253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475776146889:2049] [node 2] ICP22 created new session: [6:7587241475776146988:2049]
2025-12-24T02:11:56.999259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241475776146988:2049] [node 2] ICS09 handshake done sender: [6:7587241475776146980:2121] self: [6:7587241475776146982:0] peer: [2:7587241475904316459:0] socket: 641 qp: -1
2025-12-24T02:11:56.999264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241475776146988:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:56.999271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:56.999427Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241475776146889:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:56.999773Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241475776146987:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:13419
2025-12-24T02:11:57.000450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241477545169792:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.000940Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241474227829267:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.000958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.000965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474227829267:2048] [node 1] ICP22 created new session: [3:7587241474227829365:2048]
2025-12-24T02:11:57.000972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241474227829365:2048] [node 1] ICS09 handshake done sender: [3:7587241474227829333:2111] self: [3:7587241474227829350:0] peer: [1:7587241478010506256:0] socket: 588 qp: -1
2025-12-24T02:11:57.000978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241474227829365:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.000993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.001072Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241475776146981:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.001198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241475776146888:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.001212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.001218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241475776146888:2048] [node 1] ICP22 created new session: [6:7587241480071114286:2048]
2025-12-24T02:11:57.001222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241480071114286:2048] [node 1] ICS09 handshake done sender: [6:7587241475776146981:2122] self: [6:7587241475776146983:0] peer: [1:7587241478010506263:0] socket: 642 qp: -1
2025-12-24T02:11:57.001227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241480071114286:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.001234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.004178Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241477545169771:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63625) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241477545169792:2119] held: no
2025-12-24T02:11:57.004279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241477545169771:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.004289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.004295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241477545169771:2111] [node 4] ICP22 created new session: [5:7587241481840137095:2111]
2025-12-24T02:11:57.004301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241481840137095:2111] [node 4] ICS09 handshake done sender: [5:7587241477545169792:2119] self: [5:7587241477545169793:0] peer: [4:7587241477702889396:0] socket: 610 qp: -1
2025-12-24T02:11:57.004307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241481840137095:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:57.004315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.004094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241477702889420:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.004147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241477702889324:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.004252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829344:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.004263Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474227829344:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.004659Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241477545169796:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.004945Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481997856718:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35320
2025-12-24T02:11:57.005062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474227829344:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.005224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241477702889420:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.005278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241477702889324:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.005292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.005297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241477702889324:2050] [node 3] ICP22 created new session: [4:7587241481997856719:2050]
2025-12-24T02:11:57.005302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481997856719:2050] [node 3] ICS09 handshake done sender: [4:7587241477702889420:2120] self: [4:7587241481997856717:0] peer: [3:7587241474227829343:0] socket: 612 qp: -1
2025-12-24T02:11:57.005307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481997856719:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.005314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.006250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474227829337:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.006268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.006273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474227829337:2108] [node 4] ICP22 created new session: [3:7587241478522796664:2108]
2025-12-24T02:11:57.006278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478522796664:2108] [node 4] ICS09 handshake done sender: [3:7587241474227829344:2117] self: [3:7587241474227829343:0] peer: [4:7587241481997856717:0] socket: 644 qp: -1
2025-12-24T02:11:57.006283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478522796664:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:57.006289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.006710Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241477545169796:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.006769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241477545169702:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.007201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241474227829346:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.007210Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241474227829346:2118] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.008401Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241481840137098:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:36180
2025-12-24T02:11:57.008497Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241474227829346:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.008663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241477545169796:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.008718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241477545169702:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.008734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.008739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241477545169702:2050] [node 3] ICP22 created new session: [5:7587241481840137099:2050]
2025-12-24T02:11:57.008745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241481840137099:2050] [node 3] ICS09 handshake done sender: [5:7587241477545169796:2122] self: [5:7587241481840137097:0] peer: [3:7587241474227829345:0] socket: 647 qp: -1
2025-12-24T02:11:57.008750Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241481840137099:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.008759Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.009824Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241474227829338:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.009840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.009845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241474227829338:2109] [node 5] ICP22 created new session: [3:7587241478522796667:2109]
2025-12-24T02:11:57.009851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478522796667:2109] [node 5] ICS09 handshake done sender: [3:7587241474227829346:2118] self: [3:7587241474227829345:0] peer: [5:7587241481840137097:0] socket: 645 qp: -1
2025-12-24T02:11:57.009856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478522796667:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:57.009863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.015628Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:57.015729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241475904316469:2048] [node 1] ICS01 socket: 577 reason# EndOfStream
2025-12-24T02:11:57.015739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241475904316384:2048] [node 1] ICP30 unregister session Session# [2:7587241475904316469:2048] VirtualId# [2:7587241475904316466:0]
2025-12-24T02:11:57.015747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241475904316469:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.015795Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.015822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.016402Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:57.016473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241474227829365:2048] [node 1] ICS01 socket: 588 reason# EndOfStream
2025-12-24T02:11:57.016478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241474227829267:2048] [node 1] ICP30 unregister session Session# [3:7587241474227829365:2048] VirtualId# [3:7587241474227829350:0]
2025-12-24T02:11:57.016483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241474227829365:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.016521Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.016540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.022888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:57.022958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241477545169797:2048] [node 1] ICS01 socket: 633 reason# EndOfStream
2025-12-24T02:11:57.022964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241477545169700:2048] [node 1] ICP30 unregister session Session# [5:7587241477545169797:2048] VirtualId# [5:7587241477545169790:0]
2025-12-24T02:11:57.022970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241477545169797:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.023017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.023042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.023083Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:57.023148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241480071114286:2048] [node 1] ICS01 socket: 642 reason# EndOfStream
2025-12-24T02:11:57.023156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475776146888:2048] [node 1] ICP30 unregister session Session# [6:7587241480071114286:2048] VirtualId# [6:7587241475776146983:0]
2025-12-24T02:11:57.023161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241480071114286:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.022829Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:57.022948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241477702889414:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-12-24T02:11:57.022961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241477702889322:2048] [node 1] ICP30 unregister session Session# [4:7587241477702889414:2048] VirtualId# [4:7587241477702889402:0]
2025-12-24T02:11:57.022970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241477702889414:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.023017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.023041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.023193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.023217Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.028904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:57.029021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241474227829363:2049] [node 2] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:57.029033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241474227829268:2049] [node 2] ICP30 unregister session Session# [3:7587241474227829363:2049] VirtualId# [3:7587241474227829355:0]
2025-12-24T02:11:57.029043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241474227829363:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.029107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.029140Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.032554Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:57.032658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241477545169788:2049] [node 2] ICS01 socket: 625 reason# EndOfStream
2025-12-24T02:11:57.032667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241477545169701:2049] [node 2] ICP30 unregister session Session# [5:7587241477545169788:2049] VirtualId# [5:7587241477545169786:0]
2025-12-24T02:11:57.032675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241477545169788:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.032724Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.032750Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.036672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:57.036793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241477702889410:2049] [node 2] ICS01 socket: 604 reason# EndOfStream
2025-12-24T02:11:57.036803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241477702889323:2049] [node 2] ICP30 unregister session Session# [4:7587241477702889410:2049] VirtualId# [4:7587241477702889406:0]
2025-12-24T02:11:57.036811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241477702889410:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.036865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.036891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.036940Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:57.037004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475776146988:2049] [node 2] ICS01 socket: 641 reason# EndOfStream
2025-12-24T02:11:57.037009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475776146889:2049] [node 2] ICP30 unregister session Session# [6:7587241475776146988:2049] VirtualId# [6:7587241475776146982:0]
2025-12-24T02:11:57.037014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475776146988:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.037053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.037075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.046350Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:57.046459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241481840137099:2050] [node 3] ICS01 socket: 647 reason# EndOfStream
2025-12-24T02:11:57.046473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241477545169702:2050] [node 3] ICP30 unregister session Session# [5:7587241481840137099:2050] VirtualId# [5:7587241481840137097:0]
2025-12-24T02:11:57.046483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241481840137099:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.046544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.046576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.050918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:57.051030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475776146984:2050] [node 3] ICS01 socket: 618 reason# EndOfStream
2025-12-24T02:11:57.051042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475776146890:2050] [node 3] ICP30 unregister session Session# [6:7587241475776146984:2050] VirtualId# [6:7587241475776146978:0]
2025-12-24T02:11:57.051050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475776146984:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.051104Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.051133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.082785Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:57.082910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241475776146976:2111] [node 4] ICS07 socket disconnect 596 reason# EndOfStream
2025-12-24T02:11:57.082920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241475776146976:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.082991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:57.083015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241475776146976:2111] [node 4] ICS15 start handshake
2025-12-24T02:11:57.083433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241475776146958:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63625) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:57.083447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241475776146976:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:57.083453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241475776146958:2111] [node 4] ICP30 unregister session Session# [6:7587241475776146976:2111] VirtualId# [6:7587241475776146973:0]
2025-12-24T02:11:57.083518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:57.083525Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241475776146958:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:63625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.083528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:63625) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:57.115585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478863884372:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:57.115603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478863884373:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.117011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241481690001175:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.117019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241481690001176:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.118447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479475833456:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.118453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479475833457:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.129354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241479216095668:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.129366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241479216095669:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.129376Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241479216095670:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.135023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241479765354346:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.135041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241479765354347:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.135053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241479765354348:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.137917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241482067081843:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.137927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241482067081844:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.137954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241482067081845:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.160905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241482067081903:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:57.162369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478863884436:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.162521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478863884437:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:57.177266Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241478863884373:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(16004320ns) {Configure:104(16003633ns) {ProcessConfigured:135(12699010ns) {StartInitialHandshake:157(12697967ns) {DropHandshakes:470(567ns) {DropIncomingHandshake:442(22ns) DropOutgoingHandshake:457(22ns)} GenerateSessionVirtualId:988(216ns) PrepareNewSessionHandshake:477(1594ns) {SwitchToState:179(1234ns)}}}}}
2025-12-24T02:11:57.177454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241481690001241:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.178140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241481690001244:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.178307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241481690001245:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.177487Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478863884444:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.179212Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241481690001252:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-12-24T02:11:57.181470Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241482067081912:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:57.179016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241479216095734:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.179180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241479216095735:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.182799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479475833520:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.182926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479475833521:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.183114Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479475833522:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.189279Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479475833456:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(24619269ns) {Configure:104(24618683ns) {ProcessConfigured:135(24230103ns) {StartInitialHandshake:157(24229358ns) {DropHandshakes:470(575ns) {DropIncomingHandshake:442(92ns) DropOutgoingHandshake:457(114ns)} GenerateSessionVirtualId:988(368ns) PrepareNewSessionHandshake:477(1402ns) {SwitchToState:179(838ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.195353Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241479765354413:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:57.195544Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241479765354414:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.195891Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081914:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.196503Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478863884451:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43804
2025-12-24T02:11:57.196526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478863884452:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43804
2025-12-24T02:11:57.196540Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478863884453:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43804
2025-12-24T02:11:57.199845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884453:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.202745Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241481690001175:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27807059ns) {Configure:104(27805703ns) {ProcessConfigured:135(27398180ns) {StartInitialHandshake:157(27397596ns) {DropHandshakes:470(282ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(77ns)} GenerateSessionVirtualId:988(257ns) PrepareNewSessionHandshake:477(971ns) {SwitchToState:179(276ns)}}}}}
2025-12-24T02:11:57.202825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001252:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.203155Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241479765354346:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32798827ns) {Configure:104(32798069ns) {ProcessConfigured:135(32310898ns) {StartInitialHandshake:157(32310120ns) {DropHandshakes:470(693ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(285ns) PrepareNewSessionHandshake:477(944ns) {SwitchToState:179(311ns)}}}}}
2025-12-24T02:11:57.204084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354410:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.204435Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354411:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.203010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241481690001175:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.202993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241482067081914:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.203049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241482067081844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.208703Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884434:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.208720Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478863884434:2108] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.208785Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354410:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.208331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241482067081912:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19953840ns) {Configure:104(19952027ns) {ProcessConfigured:135(19416580ns) {StartInitialHandshake:157(19415712ns) {DropHandshakes:470(1911ns) {DropIncomingHandshake:442(363ns) DropOutgoingHandshake:457(250ns)} GenerateSessionVirtualId:988(495ns) PrepareNewSessionHandshake:477(852ns) {SwitchToState:179(256ns)}}}}}
2025-12-24T02:11:57.209934Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081922:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.208578Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833535:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.208636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833536:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.208667Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833537:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.210014Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354412:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.211234Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354412:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.211281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241479765354348:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.211444Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354411:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.211464Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241479765354413:2108] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.212840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884451:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.212906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478863884455:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43804
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.213236Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241481690001256:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-12-24T02:11:57.213283Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241481690001257:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.213319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241481690001258:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-12-24T02:11:57.213747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884443:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.213758Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478863884443:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.214443Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478863884434:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.213577Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833535:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.213607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833536:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.213631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479475833456:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.213642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479475833457:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.210361Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095746:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.210407Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095747:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.210442Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095748:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.214549Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095739:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.214556Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241479216095739:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.214593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478863884372:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.214615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.214624Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478863884372:2048] [node 2] ICP22 created new session: [1:7587241478863884459:2048]
2025-12-24T02:11:57.214630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478863884459:2048] [node 2] ICS09 handshake done sender: [1:7587241478863884434:2108] self: [1:7587241478863884433:0] peer: [2:7587241481690001255:0] socket: 586 qp: -1
2025-12-24T02:11:57.214639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478863884459:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:57.214649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.215521Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354430:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.215598Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001247:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.215609Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241481690001247:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.216088Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241481690001252:2114] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.216575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001243:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.216589Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241481690001243:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.216217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833537:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.216256Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833540:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.216829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833527:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.216835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479475833527:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.217460Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479475833527:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.215124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095746:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.215156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095747:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.215243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241479216095668:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.215253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241479216095669:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.216713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241479216095739:2112] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.221188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479475833521:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.221211Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.221221Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479475833521:2108] [node 5] ICP22 created new session: [3:7587241479475833543:2108]
2025-12-24T02:11:57.221229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479475833543:2108] [node 5] ICS09 handshake done sender: [3:7587241479475833527:2112] self: [3:7587241479475833526:0] peer: [5:7587241479765354428:0] socket: 605 qp: -1
2025-12-24T02:11:57.221238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479475833543:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.221249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.221824Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.222893Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241479216095734:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.222918Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.222926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241479216095734:2108] [node 5] ICP22 created new session: [4:7587241479216095752:2108]
2025-12-24T02:11:57.222932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241479216095752:2108] [node 5] ICS09 handshake done sender: [4:7587241479216095739:2112] self: [4:7587241479216095738:0] peer: [5:7587241479765354429:0] socket: 596 qp: -1
2025-12-24T02:11:57.222939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241479216095752:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.222964Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.227375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884452:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.227477Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478863884443:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.227922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241481690001175:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.227948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.227968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241481690001175:2048] [node 1] ICP22 created new session: [2:7587241481690001260:2048]
2025-12-24T02:11:57.227975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241481690001260:2048] [node 1] ICS09 handshake done sender: [2:7587241481690001252:2114] self: [2:7587241481690001255:0] peer: [1:7587241478863884433:0] socket: 589 qp: -1
2025-12-24T02:11:57.227981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241481690001260:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.227988Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.228666Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001257:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.228701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001256:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.228731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001240:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.228735Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241481690001240:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.229815Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241481690001247:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.230049Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241481690001243:2111] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.230815Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884441:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.230831Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478863884441:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.231367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478863884373:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.231380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.231385Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478863884373:2049] [node 3] ICP22 created new session: [1:7587241478863884463:2049]
2025-12-24T02:11:57.231390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478863884463:2049] [node 3] ICS09 handshake done sender: [1:7587241478863884443:2112] self: [1:7587241478863884440:0] peer: [3:7587241479475833538:0] socket: 587 qp: -1
2025-12-24T02:11:57.231394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478863884463:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.231400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.231609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884455:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.232171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241479765354411:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.232098Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479475833535:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.230825Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241482067081922:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.230868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241482067081903:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.232340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354422:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.232349Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241479765354422:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.232347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241481690001240:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.232906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241479765354413:2108] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.232924Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.232929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241479765354413:2108] [node 4] ICP22 created new session: [5:7587241479765354431:2108]
2025-12-24T02:11:57.232934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241479765354431:2108] [node 4] ICS09 handshake done sender: [5:7587241479765354411:2111] self: [5:7587241479765354429:0] peer: [4:7587241479216095738:0] socket: 599 qp: -1
2025-12-24T02:11:57.232941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241479765354431:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:57.232948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.233983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241481690001244:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.233999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.234008Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241481690001244:2107] [node 4] ICP22 created new session: [2:7587241481690001267:2107]
2025-12-24T02:11:57.234016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241481690001267:2107] [node 4] ICS09 handshake done sender: [2:7587241481690001247:2112] self: [2:7587241481690001246:0] peer: [4:7587241479216095751:0] socket: 592 qp: -1
2025-12-24T02:11:57.234021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241481690001267:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.234031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.234328Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354434:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.234340Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354435:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.234557Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095754:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.234766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095748:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.234947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241481690001241:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.234967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.234973Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241481690001241:2109] [node 6] ICP22 created new session: [2:7587241481690001269:2109]
2025-12-24T02:11:57.234979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241481690001269:2109] [node 6] ICS09 handshake done sender: [2:7587241481690001243:2111] self: [2:7587241481690001242:0] peer: [6:7587241482067081921:0] socket: 591 qp: -1
2025-12-24T02:11:57.234983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241481690001269:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.234991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.235282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241481690001176:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.235295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.235300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241481690001176:2049] [node 3] ICP22 created new session: [2:7587241481690001271:2049]
2025-12-24T02:11:57.235304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241481690001271:2049] [node 3] ICS09 handshake done sender: [2:7587241481690001240:2110] self: [2:7587241481690001239:0] peer: [3:7587241479475833539:0] socket: 590 qp: -1
2025-12-24T02:11:57.235309Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241481690001271:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.235315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.235795Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478863884441:2111] [node 4] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.236134Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833546:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.235920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478863884436:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.235938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.235943Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478863884436:2107] [node 4] ICP22 created new session: [1:7587241478863884468:2107]
2025-12-24T02:11:57.235949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478863884468:2107] [node 4] ICS09 handshake done sender: [1:7587241478863884441:2111] self: [1:7587241478863884438:0] peer: [4:7587241479216095750:0] socket: 585 qp: -1
2025-12-24T02:11:57.235953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478863884468:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.235974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.236315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081927:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.236436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241479216095747:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.236505Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081926:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.236690Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479475833456:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.236708Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.236715Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479475833456:2048] [node 1] ICP22 created new session: [3:7587241479475833548:2048]
2025-12-24T02:11:57.236721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479475833548:2048] [node 1] ICS09 handshake done sender: [3:7587241479475833535:2115] self: [3:7587241479475833538:0] peer: [1:7587241478863884440:0] socket: 602 qp: -1
2025-12-24T02:11:57.236727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479475833548:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.236734Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.237717Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241479216095669:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.237732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.237738Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241479216095669:2049] [node 2] ICP22 created new session: [4:7587241479216095757:2049]
2025-12-24T02:11:57.237743Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241479216095757:2049] [node 2] ICS09 handshake done sender: [4:7587241479216095747:2116] self: [4:7587241479216095751:0] peer: [2:7587241481690001246:0] socket: 621 qp: -1
2025-12-24T02:11:57.237748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241479216095757:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.237755Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.238485Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241479216095734:2108] [node 5] ICP18 handshake (actor [4:7587241479216095748:2117]) is too old
2025-12-24T02:11:57.238863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241479765354422:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.238958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241479765354412:2112] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.241147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095756:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.241317Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241479216095668:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241479216095746:2115] held: no
2025-12-24T02:11:57.241666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241479216095746:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.240598Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241479475833456:2048] [node 1] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.240688Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241479475833457:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18820) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241479475833536:2116] held: no
2025-12-24T02:11:57.240846Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833547:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.241817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479475833536:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.242971Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354437:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.242962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479475833457:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.242982Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.242989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479475833457:2049] [node 2] ICP22 created new session: [3:7587241479475833551:2049]
2025-12-24T02:11:57.242996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479475833551:2049] [node 2] ICS09 handshake done sender: [3:7587241479475833536:2116] self: [3:7587241479475833539:0] peer: [2:7587241481690001239:0] socket: 616 qp: -1
2025-12-24T02:11:57.243002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479475833551:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.243036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.243947Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241479765354348:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.243991Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.244002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241479765354348:2050] [node 3] ICP22 created new session: [5:7587241479765354438:2050]
2025-12-24T02:11:57.244009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241479765354438:2050] [node 3] ICS09 handshake done sender: [5:7587241479765354412:2112] self: [5:7587241479765354428:0] peer: [3:7587241479475833526:0] socket: 608 qp: -1
2025-12-24T02:11:57.244016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241479765354438:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.244025Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.244456Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241482067081914:2112] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.244555Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241482067081844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18820) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241482067081914:2112] held: no
2025-12-24T02:11:57.244644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241482067081844:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.244652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.244657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241482067081844:2049] [node 2] ICP22 created new session: [6:7587241482067081930:2049]
2025-12-24T02:11:57.244662Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241482067081930:2049] [node 2] ICS09 handshake done sender: [6:7587241482067081914:2112] self: [6:7587241482067081921:0] peer: [2:7587241481690001242:0] socket: 597 qp: -1
2025-12-24T02:11:57.244668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241482067081930:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.244675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.244919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241482067081924:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.244983Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081928:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.244998Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081929:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.245526Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095759:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.245795Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241482067081926:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.245863Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241482067081922:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.246252Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095759:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.246263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241479216095668:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.246272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.246276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241479216095668:2048] [node 1] ICP22 created new session: [4:7587241479216095760:2048]
2025-12-24T02:11:57.246286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241479216095760:2048] [node 1] ICS09 handshake done sender: [4:7587241479216095746:2115] self: [4:7587241479216095750:0] peer: [1:7587241478863884438:0] socket: 598 qp: -1
2025-12-24T02:11:57.246292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241479216095760:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.246303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.246530Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241479216095670:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.246578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241482067081845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.246748Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241482067081928:2120] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.246782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241482067081843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.246834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241482067081912:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.246961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241482067081903:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.246972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.246979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241482067081903:2106] [node 5] ICP22 created new session: [6:7587241482067081936:2106]
2025-12-24T02:11:57.246986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241482067081936:2106] [node 5] ICS09 handshake done sender: [6:7587241482067081922:2115] self: [6:7587241482067081925:0] peer: [5:7587241479765354421:0] socket: 615 qp: -1
2025-12-24T02:11:57.246991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241482067081936:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:57.246997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.246851Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884446:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.246862Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478863884446:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.247390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478863884446:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.247462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478863884444:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.247477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.247483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478863884444:2110] [node 6] ICP22 created new session: [1:7587241478863884471:2110]
2025-12-24T02:11:57.247489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478863884471:2110] [node 6] ICS09 handshake done sender: [1:7587241478863884446:2113] self: [1:7587241478863884445:0] peer: [6:7587241482067081934:0] socket: 633 qp: -1
2025-12-24T02:11:57.247493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478863884471:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:57.247498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.248052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241479765354414:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.248079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.248084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241479765354414:2109] [node 6] ICP22 created new session: [5:7587241479765354440:2109]
2025-12-24T02:11:57.248089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241479765354440:2109] [node 6] ICS09 handshake done sender: [5:7587241479765354422:2116] self: [5:7587241479765354421:0] peer: [6:7587241482067081925:0] socket: 614 qp: -1
2025-12-24T02:11:57.248093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241479765354440:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.248098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.248339Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354437:2121] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.248367Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241479765354435:2120] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.248963Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241479765354346:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.248985Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241479765354347:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.249184Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001249:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.249196Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241481690001249:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.248625Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833531:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.248636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479475833531:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.249181Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833529:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.249184Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479475833529:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.250928Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478863884450:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.250937Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478863884450:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.251466Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354445:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.251555Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478863884450:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.251704Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241479765354437:2121] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.251761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241479765354346:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.251772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.251778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241479765354346:2048] [node 1] ICP22 created new session: [5:7587241479765354446:2048]
2025-12-24T02:11:57.251783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241479765354446:2048] [node 1] ICS09 handshake done sender: [5:7587241479765354437:2121] self: [5:7587241479765354443:0] peer: [1:7587241478863884442:0] socket: 650 qp: -1
2025-12-24T02:11:57.251788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241479765354446:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.251797Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.252201Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241479475833521:2108] [node 5] ICP27 obsolete handshake fail ignored
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.253326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478863884437:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.253344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.253349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478863884437:2109] [node 5] ICP22 created new session: [1:7587241478863884474:2109]
2025-12-24T02:11:57.253355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478863884474:2109] [node 5] ICS09 handshake done sender: [1:7587241478863884450:2114] self: [1:7587241478863884442:0] peer: [5:7587241479765354443:0] socket: 644 qp: -1
2025-12-24T02:11:57.253361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478863884474:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:57.253370Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.253945Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241479216095741:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.253959Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241479216095741:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.254002Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241481690001273:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:33865
2025-12-24T02:11:57.254110Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095763:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.254153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479475833553:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:51993
2025-12-24T02:11:57.254784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479475833531:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.256156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479475833529:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.255637Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241479216095741:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.256250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479475833520:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.256268Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.256275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479475833520:2107] [node 4] ICP22 created new session: [3:7587241479475833556:2107]
2025-12-24T02:11:57.256281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479475833556:2107] [node 4] ICS09 handshake done sender: [3:7587241479475833531:2114] self: [3:7587241479475833530:0] peer: [4:7587241479216095762:0] socket: 601 qp: -1
2025-12-24T02:11:57.256286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479475833556:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.256293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.256490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479475833522:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.256496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.256499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479475833522:2109] [node 6] ICP22 created new session: [3:7587241479475833558:2109]
2025-12-24T02:11:57.256503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479475833558:2109] [node 6] ICS09 handshake done sender: [3:7587241479475833529:2113] self: [3:7587241479475833528:0] peer: [6:7587241482067081933:0] socket: 631 qp: -1
2025-12-24T02:11:57.256507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479475833558:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.256511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.257642Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241481690001273:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.257843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241481690001249:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.258758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241481690001245:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.258779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.258786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241481690001245:2108] [node 5] ICP22 created new session: [2:7587241481690001276:2108]
2025-12-24T02:11:57.258793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241481690001276:2108] [node 5] ICS09 handshake done sender: [2:7587241481690001249:2113] self: [2:7587241481690001248:0] peer: [5:7587241479765354444:0] socket: 637 qp: -1
2025-12-24T02:11:57.258798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241481690001276:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.258805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.262881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241479216095735:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.262910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.262921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241479216095735:2109] [node 6] ICP22 created new session: [4:7587241479216095765:2109]
2025-12-24T02:11:57.262930Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241479216095765:2109] [node 6] ICS09 handshake done sender: [4:7587241479216095741:2113] self: [4:7587241479216095740:0] peer: [6:7587241482067081935:0] socket: 642 qp: -1
2025-12-24T02:11:57.262940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241479216095765:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.262951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.263232Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241479216095670:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6603) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241479216095759:2120] held: no
2025-12-24T02:11:57.263423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241479216095759:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.263595Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241479216095670:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.263609Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.263614Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241479216095670:2050] [node 3] ICP22 created new session: [4:7587241479216095767:2050]
2025-12-24T02:11:57.263620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241479216095767:2050] [node 3] ICS09 handshake done sender: [4:7587241479216095759:2120] self: [4:7587241479216095762:0] peer: [3:7587241479475833530:0] socket: 594 qp: -1
2025-12-24T02:11:57.263625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241479216095767:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.263633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.264988Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241482067081843:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:7339) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241482067081926:2118] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.266339Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241479765354448:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59935
2025-12-24T02:11:57.265155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081939:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.266053Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081938:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.267862Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241479765354347:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18820) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241479765354435:2120] held: no
2025-12-24T02:11:57.268032Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241479765354435:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.268139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241479765354347:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.268225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.268790Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241482067081940:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43328
2025-12-24T02:11:57.269215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479475833553:2121] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.269272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241479475833522:2109] [node 6] ICP18 handshake (actor [3:7587241479475833553:2121]) is too old
2025-12-24T02:11:57.269139Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241482067081926:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.269402Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241482067081924:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.269586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241482067081928:2120] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.269647Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241482067081843:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.269663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.269669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241482067081843:2048] [node 1] ICP22 created new session: [6:7587241482067081942:2048]
2025-12-24T02:11:57.269675Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241482067081942:2048] [node 1] ICS09 handshake done sender: [6:7587241482067081926:2118] self: [6:7587241482067081934:0] peer: [1:7587241478863884445:0] socket: 639 qp: -1
2025-12-24T02:11:57.269682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241482067081942:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.269690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.268238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241479765354347:2049] [node 2] ICP22 created new session: [5:7587241479765354449:2049]
2025-12-24T02:11:57.268251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241479765354449:2049] [node 2] ICS09 handshake done sender: [5:7587241479765354435:2120] self: [5:7587241479765354444:0] peer: [2:7587241481690001248:0] socket: 611 qp: -1
2025-12-24T02:11:57.268262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241479765354449:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.268279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.275022Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241479216095769:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:15217
2025-12-24T02:11:57.274853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241482067081845:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.274880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.274895Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241482067081845:2050] [node 3] ICP22 created new session: [6:7587241482067081944:2050]
2025-12-24T02:11:57.274901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241482067081944:2050] [node 3] ICS09 handshake done sender: [6:7587241482067081924:2117] self: [6:7587241482067081933:0] peer: [3:7587241479475833528:0] socket: 632 qp: -1
2025-12-24T02:11:57.274910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241482067081944:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.274920Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.275085Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241482067081845:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.275347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241482067081912:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.275359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.275366Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241482067081912:2110] [node 4] ICP22 created new session: [6:7587241482067081946:2110]
2025-12-24T02:11:57.275372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241482067081946:2110] [node 4] ICS09 handshake done sender: [6:7587241482067081928:2120] self: [6:7587241482067081935:0] peer: [4:7587241479216095740:0] socket: 647 qp: -1
2025-12-24T02:11:57.275377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241482067081946:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:57.275383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.290276Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:57.290364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241479475833548:2048] [node 1] ICS01 socket: 602 reason# EndOfStream
2025-12-24T02:11:57.290370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479475833456:2048] [node 1] ICP30 unregister session Session# [3:7587241479475833548:2048] VirtualId# [3:7587241479475833538:0]
2025-12-24T02:11:57.290376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241479475833548:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.290413Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.290436Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.289932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:57.290058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241481690001260:2048] [node 1] ICS01 socket: 589 reason# EndOfStream
2025-12-24T02:11:57.290072Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241481690001175:2048] [node 1] ICP30 unregister session Session# [2:7587241481690001260:2048] VirtualId# [2:7587241481690001255:0]
2025-12-24T02:11:57.290081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241481690001260:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.290153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.290183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.294385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:57.294483Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241482067081942:2048] [node 1] ICS01 socket: 639 reason# EndOfStream
2025-12-24T02:11:57.294490Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241482067081843:2048] [node 1] ICP30 unregister session Session# [6:7587241482067081942:2048] VirtualId# [6:7587241482067081934:0]
2025-12-24T02:11:57.294496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241482067081942:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.294559Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.294585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.292477Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:57.292576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241479765354446:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-12-24T02:11:57.292585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241479765354346:2048] [node 1] ICP30 unregister session Session# [5:7587241479765354446:2048] VirtualId# [5:7587241479765354443:0]
2025-12-24T02:11:57.292592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241479765354446:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.292646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.292683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.295511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:57.295599Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241479216095760:2048] [node 1] ICS01 socket: 598 reason# EndOfStream
2025-12-24T02:11:57.295610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241479216095668:2048] [node 1] ICP30 unregister session Session# [4:7587241479216095760:2048] VirtualId# [4:7587241479216095750:0]
2025-12-24T02:11:57.295616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241479216095760:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.295663Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.295693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.306383Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:57.306490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241479475833551:2049] [node 2] ICS01 socket: 616 reason# EndOfStream
2025-12-24T02:11:57.306499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479475833457:2049] [node 2] ICP30 unregister session Session# [3:7587241479475833551:2049] VirtualId# [3:7587241479475833539:0]
2025-12-24T02:11:57.306506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241479475833551:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.306620Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.306651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.308345Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:57.308412Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241482067081930:2049] [node 2] ICS01 socket: 597 reason# EndOfStream
2025-12-24T02:11:57.308418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241482067081844:2049] [node 2] ICP30 unregister session Session# [6:7587241482067081930:2049] VirtualId# [6:7587241482067081921:0]
2025-12-24T02:11:57.308425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241482067081930:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.308468Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.308498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.308182Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:57.308290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241479765354449:2049] [node 2] ICS01 socket: 611 reason# EndOfStream
2025-12-24T02:11:57.308298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241479765354347:2049] [node 2] ICP30 unregister session Session# [5:7587241479765354449:2049] VirtualId# [5:7587241479765354444:0]
2025-12-24T02:11:57.308304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241479765354449:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.308352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.308385Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.308683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:57.308765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241479216095757:2049] [node 2] ICS01 socket: 621 reason# EndOfStream
2025-12-24T02:11:57.308771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241479216095669:2049] [node 2] ICP30 unregister session Session# [4:7587241479216095757:2049] VirtualId# [4:7587241479216095751:0]
2025-12-24T02:11:57.308777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241479216095757:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.308817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.308840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.322888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:57.323002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241479216095767:2050] [node 3] ICS01 socket: 594 reason# EndOfStream
2025-12-24T02:11:57.323016Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241479216095670:2050] [node 3] ICP30 unregister session Session# [4:7587241479216095767:2050] VirtualId# [4:7587241479216095762:0]
2025-12-24T02:11:57.323025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241479216095767:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.323068Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.323097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.323056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:57.323143Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241482067081944:2050] [node 3] ICS01 socket: 632 reason# EndOfStream
2025-12-24T02:11:57.323152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241482067081845:2050] [node 3] ICP30 unregister session Session# [6:7587241482067081944:2050] VirtualId# [6:7587241482067081933:0]
2025-12-24T02:11:57.323159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241482067081944:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.323199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.323213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.323270Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:57.323374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241479765354438:2050] [node 3] ICS07 socket disconnect 608 reason# EndOfStream
2025-12-24T02:11:57.323379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241479765354438:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.323445Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.323469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241479765354438:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:57.323903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241479765354348:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6603) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:57.323916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241479765354438:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:57.323921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241479765354348:2050] [node 3] ICP30 unregister session Session# [5:7587241479765354438:2050] VirtualId# [5:7587241479765354428:0]
2025-12-24T02:11:57.324013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.324019Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241479765354348:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6603) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.324022Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6603) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.334164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241479765354348:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:57.339423Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:57.339539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241482067081946:2110] [node 4] ICS01 socket: 647 reason# EndOfStream
2025-12-24T02:11:57.339550Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241482067081912:2110] [node 4] ICP30 unregister session Session# [6:7587241482067081946:2110] VirtualId# [6:7587241482067081935:0]
2025-12-24T02:11:57.339558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241482067081946:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.339624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:57.339657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:57.384953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482276144502:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.384967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482276144503:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.386583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478772683746:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.386591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478772683747:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.398815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478737191279:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.395144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481319086775:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.402727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478737191278:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:57.395165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481319086776:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.395169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481319086777:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.406136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479434574452:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.406148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479434574453:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.406159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479434574454:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.410845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482620099218:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.410867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482620099219:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.410873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482620099220:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.434095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479434574509:2103] [node 5] ICP01 ready to work
2025-12-24T02:11:57.437372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478737191343:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.437500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478737191344:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.437627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241478737191345:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.439105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086836:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.438866Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099278:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.447941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478737191357:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52781
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.451261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482276144565:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.452386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099283:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.451765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482276144564:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.452052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482276144568:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.452073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478772683811:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:57.455057Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683808:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.455304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478772683818:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:57.456046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241478772683821:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:57.458572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241479434574516:2108] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.458590Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241479434574516:2108] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.456763Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482620099278:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.457829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482620099283:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.457997Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482620099286:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:57.458414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482620099286:2111] [node 6] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.463053Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481319086841:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:57.463206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481319086842:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:57.466270Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144570:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.467082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478737191358:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52781
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.467826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574521:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.467856Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574522:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.467877Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574523:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.468469Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683824:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.468541Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683825:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.468675Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683808:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.469355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191357:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.467857Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144570:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.465038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099291:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.470835Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241481319086836:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.469988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241482276144565:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(17811892ns) {Configure:104(17810485ns) {ProcessConfigured:135(17578014ns) {StartInitialHandshake:157(17577680ns) {DropHandshakes:470(462ns) {DropIncomingHandshake:442(64ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(296ns) PrepareNewSessionHandshake:477(592ns) {SwitchToState:179(208ns)}}}}}
2025-12-24T02:11:57.471237Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241481319086776:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16722545ns) {Configure:104(16718926ns) {ProcessConfigured:135(16355218ns) {StartInitialHandshake:157(16354256ns) {DropHandshakes:470(504ns) {DropIncomingHandshake:442(31ns) DropOutgoingHandshake:457(70ns)} GenerateSessionVirtualId:988(301ns) PrepareNewSessionHandshake:477(1226ns) {SwitchToState:179(574ns)}}}}}
2025-12-24T02:11:57.471305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241482276144502:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.471891Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683825:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.472034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241479434574523:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.472026Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191358:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.472290Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479434574454:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.472344Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241479434574516:2108] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.473426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482620099293:2114] [node 4] ICP01 ready to work
2025-12-24T02:11:57.477611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683824:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.476885Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191342:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.476907Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478737191342:2110] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.478035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478737191342:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.478184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478737191278:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.478205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.478214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478737191278:2048] [node 2] ICP22 created new session: [1:7587241478737191362:2048]
2025-12-24T02:11:57.478220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478737191362:2048] [node 2] ICS09 handshake done sender: [1:7587241478737191342:2110] self: [1:7587241478737191341:0] peer: [2:7587241482276144582:0] socket: 588 qp: -1
2025-12-24T02:11:57.478227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478737191362:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:57.478235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.477955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241479434574522:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.477977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241479434574509:2103] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.477993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.478005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479434574509:2103] [node 5] ICP22 created new session: [6:7587241479434574531:2103]
2025-12-24T02:11:57.478012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241479434574531:2103] [node 5] ICS09 handshake done sender: [6:7587241479434574516:2108] self: [6:7587241479434574513:0] peer: [5:7587241482620099287:0] socket: 597 qp: -1
2025-12-24T02:11:57.478030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241479434574531:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:57.478053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.478316Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241479434574521:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.478545Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479434574533:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:57.479115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479434574533:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.479431Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241481319086848:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.479448Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241481319086848:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.479986Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241481319086848:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.480131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241481319086842:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.480152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.480160Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481319086842:2110] [node 6] ICP22 created new session: [4:7587241481319086854:2110]
2025-12-24T02:11:57.480168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481319086854:2110] [node 6] ICS09 handshake done sender: [4:7587241481319086848:2113] self: [4:7587241481319086847:0] peer: [6:7587241479434574536:0] socket: 610 qp: -1
2025-12-24T02:11:57.480175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481319086854:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:57.480182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.483509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478737191365:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52781
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.484026Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241478772683811:2109] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(31800591ns) {Configure:104(31799906ns) {ProcessConfigured:135(31423359ns) {StartInitialHandshake:157(31422516ns) {DropHandshakes:470(535ns) {DropIncomingHandshake:442(58ns) DropOutgoingHandshake:457(104ns)} GenerateSessionVirtualId:988(236ns) PrepareNewSessionHandshake:477(1736ns) {SwitchToState:179(1110ns)}}}}}
2025-12-24T02:11:57.484053Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241479434574453:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16113395ns) {Configure:104(16112539ns) {ProcessConfigured:135(15857892ns) {StartInitialHandshake:157(15857573ns) {DropHandshakes:470(296ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(56ns)} GenerateSessionVirtualId:988(167ns) PrepareNewSessionHandshake:477(1645ns) {SwitchToState:179(1170ns)}}}}}
2025-12-24T02:11:57.484257Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479434574453:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.485956Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191365:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.486181Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478737191366:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52781
2025-12-24T02:11:57.486452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191366:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.486445Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683820:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.486468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478772683820:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.483803Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482620099291:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.483872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099296:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.483885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099297:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.484162Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482620099283:2110] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.484354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482620099286:2111] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.484369Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.484377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482620099286:2111] [node 6] ICP22 created new session: [5:7587241482620099299:2111]
2025-12-24T02:11:57.484385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482620099299:2111] [node 6] ICS09 handshake done sender: [5:7587241482620099283:2110] self: [5:7587241482620099287:0] peer: [6:7587241479434574513:0] socket: 603 qp: -1
2025-12-24T02:11:57.484393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482620099299:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:57.484415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.487554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574540:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.487570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574541:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.487655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241479434574521:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.488300Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479434574533:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.488324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.488333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479434574533:2115] [node 4] ICP22 created new session: [6:7587241479434574542:2115]
2025-12-24T02:11:57.488342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241479434574542:2115] [node 4] ICS09 handshake done sender: [6:7587241479434574521:2111] self: [6:7587241479434574536:0] peer: [4:7587241481319086847:0] socket: 621 qp: -1
2025-12-24T02:11:57.488349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241479434574542:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:57.488363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.488479Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241482276144502:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241482276144570:2110] held: no
2025-12-24T02:11:57.488641Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144575:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.488652Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482276144575:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.489560Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144583:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
2025-12-24T02:11:57.489878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144583:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.489935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482276144575:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.490065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482276144568:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.490086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.490093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482276144568:2109] [node 6] ICP22 created new session: [2:7587241482276144586:2109]
2025-12-24T02:11:57.490099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482276144586:2109] [node 6] ICS09 handshake done sender: [2:7587241482276144575:2113] self: [2:7587241482276144574:0] peer: [6:7587241479434574539:0] socket: 615 qp: -1
2025-12-24T02:11:57.490105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482276144586:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.490114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.490910Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086857:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.492327Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144590:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.494636Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144592:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
2025-12-24T02:11:57.490921Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241482620099218:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(19394051ns) {Configure:104(19392056ns) {ProcessConfigured:135(19004873ns) {StartInitialHandshake:157(19004461ns) {DropHandshakes:470(474ns) {DropIncomingHandshake:442(33ns) DropOutgoingHandshake:457(82ns)} GenerateSessionVirtualId:988(243ns) PrepareNewSessionHandshake:477(991ns) {SwitchToState:179(754ns)}}}}}
2025-12-24T02:11:57.491002Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482620099218:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.492500Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482620099296:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.492533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482620099219:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.494294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191349:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.494309Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478737191349:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.494916Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478737191349:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.494912Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478772683820:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.494619Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241481319086775:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(33286438ns) {Configure:104(33284495ns) {ProcessConfigured:135(32895004ns) {StartInitialHandshake:157(32893830ns) {DropHandshakes:470(592ns) {DropIncomingHandshake:442(14ns) DropOutgoingHandshake:457(55ns)} GenerateSessionVirtualId:988(317ns) PrepareNewSessionHandshake:477(1693ns) {SwitchToState:179(1102ns)}}}}}
2025-12-24T02:11:57.494664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481319086775:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.495029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478737191344:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.495049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.495056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478737191344:2108] [node 5] ICP22 created new session: [1:7587241478737191370:2108]
2025-12-24T02:11:57.495062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478737191370:2108] [node 5] ICS09 handshake done sender: [1:7587241478737191349:2112] self: [1:7587241478737191348:0] peer: [5:7587241482620099301:0] socket: 590 qp: -1
2025-12-24T02:11:57.495068Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478737191370:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.495077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.495356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144591:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
2025-12-24T02:11:57.494668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574544:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.496232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478772683818:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.496263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.496271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478772683818:2112] [node 6] ICP22 created new session: [3:7587241478772683832:2112]
2025-12-24T02:11:57.496278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478772683832:2112] [node 6] ICS09 handshake done sender: [3:7587241478772683820:2113] self: [3:7587241478772683819:0] peer: [6:7587241479434574529:0] socket: 619 qp: -1
2025-12-24T02:11:57.496284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478772683832:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:57.496294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.495611Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241481319086846:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.495620Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241481319086846:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.496329Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241481319086846:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.496408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241481319086841:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.496425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.496432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481319086841:2109] [node 5] ICP22 created new session: [4:7587241481319086861:2109]
2025-12-24T02:11:57.496439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481319086861:2109] [node 5] ICS09 handshake done sender: [4:7587241481319086846:2112] self: [4:7587241481319086845:0] peer: [5:7587241482620099304:0] socket: 609 qp: -1
2025-12-24T02:11:57.496444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481319086861:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:57.496478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.495462Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241482620099293:2114] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(21708765ns) {Configure:104(21707507ns) {ProcessConfigured:135(21293940ns) {StartInitialHandshake:157(21292988ns) {DropHandshakes:470(952ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(249ns) PrepareNewSessionHandshake:477(1819ns) {SwitchToState:179(793ns)}}}}}
2025-12-24T02:11:57.495494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482620099293:2114] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.498872Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086864:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.499223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144590:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.499331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482276144570:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.499358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241481319086864:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.499398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481319086776:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.499484Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241479434574540:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.499529Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479434574452:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.499620Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574545:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.499663Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241481319086842:2110] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.499607Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144573:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.499627Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482276144573:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.499817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241479434574523:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.500702Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191347:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.500719Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478737191347:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.501502Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191354:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.501511Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478737191354:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.502065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241478737191372:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52781
2025-12-24T02:11:57.502571Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099306:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.502729Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478737191354:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.502614Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099305:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.501306Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683834:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.503364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144581:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.503373Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482276144581:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.503907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241482276144502:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.503922Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.503928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482276144502:2048] [node 1] ICP22 created new session: [2:7587241482276144594:2048]
2025-12-24T02:11:57.503933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482276144594:2048] [node 1] ICS09 handshake done sender: [2:7587241482276144570:2110] self: [2:7587241482276144582:0] peer: [1:7587241478737191341:0] socket: 594 qp: -1
2025-12-24T02:11:57.503940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482276144594:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.503973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.504908Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482276144593:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45950
2025-12-24T02:11:57.504984Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086866:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.505653Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144592:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.505715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482276144573:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.505857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482276144581:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.506031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241481319086864:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.505949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482276144564:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.505971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.505980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482276144564:2108] [node 5] ICP22 created new session: [2:7587241482276144599:2108]
2025-12-24T02:11:57.505985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482276144599:2108] [node 5] ICS09 handshake done sender: [2:7587241482276144573:2112] self: [2:7587241482276144572:0] peer: [5:7587241482620099302:0] socket: 614 qp: -1
2025-12-24T02:11:57.505992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482276144599:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.505999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.506456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482276144565:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.506464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.506467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482276144565:2107] [node 4] ICP22 created new session: [2:7587241482276144601:2107]
2025-12-24T02:11:57.506471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482276144601:2107] [node 4] ICS09 handshake done sender: [2:7587241482276144581:2115] self: [2:7587241482276144569:0] peer: [4:7587241481319086865:0] socket: 639 qp: -1
2025-12-24T02:11:57.506475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482276144601:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.506480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.506877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241482276144565:2107] [node 4] ICP18 handshake (actor [2:7587241482276144592:2119]) is too old
2025-12-24T02:11:57.506218Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481319086776:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.506285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.506298Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481319086776:2049] [node 2] ICP22 created new session: [4:7587241481319086867:2049]
2025-12-24T02:11:57.506305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481319086867:2049] [node 2] ICS09 handshake done sender: [4:7587241481319086864:2117] self: [4:7587241481319086865:0] peer: [2:7587241482276144569:0] socket: 641 qp: -1
2025-12-24T02:11:57.506313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481319086867:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.506326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.507197Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241479434574522:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.507992Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478737191347:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.508074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191372:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.508089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478737191345:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.508100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478737191343:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.508106Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.508110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.508115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478737191345:2109] [node 6] ICP22 created new session: [1:7587241478737191376:2109]
2025-12-24T02:11:57.508116Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478737191343:2107] [node 4] ICP22 created new session: [1:7587241478737191377:2107]
2025-12-24T02:11:57.508122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478737191377:2107] [node 4] ICS09 handshake done sender: [1:7587241478737191347:2111] self: [1:7587241478737191346:0] peer: [4:7587241481319086859:0] socket: 589 qp: -1
2025-12-24T02:11:57.508122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478737191376:2109] [node 6] ICS09 handshake done sender: [1:7587241478737191354:2113] self: [1:7587241478737191353:0] peer: [6:7587241479434574547:0] socket: 629 qp: -1
2025-12-24T02:11:57.508127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478737191376:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.508127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478737191377:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.508134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.508146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.510160Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482276144577:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.510191Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482276144577:2114] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.507321Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683834:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.507361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241478772683747:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.511873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241481319086775:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241481319086836:2108] held: no
2025-12-24T02:11:57.511526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241478737191343:2107] [node 4] ICP18 handshake (actor [1:7587241478737191372:2119]) is too old
2025-12-24T02:11:57.511375Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683837:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.515069Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479434574453:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.515100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.515110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479434574453:2049] [node 2] ICP22 created new session: [6:7587241479434574549:2049]
2025-12-24T02:11:57.515118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241479434574549:2049] [node 2] ICS09 handshake done sender: [6:7587241479434574522:2112] self: [6:7587241479434574539:0] peer: [2:7587241482276144574:0] socket: 623 qp: -1
2025-12-24T02:11:57.515126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241479434574549:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.515139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.515904Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099307:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.515977Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241482620099218:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241482620099278:2107] held: no
2025-12-24T02:11:57.516129Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482620099278:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.516204Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482620099291:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.516422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482620099293:2114] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.516435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.516442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482620099293:2114] [node 4] ICP22 created new session: [5:7587241482620099308:2114]
2025-12-24T02:11:57.516448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482620099308:2114] [node 4] ICS09 handshake done sender: [5:7587241482620099291:2113] self: [5:7587241482620099304:0] peer: [4:7587241481319086845:0] socket: 612 qp: -1
2025-12-24T02:11:57.516454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482620099308:2114] [node 4] ICS10 traffic start
2025-12-24T02:11:57.516519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.517852Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241482620099219:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32435) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241482620099296:2116] held: no
2025-12-24T02:11:57.517966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482620099218:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.517975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.517982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482620099218:2048] [node 1] ICP22 created new session: [5:7587241482620099311:2048]
2025-12-24T02:11:57.517987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482620099311:2048] [node 1] ICS09 handshake done sender: [5:7587241482620099278:2107] self: [5:7587241482620099301:0] peer: [1:7587241478737191348:0] socket: 591 qp: -1
2025-12-24T02:11:57.517993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482620099311:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.517999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.518391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099310:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.518832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482620099296:2116] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.519762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086869:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.520033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241481319086836:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.520142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481319086775:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.520164Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.520170Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481319086775:2048] [node 1] ICP22 created new session: [4:7587241481319086870:2048]
2025-12-24T02:11:57.520182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481319086870:2048] [node 1] ICS09 handshake done sender: [4:7587241481319086836:2108] self: [4:7587241481319086859:0] peer: [1:7587241478737191346:0] socket: 592 qp: -1
2025-12-24T02:11:57.520189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481319086870:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.520197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.521960Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482276144577:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.522085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482276144503:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.522103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.522117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482276144503:2049] [node 3] ICP22 created new session: [2:7587241482276144604:2049]
2025-12-24T02:11:57.522128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482276144604:2049] [node 3] ICS09 handshake done sender: [2:7587241482276144577:2114] self: [2:7587241482276144576:0] peer: [3:7587241478772683836:0] socket: 638 qp: -1
2025-12-24T02:11:57.522135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482276144604:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.522143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.518891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482620099219:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.518908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.518914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482620099219:2049] [node 2] ICP22 created new session: [5:7587241482620099313:2049]
2025-12-24T02:11:57.518920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482620099313:2049] [node 2] ICS09 handshake done sender: [5:7587241482620099296:2116] self: [5:7587241482620099302:0] peer: [2:7587241482276144572:0] socket: 617 qp: -1
2025-12-24T02:11:57.518926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482620099313:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.518933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.519955Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482620099307:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.520001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482620099220:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.522852Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683838:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.523119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478772683834:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.523205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241478772683747:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.523224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.523231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478772683747:2049] [node 2] ICP22 created new session: [3:7587241478772683839:2049]
2025-12-24T02:11:57.523238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478772683839:2049] [node 2] ICS09 handshake done sender: [3:7587241478772683834:2118] self: [3:7587241478772683836:0] peer: [2:7587241482276144576:0] socket: 649 qp: -1
2025-12-24T02:11:57.523246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478772683839:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.523257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.524139Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683827:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.524160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478772683827:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.525503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478772683827:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.525962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478772683821:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.525985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.525992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478772683821:2111] [node 5] ICP22 created new session: [3:7587241478772683843:2111]
2025-12-24T02:11:57.524995Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241479434574452:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241479434574540:2118] held: no
2025-12-24T02:11:57.525999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478772683843:2111] [node 5] ICS09 handshake done sender: [3:7587241478772683827:2116] self: [3:7587241478772683822:0] peer: [5:7587241482620099315:0] socket: 600 qp: -1
2025-12-24T02:11:57.526011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478772683843:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:57.526017Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.525212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479434574454:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.525225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.525232Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479434574454:2050] [node 3] ICP22 created new session: [6:7587241479434574551:2050]
2025-12-24T02:11:57.525238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241479434574551:2050] [node 3] ICS09 handshake done sender: [6:7587241479434574523:2113] self: [6:7587241479434574529:0] peer: [3:7587241478772683819:0] socket: 624 qp: -1
2025-12-24T02:11:57.525245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241479434574551:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.525251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.525446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241479434574454:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.524928Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482620099316:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:35601
2025-12-24T02:11:57.525726Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482620099307:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.525931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482620099220:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.525946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.525953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482620099220:2050] [node 3] ICP22 created new session: [5:7587241482620099317:2050]
2025-12-24T02:11:57.525959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482620099317:2050] [node 3] ICS09 handshake done sender: [5:7587241482620099307:2121] self: [5:7587241482620099315:0] peer: [3:7587241478772683822:0] socket: 655 qp: -1
2025-12-24T02:11:57.525966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482620099317:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.525975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.526996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241479434574548:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52028
2025-12-24T02:11:57.527257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241482276144568:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.528096Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241479434574540:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.528193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479434574452:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.528205Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.528211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479434574452:2048] [node 1] ICP22 created new session: [6:7587241479434574553:2048]
2025-12-24T02:11:57.528217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241479434574553:2048] [node 1] ICS09 handshake done sender: [6:7587241479434574540:2118] self: [6:7587241479434574547:0] peer: [1:7587241478737191353:0] socket: 630 qp: -1
2025-12-24T02:11:57.528222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241479434574553:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.528229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.532657Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241478737191356:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.532676Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241478737191356:2114] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.534155Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.534393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241481319086872:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.534429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481319086777:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.531218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683837:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.531271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241478772683746:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.533986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241478772683846:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7005
2025-12-24T02:11:57.534535Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241478772683829:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.534539Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241478772683829:2117] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.534877Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241478737191356:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.535386Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241481319086874:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21309
2025-12-24T02:11:57.535012Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241478737191279:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.535037Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.535045Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241478737191279:2049] [node 3] ICP22 created new session: [1:7587241478737191381:2049]
2025-12-24T02:11:57.535054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241478737191381:2049] [node 3] ICS09 handshake done sender: [1:7587241478737191356:2114] self: [1:7587241478737191355:0] peer: [3:7587241478772683845:0] socket: 656 qp: -1
2025-12-24T02:11:57.535061Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241478737191381:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.535085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.535722Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241481319086872:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.535795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481319086777:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.535808Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.535812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481319086777:2050] [node 3] ICP22 created new session: [4:7587241481319086875:2050]
2025-12-24T02:11:57.535817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241481319086875:2050] [node 3] ICS09 handshake done sender: [4:7587241481319086872:2120] self: [4:7587241481319086873:0] peer: [3:7587241478772683816:0] socket: 634 qp: -1
2025-12-24T02:11:57.535823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241481319086875:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.535830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.535571Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478772683829:2117] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.536864Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241478772683746:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:11726) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241478772683837:2119] held: no
2025-12-24T02:11:57.536969Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241478772683837:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.538837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241478772683811:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.538863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.538871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478772683811:2109] [node 4] ICP22 created new session: [3:7587241478772683848:2109]
2025-12-24T02:11:57.538877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478772683848:2109] [node 4] ICS09 handshake done sender: [3:7587241478772683829:2117] self: [3:7587241478772683816:0] peer: [4:7587241481319086873:0] socket: 626 qp: -1
2025-12-24T02:11:57.538884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478772683848:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:57.538899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.539212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241478772683746:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.539236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.539247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241478772683746:2048] [node 1] ICP22 created new session: [3:7587241478772683850:2048]
2025-12-24T02:11:57.539254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241478772683850:2048] [node 1] ICS09 handshake done sender: [3:7587241478772683837:2119] self: [3:7587241478772683845:0] peer: [1:7587241478737191355:0] socket: 631 qp: -1
2025-12-24T02:11:57.539261Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241478772683850:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.539270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.558831Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:57.558952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241482276144594:2048] [node 1] ICS01 socket: 594 reason# EndOfStream
2025-12-24T02:11:57.558965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241482276144502:2048] [node 1] ICP30 unregister session Session# [2:7587241482276144594:2048] VirtualId# [2:7587241482276144582:0]
2025-12-24T02:11:57.558974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241482276144594:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.559042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.559074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.562202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:57.562305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241481319086870:2048] [node 1] ICS01 socket: 592 reason# EndOfStream
2025-12-24T02:11:57.562316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241481319086775:2048] [node 1] ICP30 unregister session Session# [4:7587241481319086870:2048] VirtualId# [4:7587241481319086859:0]
2025-12-24T02:11:57.562325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241481319086870:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.562384Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.562420Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.562547Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:57.562638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482620099311:2048] [node 1] ICS01 socket: 591 reason# EndOfStream
2025-12-24T02:11:57.562646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482620099218:2048] [node 1] ICP30 unregister session Session# [5:7587241482620099311:2048] VirtualId# [5:7587241482620099301:0]
2025-12-24T02:11:57.562653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482620099311:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.562732Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.562762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.559124Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:57.559205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241478772683850:2048] [node 1] ICS01 socket: 631 reason# EndOfStream
2025-12-24T02:11:57.559210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241478772683746:2048] [node 1] ICP30 unregister session Session# [3:7587241478772683850:2048] VirtualId# [3:7587241478772683845:0]
2025-12-24T02:11:57.559215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241478772683850:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.559243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.559260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.562678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:57.562777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241479434574553:2048] [node 1] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:57.562783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479434574452:2048] [node 1] ICP30 unregister session Session# [6:7587241479434574553:2048] VirtualId# [6:7587241479434574547:0]
2025-12-24T02:11:57.562787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241479434574553:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.562826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.562850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.575533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:57.575643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241478772683839:2049] [node 2] ICS01 socket: 649 reason# EndOfStream
2025-12-24T02:11:57.575655Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241478772683747:2049] [node 2] ICP30 unregister session Session# [3:7587241478772683839:2049] VirtualId# [3:7587241478772683836:0]
2025-12-24T02:11:57.575665Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241478772683839:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.575715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.575743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.578833Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:57.578959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482620099313:2049] [node 2] ICS01 socket: 617 reason# EndOfStream
2025-12-24T02:11:57.578971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482620099219:2049] [node 2] ICP30 unregister session Session# [5:7587241482620099313:2049] VirtualId# [5:7587241482620099302:0]
2025-12-24T02:11:57.578979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482620099313:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.579038Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.579066Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.582464Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:57.582568Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241481319086867:2049] [node 2] ICS01 socket: 641 reason# EndOfStream
2025-12-24T02:11:57.582581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241481319086776:2049] [node 2] ICP30 unregister session Session# [4:7587241481319086867:2049] VirtualId# [4:7587241481319086865:0]
2025-12-24T02:11:57.582590Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241481319086867:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.582661Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.582714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.582768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:57.582845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241479434574549:2049] [node 2] ICS07 socket disconnect 623 reason# EndOfStream
2025-12-24T02:11:57.582850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241479434574549:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.582886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.582913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241479434574549:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:57.583515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241479434574453:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32435) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:57.583529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241479434574549:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:57.583535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479434574453:2049] [node 2] ICP30 unregister session Session# [6:7587241479434574549:2049] VirtualId# [6:7587241479434574539:0]
2025-12-24T02:11:57.583594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.583602Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241479434574453:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:32435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.583605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:32435) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.593348Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241479434574453:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:57.596842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:57.596948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482620099317:2050] [node 3] ICS01 socket: 655 reason# EndOfStream
2025-12-24T02:11:57.596961Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482620099220:2050] [node 3] ICP30 unregister session Session# [5:7587241482620099317:2050] VirtualId# [5:7587241482620099315:0]
2025-12-24T02:11:57.596968Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482620099317:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.597024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.597056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.598887Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:57.599012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241479434574551:2050] [node 3] ICS01 socket: 624 reason# EndOfStream
2025-12-24T02:11:57.599023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479434574454:2050] [node 3] ICP30 unregister session Session# [6:7587241479434574551:2050] VirtualId# [6:7587241479434574529:0]
2025-12-24T02:11:57.599038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241479434574551:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.599098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.599126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.612314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:57.612431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241479434574542:2115] [node 4] ICS01 socket: 621 reason# EndOfStream
2025-12-24T02:11:57.612446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479434574533:2115] [node 4] ICP30 unregister session Session# [6:7587241479434574542:2115] VirtualId# [6:7587241479434574536:0]
2025-12-24T02:11:57.612455Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241479434574542:2115] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.612509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:57.612536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:57.625406Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:57.625528Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241479434574531:2103] [node 5] ICS01 socket: 597 reason# EndOfStream
2025-12-24T02:11:57.625541Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479434574509:2103] [node 5] ICP30 unregister session Session# [6:7587241479434574531:2103] VirtualId# [6:7587241479434574513:0]
2025-12-24T02:11:57.625550Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241479434574531:2103] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.625618Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:57.625651Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:57.655117Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479272791774:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.658774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479272791775:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.660938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241482012057815:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.660956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241482012057817:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.661055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241482012057816:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.651192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241479609946057:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.651217Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241479609946056:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:57.653305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482110125684:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.653319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482110125685:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.666434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241481752378014:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.666442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241481752378015:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.666448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241481752378016:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.663032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482367019679:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.663042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482367019680:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.663046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482367019681:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.679679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241482012057876:2107] [node 6] ICP01 ready to work
2025-12-24T02:11:57.680956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241481752378072:2103] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.682260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482367019739:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:57.682493Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019740:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.686991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482110125745:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.688191Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791833:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.686885Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241479609946122:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:57.687052Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241479609946123:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:57.690872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479272791841:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:57.691023Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479272791843:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:57.690927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241482367019746:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:57.695049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482110125748:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:57.695222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241482110125749:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.697880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378086:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.697905Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378087:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.697917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378088:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.702999Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241479609946057:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(19955067ns) {Configure:104(19954397ns) {ProcessConfigured:135(18183437ns) {StartInitialHandshake:157(18182731ns) {DropHandshakes:470(905ns) {DropIncomingHandshake:442(196ns) DropOutgoingHandshake:457(107ns)} GenerateSessionVirtualId:988(261ns) PrepareNewSessionHandshake:477(1803ns) {SwitchToState:179(1207ns)}}}}}
2025-12-24T02:11:57.703229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241479609946131:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-12-24T02:11:57.704788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479272791839:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:57.705706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241482012057885:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:57.705149Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241479609946121:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:57.707010Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241481752378088:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.707056Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241481752378086:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.707192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241481752378091:2115] [node 4] ICP01 ready to work
2025-12-24T02:11:57.707559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241481752378091:2115] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.714844Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479272791775:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(25708541ns) {Configure:104(25707929ns) {ProcessConfigured:135(25462511ns) {StartInitialHandshake:157(25461663ns) {DropHandshakes:470(281ns) {DropIncomingHandshake:442(66ns) DropOutgoingHandshake:457(58ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(2494ns) {SwitchToState:179(1614ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.716469Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241481752378087:2113] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.716597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791833:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.718909Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479272791841:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(27814796ns) {Configure:104(27813349ns) {ProcessConfigured:135(27495774ns) {StartInitialHandshake:157(27494996ns) {DropHandshakes:470(638ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(188ns)} GenerateSessionVirtualId:988(306ns) PrepareNewSessionHandshake:477(1153ns) {SwitchToState:179(336ns)}}}}}
2025-12-24T02:11:57.716109Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241479609946134:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.727284Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241481752378015:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.727331Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241481752378015:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(20259871ns) {IncomingHandshake:261(20258857ns) {DropIncomingHandshake:442(239ns) PrepareNewSessionHandshake:477(666ns) {SwitchToState:179(88ns)} GenerateSessionVirtualId:988(710ns) IssueIncomingHandshakeReply:193(20250201ns) {SwitchToState:179(709ns)}}}
2025-12-24T02:11:57.727483Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791853:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.728620Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057878:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.728636Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241482012057878:2108] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.729768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057888:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.729837Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057889:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.729885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057890:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.730142Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241482012057885:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(24266043ns) {Configure:104(24265180ns) {ProcessConfigured:135(23849367ns) {StartInitialHandshake:157(23848812ns) {DropHandshakes:470(525ns) {DropIncomingHandshake:442(81ns) DropOutgoingHandshake:457(68ns)} GenerateSessionVirtualId:988(304ns) PrepareNewSessionHandshake:477(1552ns) {SwitchToState:179(605ns)}}}}}
2025-12-24T02:11:57.727811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946134:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.730102Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241479609946121:2107] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(24809738ns) {Configure:104(24808686ns) {ProcessConfigured:135(24356138ns) {StartInitialHandshake:157(24355565ns) {DropHandshakes:470(1046ns) {DropIncomingHandshake:442(208ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(334ns) PrepareNewSessionHandshake:477(1704ns) {SwitchToState:179(855ns)}}}}}
2025-12-24T02:11:57.726731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125755:2112] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.730837Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791853:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.730884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479272791774:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.731119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241482012057878:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.731221Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057890:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.732467Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241479272791774:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241479272791853:2115] held: no
2025-12-24T02:11:57.726770Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482110125755:2112] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.730790Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241482110125685:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(35448704ns) {Configure:104(35446715ns) {ProcessConfigured:135(28127746ns) {StartInitialHandshake:157(28126515ns) {DropHandshakes:470(1043ns) {DropIncomingHandshake:442(50ns) DropOutgoingHandshake:457(266ns)} GenerateSessionVirtualId:988(552ns) PrepareNewSessionHandshake:477(862ns) {SwitchToState:179(621ns)}}}}}
2025-12-24T02:11:57.731163Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946130:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.731178Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241479609946130:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.731935Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241479609946130:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.732050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241479609946057:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.732079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.732090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241479609946057:2049] [node 3] ICP22 created new session: [1:7587241479609946138:2049]
2025-12-24T02:11:57.732099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241479609946138:2049] [node 3] ICS09 handshake done sender: [1:7587241479609946130:2112] self: [1:7587241479609946119:0] peer: [3:7587241479272791854:0] socket: 619 qp: -1
2025-12-24T02:11:57.732108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241479609946138:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.732132Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.732535Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241482012057817:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25788433ns) {Configure:104(25787317ns) {ProcessConfigured:135(25497990ns) {StartInitialHandshake:157(25497167ns) {DropHandshakes:470(270ns) {DropIncomingHandshake:442(53ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(653ns) {SwitchToState:179(409ns)}}}}}
2025-12-24T02:11:57.732569Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241482012057876:2107] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.732578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.732586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241482012057876:2107] [node 6] ICP22 created new session: [4:7587241482012057897:2107]
2025-12-24T02:11:57.732591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241482012057897:2107] [node 6] ICS09 handshake done sender: [4:7587241482012057878:2108] self: [4:7587241482012057877:0] peer: [6:7587241481752378092:0] socket: 594 qp: -1
2025-12-24T02:11:57.732598Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241482012057897:2107] [node 6] ICS10 traffic start
2025-12-24T02:11:57.732614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.735201Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057889:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.735281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241482012057816:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.735360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241482012057817:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.735428Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057899:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.735715Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057899:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.735755Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241482012057815:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.737023Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057904:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.737288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241482012057899:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.737339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241482012057815:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.737355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.737361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241482012057815:2048] [node 1] ICP22 created new session: [4:7587241482012057905:2048]
2025-12-24T02:11:57.737366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241482012057905:2048] [node 1] ICS09 handshake done sender: [4:7587241482012057899:2118] self: [4:7587241482012057903:0] peer: [1:7587241479609946133:0] socket: 634 qp: -1
2025-12-24T02:11:57.737371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241482012057905:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.737379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.735275Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946131:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.735831Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946136:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.735835Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241479609946136:2115] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.736445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241479609946136:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.736554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241479609946121:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.736576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.736586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241479609946121:2107] [node 4] ICP22 created new session: [1:7587241479609946143:2107]
2025-12-24T02:11:57.736594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241479609946143:2107] [node 4] ICS09 handshake done sender: [1:7587241479609946136:2115] self: [1:7587241479609946133:0] peer: [4:7587241482012057903:0] socket: 624 qp: -1
2025-12-24T02:11:57.736603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241479609946143:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:57.736615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.739243Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791855:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.739075Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482110125762:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-12-24T02:11:57.743059Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125762:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.742731Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378094:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.742980Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241481752378086:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.746790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241481752378091:2115] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.746812Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.746833Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241481752378091:2115] [node 4] ICP22 created new session: [6:7587241481752378095:2115]
2025-12-24T02:11:57.746841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241481752378095:2115] [node 4] ICS09 handshake done sender: [6:7587241481752378086:2112] self: [6:7587241481752378092:0] peer: [4:7587241482012057877:0] socket: 599 qp: -1
2025-12-24T02:11:57.746852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241481752378095:2115] [node 4] ICS10 traffic start
2025-12-24T02:11:57.746898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.750337Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241482367019746:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(59330294ns) {Configure:104(59329732ns) {ProcessConfigured:135(58941775ns) {StartInitialHandshake:157(58941117ns) {DropHandshakes:470(698ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(1731ns) {SwitchToState:179(1033ns)}}}}}
2025-12-24T02:11:57.750065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482110125765:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-12-24T02:11:57.750156Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482110125766:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.750889Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019740:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.751105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019755:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.751170Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019753:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.751178Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019754:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.750265Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241479609946056:2048] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(67151809ns) {Configure:104(67150942ns) {ProcessConfigured:135(65373701ns) {StartInitialHandshake:157(65372767ns) {DropHandshakes:470(317ns) {DropIncomingHandshake:442(17ns) DropOutgoingHandshake:457(117ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(1351ns) {SwitchToState:179(348ns)}}}}}
2025-12-24T02:11:57.751728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482110125755:2112] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.751847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482110125749:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.751865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.751871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482110125749:2109] [node 6] ICP22 created new session: [2:7587241482110125768:2109]
2025-12-24T02:11:57.751877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482110125768:2109] [node 6] ICS09 handshake done sender: [2:7587241482110125755:2112] self: [2:7587241482110125754:0] peer: [6:7587241481752378089:0] socket: 613 qp: -1
2025-12-24T02:11:57.751885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482110125768:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.751897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.754871Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241482367019680:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(28021080ns) {Configure:104(28018704ns) {ProcessConfigured:135(27546671ns) {StartInitialHandshake:157(27544962ns) {DropHandshakes:470(1417ns) {DropIncomingHandshake:442(222ns) DropOutgoingHandshake:457(222ns)} GenerateSessionVirtualId:988(620ns) PrepareNewSessionHandshake:477(1212ns) {SwitchToState:179(949ns)}}}}}
2025-12-24T02:11:57.754935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019754:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.755427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482367019680:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.756257Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019753:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.756161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057888:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.756595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791848:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.756614Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479272791848:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.756676Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019763:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.756761Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241482367019679:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(27917320ns) {Configure:104(27916190ns) {ProcessConfigured:135(27535309ns) {StartInitialHandshake:157(27534843ns) {DropHandshakes:470(410ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(65ns)} GenerateSessionVirtualId:988(210ns) PrepareNewSessionHandshake:477(1418ns) {SwitchToState:179(820ns)}}}}}
2025-12-24T02:11:57.756794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482367019679:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.756841Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019755:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.756905Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482367019754:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.755001Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241479609946147:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-12-24T02:11:57.755554Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125751:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.755028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241479609946148:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-12-24T02:11:57.756418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946148:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.755566Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482110125751:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.756277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482110125751:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.756622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125753:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.756631Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482110125753:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.757068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482110125745:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.757083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.757092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482110125745:2107] [node 5] ICP22 created new session: [2:7587241482110125772:2107]
2025-12-24T02:11:57.757099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482110125772:2107] [node 5] ICS09 handshake done sender: [2:7587241482110125751:2110] self: [2:7587241482110125750:0] peer: [5:7587241482367019762:0] socket: 611 qp: -1
2025-12-24T02:11:57.757107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482110125772:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:57.757118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.759182Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479272791843:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(67959569ns) {Configure:104(67958161ns) {ProcessConfigured:135(67653508ns) {StartInitialHandshake:157(67652673ns) {DropHandshakes:470(563ns) {DropIncomingHandshake:442(78ns) DropOutgoingHandshake:457(57ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(1286ns) {SwitchToState:179(758ns)}}}}}
2025-12-24T02:11:57.759214Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479272791853:2115] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.759377Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791858:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.759392Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479272791774:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.759406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.759413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479272791774:2048] [node 1] ICP22 created new session: [3:7587241479272791861:2048]
2025-12-24T02:11:57.759422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479272791861:2048] [node 1] ICS09 handshake done sender: [3:7587241479272791853:2115] self: [3:7587241479272791854:0] peer: [1:7587241479609946119:0] socket: 620 qp: -1
2025-12-24T02:11:57.759430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479272791861:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.759437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.760477Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241482012057876:2107] [node 6] ICP35 event processing took too much time PendingConnection:279(27688603ns) {HandleHandshakeStatus:329(27688022ns) {DropIncomingHandshake:442(69ns) DropOutgoingHandshake:457(134ns) SwitchToState:179(63ns) ProcessPendingSessionEvents:495(20812006ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 20789005ns max# 20631691ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.758842Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125766:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.758924Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482110125774:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-12-24T02:11:57.759953Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125765:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.762597Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241481752378015:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241481752378088:2114] held: no
2025-12-24T02:11:57.762818Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378097:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.762833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378098:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.763481Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482367019680:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.763505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.763514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482367019680:2049] [node 2] ICP22 created new session: [5:7587241482367019765:2049]
2025-12-24T02:11:57.763522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482367019765:2049] [node 2] ICS09 handshake done sender: [5:7587241482367019754:2115] self: [5:7587241482367019762:0] peer: [2:7587241482110125750:0] socket: 639 qp: -1
2025-12-24T02:11:57.763530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482367019765:2049] [node 2] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.763553Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.764078Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019767:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.764212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482367019739:2106] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.764358Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791863:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.764735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791857:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.764937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791857:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.764683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019767:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.765720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241482367019681:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.765900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791858:2119] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.765939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479272791775:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.765151Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241479609946121:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(28480760ns) {HandleHandshakeStatus:329(28479867ns) {DropIncomingHandshake:442(5047ns) DropOutgoingHandshake:457(128ns) SwitchToState:179(125ns) ProcessPendingSessionEvents:495(28345720ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 28320733ns max# 28221466ns}}}
2025-12-24T02:11:57.765728Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946147:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.764658Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241482012057895:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.764671Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241482012057895:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.768651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125774:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.768770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241482110125684:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.768872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482110125753:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.768850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057910:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.769081Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241482012057889:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.768106Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479272791848:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.769180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241482012057816:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.769202Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.769214Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241482012057816:2049] [node 2] ICP22 created new session: [4:7587241482012057911:2049]
2025-12-24T02:11:57.769222Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241482012057911:2049] [node 2] ICS09 handshake done sender: [4:7587241482012057889:2113] self: [4:7587241482012057900:0] peer: [2:7587241482110125752:0] socket: 627 qp: -1
2025-12-24T02:11:57.769230Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241482012057911:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.769240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.769562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482110125748:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.769585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.769594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482110125748:2108] [node 4] ICP22 created new session: [2:7587241482110125780:2108]
2025-12-24T02:11:57.769601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482110125780:2108] [node 4] ICS09 handshake done sender: [2:7587241482110125753:2111] self: [2:7587241482110125752:0] peer: [4:7587241482012057900:0] socket: 612 qp: -1
2025-12-24T02:11:57.769607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482110125780:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:57.769615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.768987Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241481752378088:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.766839Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946125:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.768990Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241481752378014:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(61303610ns) {Configure:104(61302402ns) {ProcessConfigured:135(61001090ns) {StartInitialHandshake:157(60999955ns) {DropHandshakes:470(435ns) {DropIncomingHandshake:442(60ns) DropOutgoingHandshake:457(89ns)} GenerateSessionVirtualId:988(222ns) PrepareNewSessionHandshake:477(1078ns) {SwitchToState:179(112ns)}}}}}
2025-12-24T02:11:57.769156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241481752378014:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.769830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241481752378015:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.769843Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.769851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241481752378015:2049] [node 2] ICP22 created new session: [6:7587241481752378101:2049]
2025-12-24T02:11:57.769857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241481752378101:2049] [node 2] ICS09 handshake done sender: [6:7587241481752378088:2114] self: [6:7587241481752378089:0] peer: [2:7587241482110125754:0] socket: 615 qp: -1
2025-12-24T02:11:57.769863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241481752378101:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.769872Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.770100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241481752378097:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.770187Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241481752378072:2103] [node 5] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.771238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479272791839:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.771263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.771271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479272791839:2108] [node 4] ICP22 created new session: [3:7587241479272791867:2108]
2025-12-24T02:11:57.771279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479272791867:2108] [node 4] ICS09 handshake done sender: [3:7587241479272791848:2113] self: [3:7587241479272791847:0] peer: [4:7587241482012057901:0] socket: 623 qp: -1
2025-12-24T02:11:57.771284Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479272791867:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:57.771303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.766856Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241479609946125:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.770899Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241482367019752:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.769295Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241479609946125:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.769877Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946127:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.770912Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241482367019752:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.771723Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482367019752:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.769881Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241479609946127:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.772623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241482367019746:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.772641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.772648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482367019746:2110] [node 6] ICP22 created new session: [5:7587241482367019772:2110]
2025-12-24T02:11:57.772655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482367019772:2110] [node 6] ICS09 handshake done sender: [5:7587241482367019752:2113] self: [5:7587241482367019748:0] peer: [6:7587241481752378103:0] socket: 642 qp: -1
2025-12-24T02:11:57.772661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482367019772:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:57.772668Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.774876Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378105:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.774860Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791852:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.774883Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479272791852:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.775024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241482012057909:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:29527
2025-12-24T02:11:57.776477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241482367019679:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241482367019753:2114] held: no
2025-12-24T02:11:57.776746Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241481752378087:2113] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.770359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241479609946122:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.770381Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.770389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241479609946122:2108] [node 5] ICP22 created new session: [1:7587241479609946152:2108]
2025-12-24T02:11:57.774886Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241482110125763:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.774905Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241482110125763:2115] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.775357Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241482110125745:2107] [node 5] ICP35 event processing took too much time PendingConnection:279(18174046ns) {HandleHandshakeStatus:329(18172419ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(269ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(8425308ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 8398620ns max# 8311964ns}}}
2025-12-24T02:11:57.778822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241481752378014:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.778847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.770396Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241479609946152:2108] [node 5] ICS09 handshake done sender: [1:7587241479609946125:2110] self: [1:7587241479609946124:0] peer: [5:7587241482367019764:0] socket: 604 qp: -1
2025-12-24T02:11:57.770403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241479609946152:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:57.770412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.781137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482110125763:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.782944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479272791852:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.783104Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241482012057890:2114] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.783927Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019774:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.784150Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479272791841:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.784168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.784177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479272791841:2109] [node 5] ICP22 created new session: [3:7587241479272791871:2109]
2025-12-24T02:11:57.784183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479272791871:2109] [node 5] ICS09 handshake done sender: [3:7587241479272791852:2114] self: [3:7587241479272791844:0] peer: [5:7587241482367019770:0] socket: 653 qp: -1
2025-12-24T02:11:57.784190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479272791871:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:57.784213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.784920Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241482012057817:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241482012057890:2114] held: no
2025-12-24T02:11:57.785014Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241482012057817:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.785024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.785030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241482012057817:2050] [node 3] ICP22 created new session: [4:7587241482012057914:2050]
2025-12-24T02:11:57.785036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241482012057914:2050] [node 3] ICS09 handshake done sender: [4:7587241482012057890:2114] self: [4:7587241482012057901:0] peer: [3:7587241479272791847:0] socket: 628 qp: -1
2025-12-24T02:11:57.785042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241482012057914:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.785049Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.785453Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791863:2120] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.785495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241479272791841:2109] [node 5] ICP18 handshake (actor [3:7587241479272791863:2120]) is too old
2025-12-24T02:11:57.786123Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241482012057895:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.786213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241482012057885:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.786229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.786235Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241482012057885:2111] [node 5] ICP22 created new session: [4:7587241482012057916:2111]
2025-12-24T02:11:57.786240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241482012057916:2111] [node 5] ICS09 handshake done sender: [4:7587241482012057895:2116] self: [4:7587241482012057886:0] peer: [5:7587241482367019769:0] socket: 635 qp: -1
2025-12-24T02:11:57.786244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241482012057916:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:57.786251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.786996Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019775:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.789215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241479272791856:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.789229Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241479272791856:2117] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.790824Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479272791856:2117] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.778856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241481752378014:2048] [node 1] ICP22 created new session: [6:7587241481752378106:2048]
2025-12-24T02:11:57.778864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241481752378106:2048] [node 1] ICS09 handshake done sender: [6:7587241481752378087:2113] self: [6:7587241481752378100:0] peer: [1:7587241479609946126:0] socket: 614 qp: -1
2025-12-24T02:11:57.778872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241481752378106:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.778882Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.783878Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241481752378072:2103] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:23992) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241481752378097:2117] held: no
2025-12-24T02:11:57.784085Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378108:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.785366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241481752378097:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:57.785643Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241481752378072:2103] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.785652Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:57.785657Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241481752378072:2103] [node 5] ICP22 created new session: [6:7587241481752378109:2103]
2025-12-24T02:11:57.785663Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241481752378109:2103] [node 5] ICS09 handshake done sender: [6:7587241481752378097:2117] self: [6:7587241481752378103:0] peer: [5:7587241482367019748:0] socket: 646 qp: -1
2025-12-24T02:11:57.785669Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241481752378109:2103] [node 5] ICS10 traffic start
2025-12-24T02:11:57.785675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:57.788683Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378111:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.789068Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241481752378111:2122] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.789101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241481752378016:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:57.789951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241481752378113:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:12079
2025-12-24T02:11:57.790761Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241482367019681:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15738) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241482367019767:2120] held: no
2025-12-24T02:11:57.790898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482367019753:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.791080Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241481752378111:2122] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.791135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241481752378016:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.791151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.791157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241481752378016:2050] [node 3] ICP22 created new session: [6:7587241481752378114:2050]
2025-12-24T02:11:57.791164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241481752378114:2050] [node 3] ICS09 handshake done sender: [6:7587241481752378111:2122] self: [6:7587241481752378112:0] peer: [3:7587241479272791845:0] socket: 597 qp: -1
2025-12-24T02:11:57.791170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241481752378114:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.791177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.770873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241479609946127:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:57.774948Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241479609946145:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:57.774960Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241479609946145:2116] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:57.776766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241479609946123:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.776779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.776788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241479609946123:2109] [node 6] ICP22 created new session: [1:7587241479609946155:2109]
2025-12-24T02:11:57.776796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241479609946155:2109] [node 6] ICS09 handshake done sender: [1:7587241479609946127:2111] self: [1:7587241479609946126:0] peer: [6:7587241481752378100:0] socket: 605 qp: -1
2025-12-24T02:11:57.776803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241479609946155:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:57.776827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:57.782134Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241479609946145:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.782247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241479609946056:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.782263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.782271Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241479609946056:2048] [node 2] ICP22 created new session: [1:7587241479609946158:2048]
2025-12-24T02:11:57.782277Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241479609946158:2048] [node 2] ICS09 handshake done sender: [1:7587241479609946145:2116] self: [1:7587241479609946120:0] peer: [2:7587241482110125779:0] socket: 621 qp: -1
2025-12-24T02:11:57.782282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241479609946158:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:57.782289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.790951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241479609946160:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34858
2025-12-24T02:11:57.794798Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241479609946123:2109] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:57.795410Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241479272791839:2108] [node 4] ICP35 event processing took too much time PendingConnection:279(24066893ns) {HandleHandshakeStatus:329(24066215ns) {DropIncomingHandshake:442(4328ns) DropOutgoingHandshake:457(138ns) SwitchToState:179(70ns) ProcessPendingSessionEvents:495(20286023ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7824230ns max# 7732563ns}}}
2025-12-24T02:11:57.795556Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241479272791775:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:14944) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241479272791858:2119] held: no
2025-12-24T02:11:57.795674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241479272791843:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.795693Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:57.795699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479272791843:2110] [node 6] ICP22 created new session: [3:7587241479272791874:2110]
2025-12-24T02:11:57.795704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479272791874:2110] [node 6] ICS09 handshake done sender: [3:7587241479272791856:2117] self: [3:7587241479272791845:0] peer: [6:7587241481752378112:0] socket: 618 qp: -1
2025-12-24T02:11:57.795710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479272791874:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:57.795718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:57.781269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241482110125685:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:57.781288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.781308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482110125685:2049] [node 3] ICP22 created new session: [2:7587241482110125783:2049]
2025-12-24T02:11:57.781317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482110125783:2049] [node 3] ICS09 handshake done sender: [2:7587241482110125763:2115] self: [2:7587241482110125761:0] peer: [3:7587241479272791865:0] socket: 650 qp: -1
2025-12-24T02:11:57.781324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482110125783:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:57.781332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.783214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241482110125684:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10888) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241482110125774:2118] held: no
2025-12-24T02:11:57.786902Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241482110125785:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:24634
2025-12-24T02:11:57.788051Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241482110125774:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:57.788488Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241482110125684:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.788500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.788508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241482110125684:2048] [node 1] ICP22 created new session: [2:7587241482110125786:2048]
2025-12-24T02:11:57.788515Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241482110125786:2048] [node 1] ICS09 handshake done sender: [2:7587241482110125774:2118] self: [2:7587241482110125779:0] peer: [1:7587241479609946120:0] socket: 638 qp: -1
2025-12-24T02:11:57.788523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241482110125786:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.788529Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.799039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241482367019776:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47197
2025-12-24T02:11:57.799282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482367019679:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.799303Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:57.799312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482367019679:2048] [node 1] ICP22 created new session: [5:7587241482367019777:2048]
2025-12-24T02:11:57.799320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482367019777:2048] [node 1] ICS09 handshake done sender: [5:7587241482367019753:2114] self: [5:7587241482367019764:0] peer: [1:7587241479609946124:0] socket: 610 qp: -1
2025-12-24T02:11:57.799329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482367019777:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:57.799339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:57.799674Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241482367019739:2106] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22387) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241482367019755:2116] held: no
2025-12-24T02:11:57.799784Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482367019767:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:57.800196Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241482367019755:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:57.800273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482367019681:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.800287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:57.800293Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482367019681:2050] [node 3] ICP22 created new session: [5:7587241482367019779:2050]
2025-12-24T02:11:57.800299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482367019779:2050] [node 3] ICS09 handshake done sender: [5:7587241482367019767:2120] self: [5:7587241482367019770:0] peer: [3:7587241479272791844:0] socket: 654 qp: -1
2025-12-24T02:11:57.800307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482367019779:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:57.800315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:57.800508Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241482367019739:2106] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.800517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:57.800521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241482367019739:2106] [node 4] ICP22 created new session: [5:7587241482367019781:2106]
2025-12-24T02:11:57.800526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241482367019781:2106] [node 4] ICS09 handshake done sender: [5:7587241482367019755:2116] self: [5:7587241482367019769:0] peer: [4:7587241482012057886:0] socket: 640 qp: -1
2025-12-24T02:11:57.800531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241482367019781:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:57.800537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:57.806828Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241479272791876:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31293
2025-12-24T02:11:57.810984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241479272791858:2119] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:57.811107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479272791775:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:57.811133Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:57.811145Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479272791775:2049] [node 2] ICP22 created new session: [3:7587241479272791877:2049]
2025-12-24T02:11:57.811153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241479272791877:2049] [node 2] ICS09 handshake done sender: [3:7587241479272791858:2119] self: [3:7587241479272791865:0] peer: [2:7587241482110125761:0] socket: 652 qp: -1
2025-12-24T02:11:57.811165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241479272791877:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:57.811196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:57.831379Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:57.831467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241479272791861:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:57.831474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479272791774:2048] [node 1] ICP30 unregister session Session# [3:7587241479272791861:2048] VirtualId# [3:7587241479272791854:0]
2025-12-24T02:11:57.831481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241479272791861:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.831520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.831545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.831052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:57.831176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241482110125786:2048] [node 1] ICS01 socket: 638 reason# EndOfStream
2025-12-24T02:11:57.831189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241482110125684:2048] [node 1] ICP30 unregister session Session# [2:7587241482110125786:2048] VirtualId# [2:7587241482110125779:0]
2025-12-24T02:11:57.831200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241482110125786:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.831267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.831296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.839013Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:57.839127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482367019777:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-12-24T02:11:57.839140Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482367019679:2048] [node 1] ICP30 unregister session Session# [5:7587241482367019777:2048] VirtualId# [5:7587241482367019764:0]
2025-12-24T02:11:57.839150Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482367019777:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.839214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.839250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.842052Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:57.842159Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241481752378106:2048] [node 1] ICS01 socket: 614 reason# EndOfStream
2025-12-24T02:11:57.842168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241481752378014:2048] [node 1] ICP30 unregister session Session# [6:7587241481752378106:2048] VirtualId# [6:7587241481752378100:0]
2025-12-24T02:11:57.842177Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241481752378106:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.842239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.842266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.842950Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:57.843036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241482012057905:2048] [node 1] ICS01 socket: 634 reason# EndOfStream
2025-12-24T02:11:57.843043Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241482012057815:2048] [node 1] ICP30 unregister session Session# [4:7587241482012057905:2048] VirtualId# [4:7587241482012057903:0]
2025-12-24T02:11:57.843051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241482012057905:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.843105Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:57.843153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:57.871007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:57.871144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241479272791877:2049] [node 2] ICS01 socket: 652 reason# EndOfStream
2025-12-24T02:11:57.871158Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479272791775:2049] [node 2] ICP30 unregister session Session# [3:7587241479272791877:2049] VirtualId# [3:7587241479272791865:0]
2025-12-24T02:11:57.871169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241479272791877:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.871244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.871276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.877381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:57.877526Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241481752378101:2049] [node 2] ICS01 socket: 615 reason# EndOfStream
2025-12-24T02:11:57.877538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241481752378015:2049] [node 2] ICP30 unregister session Session# [6:7587241481752378101:2049] VirtualId# [6:7587241481752378089:0]
2025-12-24T02:11:57.877547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241481752378101:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.877625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.877658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.878896Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:57.878983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482367019765:2049] [node 2] ICS01 socket: 639 reason# EndOfStream
2025-12-24T02:11:57.878783Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:57.878991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482367019680:2049] [node 2] ICP30 unregister session Session# [5:7587241482367019765:2049] VirtualId# [5:7587241482367019762:0]
2025-12-24T02:11:57.878998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482367019765:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.878886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241482012057911:2049] [node 2] ICS01 socket: 627 reason# EndOfStream
2025-12-24T02:11:57.879054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.878894Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241482012057816:2049] [node 2] ICP30 unregister session Session# [4:7587241482012057911:2049] VirtualId# [4:7587241482012057900:0]
2025-12-24T02:11:57.878902Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241482012057911:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.878986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:57.879015Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.879082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:57.891150Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:57.891260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241481752378114:2050] [node 3] ICS07 socket disconnect 597 reason# EndOfStream
2025-12-24T02:11:57.891267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241481752378114:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.891332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.891355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241481752378114:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:57.892447Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241481752378016:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15738) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:57.892465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241481752378114:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:57.892471Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241481752378016:2050] [node 3] ICP30 unregister session Session# [6:7587241481752378114:2050] VirtualId# [6:7587241481752378112:0]
2025-12-24T02:11:57.892534Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.892542Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241481752378016:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:15738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.892545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:15738) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:57.891694Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:57.891764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241482367019779:2050] [node 3] ICS01 socket: 654 reason# EndOfStream
2025-12-24T02:11:57.891771Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241482367019681:2050] [node 3] ICP30 unregister session Session# [5:7587241482367019779:2050] VirtualId# [5:7587241482367019770:0]
2025-12-24T02:11:57.891778Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241482367019779:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.891814Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:57.891839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:57.903204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241481752378016:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:57.910236Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:57.910333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241481752378095:2115] [node 4] ICS01 socket: 599 reason# EndOfStream
2025-12-24T02:11:57.910342Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241481752378091:2115] [node 4] ICP30 unregister session Session# [6:7587241481752378095:2115] VirtualId# [6:7587241481752378092:0]
2025-12-24T02:11:57.910353Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241481752378095:2115] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:57.910396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:57.910421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:57.952105Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241481222792997:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:57.952113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241481222792998:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.953831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241480134570073:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.953845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241480134570074:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:57.957752Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481286028409:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.957761Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481286028410:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.957767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241481286028411:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.958891Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479797131325:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.958900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241479797131326:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.971475Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241481852901848:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.971486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241481852901849:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.971491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241481852901850:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:57.975144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479980817490:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:57.975154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479980817491:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:57.975159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241479980817492:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.008694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484275784847:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.013657Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869205:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.015789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485517760358:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:58.020033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241484092098684:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.019404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485517760362:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:58.020163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485517760363:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:58.020060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869205:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.020166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241484092098685:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.020304Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241484092098686:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.022047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241484429537435:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.022208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241484429537434:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.022594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241484429537433:2107] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.023131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098689:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.025119Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486147869207:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.028054Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995775:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.028084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995776:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.028207Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995774:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.027566Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098689:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.030929Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995774:2111] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.031122Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241485580995770:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.031147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241485580995771:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.034762Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537444:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.034907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484275784861:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:58.036501Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537444:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.036559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241480134570073:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.036733Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760354:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.036749Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485517760370:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48139
2025-12-24T02:11:58.036759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485517760354:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.036768Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485517760371:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48139
2025-12-24T02:11:58.036863Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241485517760363:2110] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16569929ns) {Configure:104(16569275ns) {ProcessConfigured:135(16295335ns) {StartInitialHandshake:157(16294645ns) {DropHandshakes:470(384ns) {DropIncomingHandshake:442(42ns) DropOutgoingHandshake:457(69ns)} GenerateSessionVirtualId:988(280ns) PrepareNewSessionHandshake:477(762ns) {SwitchToState:179(251ns)}}}}}
2025-12-24T02:11:58.037456Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537441:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.037468Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241484429537441:2111] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.038125Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241485517760362:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(18557985ns) {Configure:104(18556743ns) {ProcessConfigured:135(17983055ns) {StartInitialHandshake:157(17982220ns) {DropHandshakes:470(1050ns) {DropIncomingHandshake:442(65ns) DropOutgoingHandshake:457(197ns)} GenerateSessionVirtualId:988(345ns) PrepareNewSessionHandshake:477(1123ns) {SwitchToState:179(587ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.038241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241484429537441:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.038093Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784863:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.037183Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241481286028409:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(16606921ns) {Configure:104(16605965ns) {ProcessConfigured:135(16119324ns) {StartInitialHandshake:157(16118905ns) {DropHandshakes:470(472ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(73ns)} GenerateSessionVirtualId:988(276ns) PrepareNewSessionHandshake:477(928ns) {SwitchToState:179(649ns)}}}}}
2025-12-24T02:11:58.037249Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995775:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.037276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481286028409:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.037360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481286028410:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.038182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995784:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.038225Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241484092098685:2108] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(17281941ns) {Configure:104(17280459ns) {ProcessConfigured:135(15610403ns) {StartInitialHandshake:157(15609407ns) {DropHandshakes:470(177ns) {DropIncomingHandshake:442(13ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(204ns) PrepareNewSessionHandshake:477(1102ns) {SwitchToState:179(888ns)}}}}}
2025-12-24T02:11:58.038457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241484429537433:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.038476Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.038483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241484429537433:2107] [node 4] ICP22 created new session: [2:7587241484429537451:2107]
2025-12-24T02:11:58.038489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241484429537451:2107] [node 4] ICS09 handshake done sender: [2:7587241484429537441:2111] self: [2:7587241484429537440:0] peer: [4:7587241485580995782:0] socket: 611 qp: -1
2025-12-24T02:11:58.038498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241484429537451:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.038520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.038867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760370:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.038858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484275784863:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.038905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479980817492:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.040043Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760367:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.040058Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485517760367:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.039008Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098691:2111] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.039017Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241484092098691:2111] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.040302Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479797131326:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16557612ns) {Configure:104(13305162ns) {ProcessConfigured:135(12992203ns) {StartInitialHandshake:157(12991506ns) {DropHandshakes:470(421ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(228ns) PrepareNewSessionHandshake:477(565ns) {SwitchToState:179(92ns)}}}}}
2025-12-24T02:11:58.039391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241485580995775:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.039606Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481286028410:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.039623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.039632Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481286028410:2049] [node 2] ICP22 created new session: [4:7587241485580995786:2049]
2025-12-24T02:11:58.039639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241485580995786:2049] [node 2] ICS09 handshake done sender: [4:7587241485580995775:2112] self: [4:7587241485580995782:0] peer: [2:7587241484429537440:0] socket: 622 qp: -1
2025-12-24T02:11:58.039650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241485580995786:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.039673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.046900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760371:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.047668Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537453:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.047689Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537454:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.047434Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241485580995771:2109] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16151658ns) {Configure:104(16150334ns) {ProcessConfigured:135(15534608ns) {StartInitialHandshake:157(15533728ns) {DropHandshakes:470(419ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(319ns) PrepareNewSessionHandshake:477(13383ns) {SwitchToState:179(905ns)}}}}}
2025-12-24T02:11:58.049481Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995776:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.049554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241481286028411:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.047567Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098700:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
2025-12-24T02:11:58.049802Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241479797131325:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(23009139ns) {Configure:104(23007861ns) {ProcessConfigured:135(22622930ns) {StartInitialHandshake:157(22622386ns) {DropHandshakes:470(465ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(66ns)} GenerateSessionVirtualId:988(283ns) PrepareNewSessionHandshake:477(1622ns) {SwitchToState:179(843ns)}}}}}
2025-12-24T02:11:58.049918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479797131325:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.050074Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098695:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.050081Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241484092098695:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.050217Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098700:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.050101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760361:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.050112Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485517760361:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.050653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485517760361:2111] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.050881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241481222792998:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.050900Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.050907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241481222792998:2049] [node 3] ICP22 created new session: [1:7587241485517760377:2049]
2025-12-24T02:11:58.050914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485517760377:2049] [node 3] ICS09 handshake done sender: [1:7587241485517760361:2111] self: [1:7587241485517760356:0] peer: [3:7587241484092098702:0] socket: 606 qp: -1
2025-12-24T02:11:58.050922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485517760377:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.050941Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.051656Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537453:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.051522Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485517760381:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48139
2025-12-24T02:11:58.051579Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485517760382:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48139
2025-12-24T02:11:58.051836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485517760354:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.051959Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760381:2117] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.051982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241481222792997:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.051999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.052004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241481222792997:2048] [node 2] ICP22 created new session: [1:7587241485517760383:2048]
2025-12-24T02:11:58.052010Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485517760383:2048] [node 2] ICS09 handshake done sender: [1:7587241485517760354:2107] self: [1:7587241485517760353:0] peer: [2:7587241484429537448:0] socket: 600 qp: -1
2025-12-24T02:11:58.052016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485517760383:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:58.052024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.050758Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098704:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
2025-12-24T02:11:58.050899Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241484092098689:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.050972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479797131325:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.050987Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.050994Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479797131325:2048] [node 1] ICP22 created new session: [3:7587241484092098705:2048]
2025-12-24T02:11:58.051002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241484092098705:2048] [node 1] ICS09 handshake done sender: [3:7587241484092098689:2110] self: [3:7587241484092098702:0] peer: [1:7587241485517760356:0] socket: 607 qp: -1
2025-12-24T02:11:58.051009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241484092098705:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.051026Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.055042Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784866:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.055052Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784867:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.056858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760372:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.056870Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485517760372:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.056644Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484275784867:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.056712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479980817490:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.057694Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784869:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.057880Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484275784863:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.057548Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241484092098691:2111] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.058024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241484092098686:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.058035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.058042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241484092098686:2109] [node 6] ICP22 created new session: [3:7587241484092098709:2109]
2025-12-24T02:11:58.058048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241484092098709:2109] [node 6] ICS09 handshake done sender: [3:7587241484092098691:2111] self: [3:7587241484092098690:0] peer: [6:7587241484275784864:0] socket: 609 qp: -1
2025-12-24T02:11:58.057976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479980817492:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.058004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.058010Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479980817492:2050] [node 3] ICP22 created new session: [6:7587241484275784870:2050]
2025-12-24T02:11:58.058017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484275784870:2050] [node 3] ICS09 handshake done sender: [6:7587241484275784863:2114] self: [6:7587241484275784864:0] peer: [3:7587241484092098690:0] socket: 615 qp: -1
2025-12-24T02:11:58.058037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484275784870:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.058046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.061036Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484275784866:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.061080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241479980817491:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.061503Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485517760367:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.058062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241484092098709:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.058069Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.058868Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995792:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.060472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995792:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.061641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869218:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.062732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486147869214:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:58.063075Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869218:2113] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.063113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241481852901849:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.063067Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [2:7587241480134570074:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(40875542ns) {Configure:104(40874302ns) {ProcessConfigured:135(40450480ns) {StartInitialHandshake:157(40449838ns) {DropHandshakes:470(373ns) {DropIncomingHandshake:442(52ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(244ns) PrepareNewSessionHandshake:477(1116ns) {SwitchToState:179(553ns)}}}}}
2025-12-24T02:11:58.064146Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537439:2110] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.064158Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241484429537439:2110] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.064226Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241484092098695:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.064185Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760382:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.064220Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485517760358:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.064240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.064247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485517760358:2108] [node 4] ICP22 created new session: [1:7587241485517760388:2108]
2025-12-24T02:11:58.064253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485517760388:2108] [node 4] ICS09 handshake done sender: [1:7587241485517760367:2112] self: [1:7587241485517760364:0] peer: [4:7587241485580995781:0] socket: 608 qp: -1
2025-12-24T02:11:58.064262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485517760388:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:58.064274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.064357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241484092098684:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.064379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.064386Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241484092098684:2107] [node 4] ICP22 created new session: [3:7587241484092098712:2107]
2025-12-24T02:11:58.064394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241484092098712:2107] [node 4] ICS09 handshake done sender: [3:7587241484092098695:2112] self: [3:7587241484092098687:0] peer: [4:7587241485580995791:0] socket: 619 qp: -1
2025-12-24T02:11:58.064400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241484092098712:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.064408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.065332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869225:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.065398Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869226:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.065431Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869227:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.065048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241480134570073:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241484429537444:2112] held: no
2025-12-24T02:11:58.065672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485517760372:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.065771Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869225:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.065873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537447:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.065889Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241484429537447:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.066053Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241484429537439:2110] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.066151Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241484429537434:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.066167Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.066174Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241484429537434:2108] [node 5] ICP22 created new session: [2:7587241484429537459:2108]
2025-12-24T02:11:58.066181Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241484429537459:2108] [node 5] ICS09 handshake done sender: [2:7587241484429537439:2110] self: [2:7587241484429537438:0] peer: [5:7587241486147869223:0] socket: 610 qp: -1
2025-12-24T02:11:58.066188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241484429537459:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.066207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.069395Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784875:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.069419Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784874:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.068528Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241481222792997:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(16440318ns) {HandleHandshakeStatus:329(16439728ns) {DropIncomingHandshake:442(3273ns) DropOutgoingHandshake:457(116ns) SwitchToState:179(73ns) ProcessPendingSessionEvents:495(7606746ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 7585730ns max# 7503277ns}}}
2025-12-24T02:11:58.068679Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485517760363:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.068692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.068696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485517760363:2110] [node 6] ICP22 created new session: [1:7587241485517760392:2110]
2025-12-24T02:11:58.068700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485517760392:2110] [node 6] ICS09 handshake done sender: [1:7587241485517760372:2115] self: [1:7587241485517760369:0] peer: [6:7587241484275784868:0] socket: 645 qp: -1
2025-12-24T02:11:58.068705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485517760392:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.068712Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.066850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241481286028411:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6862) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241485580995776:2113] held: no
2025-12-24T02:11:58.069181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241481286028409:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241485580995774:2111] held: no
2025-12-24T02:11:58.070753Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484275784867:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.070651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869226:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.070815Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241481852901848:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.070941Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537461:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.071028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537462:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.070989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479980817490:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.071013Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.071022Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479980817490:2048] [node 1] ICP22 created new session: [6:7587241484275784876:2048]
2025-12-24T02:11:58.071019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484275784874:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.071028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484275784876:2048] [node 1] ICS09 handshake done sender: [6:7587241484275784867:2117] self: [6:7587241484275784868:0] peer: [1:7587241485517760369:0] socket: 650 qp: -1
2025-12-24T02:11:58.071035Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484275784876:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.071055Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.071075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484275784847:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.071216Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241484429537444:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.071300Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869219:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.071310Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241486147869219:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.071900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486147869219:2114] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.072067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241480134570073:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.072087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.072093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241480134570073:2048] [node 1] ICP22 created new session: [2:7587241484429537464:2048]
2025-12-24T02:11:58.072100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241484429537464:2048] [node 1] ICS09 handshake done sender: [2:7587241484429537444:2112] self: [2:7587241484429537448:0] peer: [1:7587241485517760353:0] socket: 605 qp: -1
2025-12-24T02:11:58.072107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241484429537464:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.072114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.072491Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537461:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.072580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241486147869207:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.072600Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.072607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486147869207:2109] [node 6] ICP22 created new session: [5:7587241486147869230:2109]
2025-12-24T02:11:58.072613Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486147869230:2109] [node 6] ICS09 handshake done sender: [5:7587241486147869219:2114] self: [5:7587241486147869213:0] peer: [6:7587241484275784878:0] socket: 657 qp: -1
2025-12-24T02:11:58.072617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486147869230:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.072623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.076875Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485517760373:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.076900Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485517760373:2116] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.076860Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241484429537463:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:25129
2025-12-24T02:11:58.076980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241484429537434:2108] [node 5] ICP18 handshake (actor [2:7587241484429537461:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.079224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098714:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
2025-12-24T02:11:58.080671Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [3:7587241484092098684:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(16191265ns) {HandleHandshakeStatus:329(16190553ns) {DropIncomingHandshake:442(4938ns) DropOutgoingHandshake:457(118ns) SwitchToState:179(102ns) ProcessPendingSessionEvents:495(14599189ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 14568558ns max# 14476485ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.078933Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995796:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.079105Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995797:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.079217Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241485580995776:2113] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.079305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481286028411:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.079320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.079328Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481286028411:2050] [node 3] ICP22 created new session: [4:7587241485580995798:2050]
2025-12-24T02:11:58.079337Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241485580995798:2050] [node 3] ICS09 handshake done sender: [4:7587241485580995776:2113] self: [4:7587241485580995791:0] peer: [3:7587241484092098687:0] socket: 623 qp: -1
2025-12-24T02:11:58.079344Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241485580995798:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.079352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.079529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241485580995774:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.081340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241481286028409:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.081353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.081361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241481286028409:2048] [node 1] ICP22 created new session: [4:7587241485580995800:2048]
2025-12-24T02:11:58.081369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241485580995800:2048] [node 1] ICS09 handshake done sender: [4:7587241485580995774:2111] self: [4:7587241485580995781:0] peer: [1:7587241485517760364:0] socket: 620 qp: -1
2025-12-24T02:11:58.081378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241485580995800:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.081399Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.082978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241479980817491:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241484275784866:2116] held: no
2025-12-24T02:11:58.085136Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784879:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.085273Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784880:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.085476Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484275784847:2107] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27879) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241484275784874:2119] held: no
2025-12-24T02:11:58.085694Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484275784874:2119] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.086018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484275784879:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.086034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484275784847:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.086046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.086055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484275784847:2107] [node 5] ICP22 created new session: [6:7587241484275784881:2107]
2025-12-24T02:11:58.086066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484275784881:2107] [node 5] ICS09 handshake done sender: [6:7587241484275784874:2119] self: [6:7587241484275784878:0] peer: [5:7587241486147869213:0] socket: 658 qp: -1
2025-12-24T02:11:58.086075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484275784881:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:58.086084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.086484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484275784861:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.086647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995790:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.086666Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241485580995790:2118] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.081319Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241481852901850:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(34958284ns) {Configure:104(34956164ns) {ProcessConfigured:135(34546584ns) {StartInitialHandshake:157(34545249ns) {DropHandshakes:470(678ns) {DropIncomingHandshake:442(57ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(205ns) PrepareNewSessionHandshake:477(1311ns) {SwitchToState:179(834ns)}}}}}
2025-12-24T02:11:58.081360Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486147869227:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.081526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241481852901850:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.082833Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241486147869214:2110] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(19929715ns) {Configure:104(19928905ns) {ProcessConfigured:135(19267881ns) {StartInitialHandshake:157(19267102ns) {DropHandshakes:470(749ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(199ns)} GenerateSessionVirtualId:988(433ns) PrepareNewSessionHandshake:477(1210ns) {SwitchToState:179(718ns)}}}}}
2025-12-24T02:11:58.084091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486147869214:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.084227Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241481852901849:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486147869218:2113] held: no
2025-12-24T02:11:58.086602Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869237:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.086674Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486147869218:2113] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.080406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537463:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.080735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241484429537447:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.080846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241484429537435:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.080868Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.080877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241484429537435:2109] [node 6] ICP22 created new session: [2:7587241484429537468:2109]
2025-12-24T02:11:58.080885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241484429537468:2109] [node 6] ICS09 handshake done sender: [2:7587241484429537447:2114] self: [2:7587241484429537437:0] peer: [6:7587241484275784873:0] socket: 633 qp: -1
2025-12-24T02:11:58.080898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241484429537468:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.080907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.086807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485517760373:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.086899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485517760362:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.086928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.086938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485517760362:2109] [node 5] ICP22 created new session: [1:7587241485517760395:2109]
2025-12-24T02:11:58.086945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485517760395:2109] [node 5] ICS09 handshake done sender: [1:7587241485517760373:2116] self: [1:7587241485517760366:0] peer: [5:7587241486147869228:0] socket: 649 qp: -1
2025-12-24T02:11:58.086954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485517760395:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:58.086972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.087664Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098714:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.087709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241479797131326:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.089628Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241481852901849:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.089658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.089667Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241481852901849:2049] [node 2] ICP22 created new session: [5:7587241486147869238:2049]
2025-12-24T02:11:58.089673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486147869238:2049] [node 2] ICS09 handshake done sender: [5:7587241486147869218:2113] self: [5:7587241486147869223:0] peer: [2:7587241484429537438:0] socket: 618 qp: -1
2025-12-24T02:11:58.089680Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486147869238:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.089690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.091744Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241481852901848:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486147869226:2117] held: no
2025-12-24T02:11:58.091281Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995788:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.091306Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241485580995788:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.091365Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241484429537456:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.091382Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241484429537456:2117] [node 3] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.094730Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098697:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.094753Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241484092098697:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.096271Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241484092098697:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.096387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241484092098685:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.096410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.095850Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241485580995802:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18282
2025-12-24T02:11:58.096096Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241485580995802:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.096770Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241485580995790:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.096856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241485580995771:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.096871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.096876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241485580995771:2109] [node 6] ICP22 created new session: [4:7587241485580995805:2109]
2025-12-24T02:11:58.096882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241485580995805:2109] [node 6] ICS09 handshake done sender: [4:7587241485580995790:2118] self: [4:7587241485580995779:0] peer: [6:7587241484275784883:0] socket: 629 qp: -1
2025-12-24T02:11:58.096886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241485580995805:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.096898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.096426Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241484092098685:2108] [node 5] ICP22 created new session: [3:7587241484092098717:2108]
2025-12-24T02:11:58.096443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241484092098717:2108] [node 5] ICS09 handshake done sender: [3:7587241484092098697:2113] self: [3:7587241484092098688:0] peer: [5:7587241486147869233:0] socket: 632 qp: -1
2025-12-24T02:11:58.096454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241484092098717:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.096486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.100808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241484429537456:2117] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.100755Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869240:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.100766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869241:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.100916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098719:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
2025-12-24T02:11:58.101262Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486147869226:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.101335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241484092098719:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.101372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241484092098685:2108] [node 5] ICP18 handshake (actor [3:7587241484092098719:2119]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.101177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241480134570074:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.101225Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.101233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241480134570074:2049] [node 3] ICP22 created new session: [2:7587241484429537471:2049]
2025-12-24T02:11:58.101239Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241484429537471:2049] [node 3] ICS09 handshake done sender: [2:7587241484429537456:2117] self: [2:7587241484429537436:0] peer: [3:7587241484092098715:0] socket: 663 qp: -1
2025-12-24T02:11:58.101246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241484429537471:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.101266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.101656Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484275784861:2112] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241484275784879:2121] held: no
2025-12-24T02:11:58.101661Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784884:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.100787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241485580995788:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.102101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241485580995770:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.102119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.102125Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241485580995770:2108] [node 5] ICP22 created new session: [4:7587241485580995808:2108]
2025-12-24T02:11:58.102133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241485580995808:2108] [node 5] ICS09 handshake done sender: [4:7587241485580995788:2117] self: [4:7587241485580995778:0] peer: [5:7587241486147869235:0] socket: 656 qp: -1
2025-12-24T02:11:58.102138Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241485580995808:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.102146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.102766Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484275784885:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19549
2025-12-24T02:11:58.102864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484275784879:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.102922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484275784861:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.102938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.102946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484275784861:2112] [node 4] ICP22 created new session: [6:7587241484275784886:2112]
2025-12-24T02:11:58.102956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484275784886:2112] [node 4] ICS09 handshake done sender: [6:7587241484275784879:2121] self: [6:7587241484275784883:0] peer: [4:7587241485580995779:0] socket: 639 qp: -1
2025-12-24T02:11:58.102962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484275784886:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:58.102972Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.103475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241479797131326:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241484092098714:2118] held: no
2025-12-24T02:11:58.103502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241484092098720:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:52762
2025-12-24T02:11:58.104028Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241484092098714:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.104102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241479797131326:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.104124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.104132Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241479797131326:2049] [node 2] ICP22 created new session: [3:7587241484092098721:2049]
2025-12-24T02:11:58.104139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241484092098721:2049] [node 2] ICS09 handshake done sender: [3:7587241484092098714:2118] self: [3:7587241484092098715:0] peer: [2:7587241484429537436:0] socket: 664 qp: -1
2025-12-24T02:11:58.104145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241484092098721:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.104156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.102776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241481852901848:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.102794Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.102801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241481852901848:2048] [node 1] ICP22 created new session: [5:7587241486147869243:2048]
2025-12-24T02:11:58.102807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486147869243:2048] [node 1] ICS09 handshake done sender: [5:7587241486147869226:2117] self: [5:7587241486147869228:0] peer: [1:7587241485517760366:0] socket: 625 qp: -1
2025-12-24T02:11:58.102813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486147869243:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.102836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.102855Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486147869242:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:59244
2025-12-24T02:11:58.103174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241481852901850:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6862) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486147869225:2116] held: no
2025-12-24T02:11:58.103359Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486147869225:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.106419Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484275784866:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.107256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241479980817491:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.107273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.107281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241479980817491:2049] [node 2] ICP22 created new session: [6:7587241484275784888:2049]
2025-12-24T02:11:58.107287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484275784888:2049] [node 2] ICS09 handshake done sender: [6:7587241484275784866:2116] self: [6:7587241484275784873:0] peer: [2:7587241484429537437:0] socket: 637 qp: -1
2025-12-24T02:11:58.107294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484275784888:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.107332Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.108607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241486147869214:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27207) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486147869227:2118] held: no
2025-12-24T02:11:58.108714Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241481852901850:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.108725Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.108731Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241481852901850:2050] [node 3] ICP22 created new session: [5:7587241486147869245:2050]
2025-12-24T02:11:58.108738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486147869245:2050] [node 3] ICS09 handshake done sender: [5:7587241486147869225:2116] self: [5:7587241486147869233:0] peer: [3:7587241484092098688:0] socket: 638 qp: -1
2025-12-24T02:11:58.108744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486147869245:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.108751Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.108974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486147869227:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.114886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486147869214:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.114915Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.114927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486147869214:2110] [node 4] ICP22 created new session: [5:7587241486147869247:2110]
2025-12-24T02:11:58.114934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486147869247:2110] [node 4] ICS09 handshake done sender: [5:7587241486147869227:2118] self: [5:7587241486147869235:0] peer: [4:7587241485580995778:0] socket: 660 qp: -1
2025-12-24T02:11:58.114951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486147869247:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:58.114963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.130491Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:58.130576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241484429537464:2048] [node 1] ICS01 socket: 605 reason# EndOfStream
2025-12-24T02:11:58.130586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241480134570073:2048] [node 1] ICP30 unregister session Session# [2:7587241484429537464:2048] VirtualId# [2:7587241484429537448:0]
2025-12-24T02:11:58.130593Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241484429537464:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.130626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.130645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.131253Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:58.131350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241484092098705:2048] [node 1] ICS07 socket disconnect 607 reason# EndOfStream
2025-12-24T02:11:58.131361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241484092098705:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.131419Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.131454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241484092098705:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:58.133952Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:58.134030Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241484275784876:2048] [node 1] ICS01 socket: 650 reason# EndOfStream
2025-12-24T02:11:58.134037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479980817490:2048] [node 1] ICP30 unregister session Session# [6:7587241484275784876:2048] VirtualId# [6:7587241484275784868:0]
2025-12-24T02:11:58.134043Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241484275784876:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.134089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.134107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.133413Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:58.133513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241485580995800:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:58.133523Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241481286028409:2048] [node 1] ICP30 unregister session Session# [4:7587241485580995800:2048] VirtualId# [4:7587241485580995781:0]
2025-12-24T02:11:58.133533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241485580995800:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.133589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.133617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.134799Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:58.134883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486147869243:2048] [node 1] ICS01 socket: 625 reason# EndOfStream
2025-12-24T02:11:58.134890Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241481852901848:2048] [node 1] ICP30 unregister session Session# [5:7587241486147869243:2048] VirtualId# [5:7587241486147869228:0]
2025-12-24T02:11:58.134897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486147869243:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.134907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.134953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.135167Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241479797131325:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.135188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241484092098705:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.135193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479797131325:2048] [node 1] ICP30 unregister session Session# [3:7587241484092098705:2048] VirtualId# [3:7587241484092098702:0]
2025-12-24T02:11:58.135250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.135258Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241479797131325:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.135261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3004) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.145280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241479797131325:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:58.148693Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:58.148817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241484092098721:2049] [node 2] ICS01 socket: 664 reason# EndOfStream
2025-12-24T02:11:58.148829Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241479797131326:2049] [node 2] ICP30 unregister session Session# [3:7587241484092098721:2049] VirtualId# [3:7587241484092098715:0]
2025-12-24T02:11:58.148839Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241484092098721:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.148902Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.148939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.150108Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:58.150126Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:58.150208Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241484275784888:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:58.150226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479980817491:2049] [node 2] ICP30 unregister session Session# [6:7587241484275784888:2049] VirtualId# [6:7587241484275784873:0]
2025-12-24T02:11:58.150234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241484275784888:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.150269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.150297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.150840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:58.150934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486147869238:2049] [node 2] ICS01 socket: 618 reason# EndOfStream
2025-12-24T02:11:58.150941Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241481852901849:2049] [node 2] ICP30 unregister session Session# [5:7587241486147869238:2049] VirtualId# [5:7587241486147869223:0]
2025-12-24T02:11:58.150947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486147869238:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.150986Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.151010Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.150224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241485580995786:2049] [node 2] ICS07 socket disconnect 622 reason# EndOfStream
2025-12-24T02:11:58.150235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241485580995786:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.150296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.150319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241485580995786:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:58.151057Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241481286028410:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.151069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241485580995786:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.151075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241481286028410:2049] [node 2] ICP30 unregister session Session# [4:7587241485580995786:2049] VirtualId# [4:7587241485580995782:0]
2025-12-24T02:11:58.151141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.151150Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241481286028410:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.151152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:10594) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.161563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241481286028410:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:58.171516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:58.171636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241485580995798:2050] [node 3] ICS01 socket: 623 reason# EndOfStream
2025-12-24T02:11:58.171649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241481286028411:2050] [node 3] ICP30 unregister session Session# [4:7587241485580995798:2050] VirtualId# [4:7587241485580995791:0]
2025-12-24T02:11:58.171659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241485580995798:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.171742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.171776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.174782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:58.174854Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486147869245:2050] [node 3] ICS01 socket: 638 reason# EndOfStream
2025-12-24T02:11:58.174861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241481852901850:2050] [node 3] ICP30 unregister session Session# [5:7587241486147869245:2050] VirtualId# [5:7587241486147869233:0]
2025-12-24T02:11:58.174868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486147869245:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.174921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.174948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.173794Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:58.173901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241484275784870:2050] [node 3] ICS01 socket: 615 reason# EndOfStream
2025-12-24T02:11:58.173913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241479980817492:2050] [node 3] ICP30 unregister session Session# [6:7587241484275784870:2050] VirtualId# [6:7587241484275784864:0]
2025-12-24T02:11:58.173923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241484275784870:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.173989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.174020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.218966Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:58.219083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241484275784886:2112] [node 4] ICS01 socket: 639 reason# EndOfStream
2025-12-24T02:11:58.219100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241484275784861:2112] [node 4] ICP30 unregister session Session# [6:7587241484275784886:2112] VirtualId# [6:7587241484275784883:0]
2025-12-24T02:11:58.219108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241484275784886:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.219177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.219207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:58.261233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485785202109:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:58.261249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485785202110:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.262718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241485671287575:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.262740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241485671287576:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.264349Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483242919878:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.264365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483242919879:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.265847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483054357849:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.265860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483054357850:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.265864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483054357851:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.267565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486039102090:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.267580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486039102091:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.267585Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486039102092:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.270413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483284603122:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.270427Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483284603123:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.270432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483284603124:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.282873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483284603184:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.285418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485785202175:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:58.285618Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485785202176:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:58.285788Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485785202177:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:58.283671Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102150:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.287529Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486039102150:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.287719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486039102152:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:58.288181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486039102152:2108] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.291092Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483284603186:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.291124Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241483284603186:2109] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.299534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241485671287639:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.299804Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241485671287640:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.300033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241485671287641:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.305178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483284603189:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:58.308322Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241483054357851:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(17410975ns) {Configure:104(17409705ns) {ProcessConfigured:135(17022766ns) {StartInitialHandshake:157(17021831ns) {DropHandshakes:470(1129ns) {DropIncomingHandshake:442(19ns) DropOutgoingHandshake:457(47ns)} GenerateSessionVirtualId:988(395ns) PrepareNewSessionHandshake:477(1954ns) {SwitchToState:179(1114ns)}}}}}
2025-12-24T02:11:58.308765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483054357914:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.308918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483054357915:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.311566Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287638:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
2025-12-24T02:11:58.311218Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [1:7587241485785202110:2049] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(25290235ns) {Configure:104(25288313ns) {ProcessConfigured:135(24713535ns) {StartInitialHandshake:157(24713128ns) {DropHandshakes:470(1488ns) {DropIncomingHandshake:442(336ns) DropOutgoingHandshake:457(356ns)} GenerateSessionVirtualId:988(521ns) PrepareNewSessionHandshake:477(1577ns) {SwitchToState:179(1132ns)}}}}}
2025-12-24T02:11:58.319067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483284603186:2109] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.320457Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287653:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
2025-12-24T02:11:58.319285Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241483284603184:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.319318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.319327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483284603184:2108] [node 5] ICP22 created new session: [6:7587241483284603201:2108]
2025-12-24T02:11:58.319334Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483284603201:2108] [node 5] ICS09 handshake done sender: [6:7587241483284603186:2109] self: [6:7587241483284603185:0] peer: [5:7587241486039102153:0] socket: 607 qp: -1
2025-12-24T02:11:58.319342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483284603201:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.319355Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.320906Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241483242919878:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(32985560ns) {Configure:104(32985042ns) {ProcessConfigured:135(32535855ns) {StartInitialHandshake:157(32535603ns) {DropHandshakes:470(662ns) {DropIncomingHandshake:442(56ns) DropOutgoingHandshake:457(59ns)} GenerateSessionVirtualId:988(268ns) PrepareNewSessionHandshake:477(1413ns) {SwitchToState:179(304ns)}}}}}
2025-12-24T02:11:58.321244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483242919943:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.321422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483242919944:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.321577Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483242919945:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.322994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919946:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.323021Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919947:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.323334Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919957:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.324580Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241486039102090:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(26578926ns) {Configure:104(26577772ns) {ProcessConfigured:135(26162141ns) {StartInitialHandshake:157(26161582ns) {DropHandshakes:470(1976ns) {DropIncomingHandshake:442(285ns) DropOutgoingHandshake:457(306ns)} GenerateSessionVirtualId:988(462ns) PrepareNewSessionHandshake:477(2043ns) {SwitchToState:179(1891ns)}}}}}
2025-12-24T02:11:58.324924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486039102158:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:58.325153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287652:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.328221Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102159:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.327127Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287653:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.332564Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486039102150:2107] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.332864Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486039102152:2108] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.332884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.332892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486039102152:2108] [node 6] ICP22 created new session: [5:7587241486039102169:2108]
2025-12-24T02:11:58.332900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486039102169:2108] [node 6] ICS09 handshake done sender: [5:7587241486039102150:2107] self: [5:7587241486039102153:0] peer: [6:7587241483284603185:0] socket: 608 qp: -1
2025-12-24T02:11:58.332908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486039102169:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:58.332933Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.336399Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919947:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.336447Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919957:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.336589Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241483242919879:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.336713Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919960:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.338311Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485785202186:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:11:58.338346Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485785202187:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.336662Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287657:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
2025-12-24T02:11:58.336757Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287658:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
2025-12-24T02:11:58.336793Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287645:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:58.336798Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241485671287645:2112] [node 3] ICRDMA Non success qp response from remote side
::ffff:127.1.0.0
2025-12-24T02:11:58.337254Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287657:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:58.340988Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357927:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.337499Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919963:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.338842Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919964:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.341958Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483242919947:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.342018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241483242919879:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.342042Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.342050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483242919879:2049] [node 2] ICP22 created new session: [3:7587241483242919965:2049]
2025-12-24T02:11:58.342058Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483242919965:2049] [node 2] ICS09 handshake done sender: [3:7587241483242919947:2112] self: [3:7587241483242919961:0] peer: [2:7587241485671287644:0] socket: 626 qp: -1
2025-12-24T02:11:58.342067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483242919965:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.342075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.341669Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241485671287645:2112] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.343500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241485671287576:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.343519Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.343524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241485671287576:2049] [node 3] ICP22 created new session: [2:7587241485671287662:2049]
2025-12-24T02:11:58.343530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241485671287662:2049] [node 3] ICS09 handshake done sender: [2:7587241485671287645:2112] self: [2:7587241485671287644:0] peer: [3:7587241483242919961:0] socket: 624 qp: -1
2025-12-24T02:11:58.343535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241485671287662:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.343543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.342925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919960:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:58.346058Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287658:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287638:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241485671287575:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.346471Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357928:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.346557Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357927:2115] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346153Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919953:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346165Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483242919953:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.344223Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102171:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.338615Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485785202190:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:11:58.346737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483054357851:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: 2025-12-24T02:11:58.346033Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486039102171:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346064Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486039102092:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
::ffff:127.1.0.0
No RDMA context found for GID: No RDMA context found for GID: ::ffff:127.1.0.0::ffff:127.1.0.0
2025-12-24T02:11:58.338675Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485785202188:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:11:58.338767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485785202189:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54551
2025-12-24T02:11:58.344309Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202188:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.344331Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202189:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.344729Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202187:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346885Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357929:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.346906Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483242919953:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.346952Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357928:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346986Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483054357850:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.347496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919946:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.347540Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919951:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.347543Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483242919951:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: 2025-12-24T02:11:58.349438Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357932:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.349783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483242919944:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.349803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.349808Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483242919944:2108] [node 5] ICP22 created new session: [3:7587241483242919970:2108]
::ffff:127.1.0.0
2025-12-24T02:11:58.349813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483242919970:2108] [node 5] ICS09 handshake done sender: [3:7587241483242919953:2115] self: [3:7587241483242919952:0] peer: [5:7587241486039102172:0] socket: 630 qp: -1
2025-12-24T02:11:58.349821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483242919970:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.349827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.346865Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287652:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.346779Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102173:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.346816Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102174:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.348060Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287647:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.347120Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486039102171:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.347178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486039102092:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.347194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.347200Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486039102092:2050] [node 3] ICP22 created new session: [5:7587241486039102175:2050]
2025-12-24T02:11:58.347206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486039102175:2050] [node 3] ICS09 handshake done sender: [5:7587241486039102171:2116] self: [5:7587241486039102172:0] peer: [3:7587241483242919952:0] socket: 654 qp: -1
2025-12-24T02:11:58.347213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486039102175:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.347223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.348065Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241485671287647:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.349759Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241485671287647:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.349855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241485671287639:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.349867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.349872Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241485671287639:2107] [node 4] ICP22 created new session: [2:7587241485671287669:2107]
2025-12-24T02:11:58.349877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241485671287669:2107] [node 4] ICS09 handshake done sender: [2:7587241485671287647:2113] self: [2:7587241485671287646:0] peer: [4:7587241483054357931:0] socket: 634 qp: -1
2025-12-24T02:11:58.349881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241485671287669:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.349886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.350806Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202190:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.354529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357933:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.354845Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483054357928:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.355130Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202170:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.355160Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485785202170:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.354791Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486039102173:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.355597Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202186:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.355805Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603204:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.357167Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483284603204:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.357201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483284603123:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.358033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486039102091:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.358282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483242919951:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.358744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483242919943:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.358766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.358773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483242919943:2107] [node 4] ICP22 created new session: [3:7587241483242919975:2107]
2025-12-24T02:11:58.358779Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483242919975:2107] [node 4] ICS09 handshake done sender: [3:7587241483242919951:2114] self: [3:7587241483242919950:0] peer: [4:7587241483054357930:0] socket: 631 qp: -1
2025-12-24T02:11:58.358787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483242919975:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.356617Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241485671287672:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48975
2025-12-24T02:11:58.358801Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.357541Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287651:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.357546Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241485671287651:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.358394Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241485671287651:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.358864Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485785202170:2107] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.359035Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603206:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.356846Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483054357850:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.356874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.356883Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483054357850:2049] [node 2] ICP22 created new session: [4:7587241483054357934:2049]
2025-12-24T02:11:58.356892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483054357934:2049] [node 2] ICS09 handshake done sender: [4:7587241483054357928:2116] self: [4:7587241483054357931:0] peer: [2:7587241485671287646:0] socket: 649 qp: -1
2025-12-24T02:11:58.356899Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483054357934:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.356910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.357376Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357929:2117] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.357511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483054357849:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.359655Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357932:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.359647Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202181:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.359658Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485785202181:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.360139Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485785202109:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.360156Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.360161Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485785202109:2048] [node 2] ICP22 created new session: [1:7587241485785202199:2048]
2025-12-24T02:11:58.360166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485785202199:2048] [node 2] ICS09 handshake done sender: [1:7587241485785202170:2107] self: [1:7587241485785202169:0] peer: [2:7587241485671287666:0] socket: 606 qp: -1
2025-12-24T02:11:58.360171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485785202199:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:58.360181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.361190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485785202181:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.361182Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102179:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.361301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485785202175:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.361335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.361341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485785202175:2108] [node 4] ICP22 created new session: [1:7587241485785202203:2108]
2025-12-24T02:11:58.361347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485785202203:2108] [node 4] ICS09 handshake done sender: [1:7587241485785202181:2112] self: [1:7587241485785202180:0] peer: [4:7587241483054357936:0] socket: 651 qp: -1
2025-12-24T02:11:58.361352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485785202203:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:58.361359Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.359011Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241485671287649:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.359024Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241485671287649:2114] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.359526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241485671287641:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.359542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.359549Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241485671287641:2109] [node 6] ICP22 created new session: [2:7587241485671287674:2109]
2025-12-24T02:11:58.359555Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241485671287674:2109] [node 6] ICS09 handshake done sender: [2:7587241485671287651:2115] self: [2:7587241485671287650:0] peer: [6:7587241483284603205:0] socket: 621 qp: -1
2025-12-24T02:11:58.359562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241485671287674:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.359575Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.363752Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919964:2120] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.363794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241483242919878:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.364986Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603207:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.365100Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483284603204:2115] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.365811Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202179:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.365826Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485785202179:2111] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.364364Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486039102179:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.364406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486039102090:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.366313Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202183:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.366324Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485785202183:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.367296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483242919978:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:11608
2025-12-24T02:11:58.367408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483284603123:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.367441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.367451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483284603123:2049] [node 2] ICP22 created new session: [6:7587241483284603209:2049]
2025-12-24T02:11:58.367460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483284603209:2049] [node 2] ICS09 handshake done sender: [6:7587241483284603204:2115] self: [6:7587241483284603205:0] peer: [2:7587241485671287650:0] socket: 640 qp: -1
2025-12-24T02:11:58.367435Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485785202179:2111] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.367469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483284603209:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.367480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.367624Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483242919964:2120] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.367700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241483242919878:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.367715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.367721Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483242919878:2048] [node 1] ICP22 created new session: [3:7587241483242919979:2048]
2025-12-24T02:11:58.367727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483242919979:2048] [node 1] ICS09 handshake done sender: [3:7587241483242919964:2120] self: [3:7587241483242919977:0] peer: [1:7587241485785202178:0] socket: 652 qp: -1
2025-12-24T02:11:58.367733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483242919979:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.367740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.368038Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102181:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.369010Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485785202183:2113] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.369125Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241483284603123:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.369321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603208:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.369330Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483054357851:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22573) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241483054357927:2115] held: no
2025-12-24T02:11:58.370447Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486039102179:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.370590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241485671287638:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.370602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486039102090:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.370619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.370626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486039102090:2048] [node 1] ICP22 created new session: [5:7587241486039102183:2048]
2025-12-24T02:11:58.370633Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486039102183:2048] [node 1] ICS09 handshake done sender: [5:7587241486039102179:2119] self: [5:7587241486039102180:0] peer: [1:7587241485785202182:0] socket: 668 qp: -1
2025-12-24T02:11:58.370640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486039102183:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.370649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.370256Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485785202110:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.370283Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.370291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485785202110:2049] [node 3] ICP22 created new session: [1:7587241485785202208:2049]
2025-12-24T02:11:58.370299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485785202208:2049] [node 3] ICS09 handshake done sender: [1:7587241485785202179:2111] self: [1:7587241485785202178:0] peer: [3:7587241483242919977:0] socket: 650 qp: -1
2025-12-24T02:11:58.370307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485785202208:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.370317Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.370858Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241483054357850:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.370975Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483054357849:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241483054357929:2117] held: no
2025-12-24T02:11:58.371313Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357937:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.371575Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483284603207:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.371338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485785202176:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.371360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.371365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485785202176:2109] [node 5] ICP22 created new session: [1:7587241485785202210:2109]
2025-12-24T02:11:58.371371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485785202210:2109] [node 5] ICS09 handshake done sender: [1:7587241485785202183:2113] self: [1:7587241485785202182:0] peer: [5:7587241486039102180:0] socket: 642 qp: -1
2025-12-24T02:11:58.371375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485785202210:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:58.371380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.371360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102182:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.371962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483284603124:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.372058Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603212:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.372382Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483284603208:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.372402Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483284603122:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.372094Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483242919955:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.372108Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483242919955:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.372687Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357938:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.372932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603215:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.372951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483242919955:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.372949Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483054357939:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:31843
2025-12-24T02:11:58.373097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483054357927:2115] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.373054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483242919945:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.373072Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.373078Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483242919945:2109] [node 6] ICP22 created new session: [3:7587241483242919982:2109]
2025-12-24T02:11:58.373083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483242919982:2109] [node 6] ICS09 handshake done sender: [3:7587241483242919955:2116] self: [3:7587241483242919954:0] peer: [6:7587241483284603213:0] socket: 665 qp: -1
2025-12-24T02:11:58.373089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483242919982:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.373098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.374403Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483054357929:2117] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.371473Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241485671287575:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241485671287638:2110] held: no
2025-12-24T02:11:58.371590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241485671287575:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.371602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.371607Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241485671287575:2048] [node 1] ICP22 created new session: [2:7587241485671287677:2048]
2025-12-24T02:11:58.371616Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241485671287677:2048] [node 1] ICS09 handshake done sender: [2:7587241485671287638:2110] self: [2:7587241485671287666:0] peer: [1:7587241485785202169:0] socket: 610 qp: -1
2025-12-24T02:11:58.371626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241485671287677:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.371638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.372283Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241485671287649:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.373241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241485671287640:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.373256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.373262Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241485671287640:2108] [node 5] ICP22 created new session: [2:7587241485671287679:2108]
2025-12-24T02:11:58.373268Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241485671287679:2108] [node 5] ICS09 handshake done sender: [2:7587241485671287649:2114] self: [2:7587241485671287648:0] peer: [5:7587241486039102178:0] socket: 635 qp: -1
2025-12-24T02:11:58.373273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241485671287679:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.373281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.374963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486039102173:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.374903Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483284603124:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:22573) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483284603207:2117] held: no
2025-12-24T02:11:58.375033Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483284603207:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.374680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483054357851:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.374717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.374724Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483054357851:2050] [node 3] ICP22 created new session: [4:7587241483054357943:2050]
2025-12-24T02:11:58.374730Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483054357943:2050] [node 3] ICS09 handshake done sender: [4:7587241483054357927:2115] self: [4:7587241483054357930:0] peer: [3:7587241483242919950:0] socket: 641 qp: -1
2025-12-24T02:11:58.374739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483054357943:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.374746Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.374999Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483054357849:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.375006Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.375011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483054357849:2048] [node 1] ICP22 created new session: [4:7587241483054357945:2048]
2025-12-24T02:11:58.375015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483054357945:2048] [node 1] ICS09 handshake done sender: [4:7587241483054357929:2117] self: [4:7587241483054357936:0] peer: [1:7587241485785202180:0] socket: 653 qp: -1
2025-12-24T02:11:58.375021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483054357945:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.375028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.375104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486039102091:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.375123Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.375130Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486039102091:2049] [node 2] ICP22 created new session: [5:7587241486039102185:2049]
2025-12-24T02:11:58.375137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486039102185:2049] [node 2] ICS09 handshake done sender: [5:7587241486039102173:2117] self: [5:7587241486039102178:0] peer: [2:7587241485671287648:0] socket: 656 qp: -1
2025-12-24T02:11:58.375144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486039102185:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.375152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.376445Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483284603124:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.376461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.376468Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483284603124:2050] [node 3] ICP22 created new session: [6:7587241483284603216:2050]
2025-12-24T02:11:58.376474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483284603216:2050] [node 3] ICS09 handshake done sender: [6:7587241483284603207:2117] self: [6:7587241483284603213:0] peer: [3:7587241483242919954:0] socket: 667 qp: -1
2025-12-24T02:11:58.376481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483284603216:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.376490Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.376779Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485785202185:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.376793Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485785202185:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.378416Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102187:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.378983Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485785202185:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.379260Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485785202177:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.379280Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.379288Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485785202177:2110] [node 6] ICP22 created new session: [1:7587241485785202213:2110]
2025-12-24T02:11:58.379294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485785202213:2110] [node 6] ICS09 handshake done sender: [1:7587241485785202185:2114] self: [1:7587241485785202184:0] peer: [6:7587241483284603214:0] socket: 618 qp: -1
2025-12-24T02:11:58.379300Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485785202213:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.379308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.380495Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357939:2122] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.380736Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486039102187:2122] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.380772Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486039102158:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.380930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357921:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.380940Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241483054357921:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.381630Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486039102189:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:6679
2025-12-24T02:11:58.381681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483284603212:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.381719Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483284603189:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.381771Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483284603122:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6101) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483284603208:2118] held: no
2025-12-24T02:11:58.381993Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603218:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.382180Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483284603208:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.382239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483284603122:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.382254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.382259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483284603122:2048] [node 1] ICP22 created new session: [6:7587241483284603220:2048]
2025-12-24T02:11:58.382264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483284603220:2048] [node 1] ICS09 handshake done sender: [6:7587241483284603208:2118] self: [6:7587241483284603214:0] peer: [1:7587241485785202184:0] socket: 669 qp: -1
2025-12-24T02:11:58.382271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483284603220:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.382278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.382806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483054357921:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.382961Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483054357923:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.382971Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241483054357923:2114] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.383559Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486039102187:2122] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.383447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241483054357914:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.383472Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.383478Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483054357914:2108] [node 5] ICP22 created new session: [4:7587241483054357949:2108]
2025-12-24T02:11:58.383484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483054357949:2108] [node 5] ICS09 handshake done sender: [4:7587241483054357921:2113] self: [4:7587241483054357920:0] peer: [5:7587241486039102188:0] socket: 612 qp: -1
2025-12-24T02:11:58.383490Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483054357949:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.383497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.384576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483054357923:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.383662Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486039102158:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.383678Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.383684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486039102158:2109] [node 4] ICP22 created new session: [5:7587241486039102190:2109]
2025-12-24T02:11:58.383690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486039102190:2109] [node 4] ICS09 handshake done sender: [5:7587241486039102187:2122] self: [5:7587241486039102188:0] peer: [4:7587241483054357920:0] socket: 628 qp: -1
2025-12-24T02:11:58.383697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486039102190:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:58.383705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.384912Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483284603222:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:43605
2025-12-24T02:11:58.385066Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483284603212:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.384977Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241483054357915:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.384997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.385003Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483054357915:2109] [node 6] ICP22 created new session: [4:7587241483054357952:2109]
2025-12-24T02:11:58.385009Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483054357952:2109] [node 6] ICS09 handshake done sender: [4:7587241483054357923:2114] self: [4:7587241483054357922:0] peer: [6:7587241483284603219:0] socket: 629 qp: -1
2025-12-24T02:11:58.385015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483054357952:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.385023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.385138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483284603189:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.385152Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.385157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483284603189:2110] [node 4] ICP22 created new session: [6:7587241483284603223:2110]
2025-12-24T02:11:58.385162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483284603223:2110] [node 4] ICS09 handshake done sender: [6:7587241483284603212:2119] self: [6:7587241483284603219:0] peer: [4:7587241483054357922:0] socket: 675 qp: -1
2025-12-24T02:11:58.385167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483284603223:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:58.385174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.396847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:58.396958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486039102183:2048] [node 1] ICS01 socket: 668 reason# EndOfStream
2025-12-24T02:11:58.396972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486039102090:2048] [node 1] ICP30 unregister session Session# [5:7587241486039102183:2048] VirtualId# [5:7587241486039102180:0]
2025-12-24T02:11:58.396981Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486039102183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.397053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.397093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.395034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:58.395169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241485671287677:2048] [node 1] ICS01 socket: 610 reason# EndOfStream
2025-12-24T02:11:58.395180Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241485671287575:2048] [node 1] ICP30 unregister session Session# [2:7587241485671287677:2048] VirtualId# [2:7587241485671287666:0]
2025-12-24T02:11:58.397224Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:58.397312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483284603220:2048] [node 1] ICS01 socket: 669 reason# EndOfStream
2025-12-24T02:11:58.395190Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241485671287677:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.397320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483284603122:2048] [node 1] ICP30 unregister session Session# [6:7587241483284603220:2048] VirtualId# [6:7587241483284603214:0]
2025-12-24T02:11:58.397327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483284603220:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.395256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.397376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.395289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.397398Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.397234Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:58.397322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241483054357945:2048] [node 1] ICS01 socket: 653 reason# EndOfStream
2025-12-24T02:11:58.397329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241483054357849:2048] [node 1] ICP30 unregister session Session# [4:7587241483054357945:2048] VirtualId# [4:7587241483054357936:0]
2025-12-24T02:11:58.397336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241483054357945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.397394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.397417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.398855Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:58.398932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241483242919979:2048] [node 1] ICS01 socket: 652 reason# EndOfStream
2025-12-24T02:11:58.398948Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241483242919878:2048] [node 1] ICP30 unregister session Session# [3:7587241483242919979:2048] VirtualId# [3:7587241483242919977:0]
2025-12-24T02:11:58.398954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241483242919979:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.399011Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.399044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.404635Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:58.404725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241483242919965:2049] [node 2] ICS01 socket: 626 reason# EndOfStream
2025-12-24T02:11:58.404740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241483242919879:2049] [node 2] ICP30 unregister session Session# [3:7587241483242919965:2049] VirtualId# [3:7587241483242919961:0]
2025-12-24T02:11:58.404747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241483242919965:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.404793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.404825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.405807Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:58.405935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241483054357934:2049] [node 2] ICS07 socket disconnect 649 reason# EndOfStream
2025-12-24T02:11:58.405948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241483054357934:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.406000Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.406038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241483054357934:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:58.406058Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:58.406156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486039102185:2049] [node 2] ICS01 socket: 656 reason# EndOfStream
2025-12-24T02:11:58.406169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486039102091:2049] [node 2] ICP30 unregister session Session# [5:7587241486039102185:2049] VirtualId# [5:7587241486039102178:0]
2025-12-24T02:11:58.406174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486039102185:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.406226Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.406257Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.406484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483054357850:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20415) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.406498Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241483054357934:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.406503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241483054357850:2049] [node 2] ICP30 unregister session Session# [4:7587241483054357934:2049] VirtualId# [4:7587241483054357931:0]
2025-12-24T02:11:58.406554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.406565Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241483054357850:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20415) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.406568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20415) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.407658Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:58.407753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483284603209:2049] [node 2] ICS01 socket: 640 reason# EndOfStream
2025-12-24T02:11:58.407767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483284603123:2049] [node 2] ICP30 unregister session Session# [6:7587241483284603209:2049] VirtualId# [6:7587241483284603205:0]
2025-12-24T02:11:58.407773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483284603209:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.407821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.407856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.414135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:58.414262Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241483054357943:2050] [node 3] ICS01 socket: 641 reason# EndOfStream
2025-12-24T02:11:58.414272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241483054357851:2050] [node 3] ICP30 unregister session Session# [4:7587241483054357943:2050] VirtualId# [4:7587241483054357930:0]
2025-12-24T02:11:58.414278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241483054357943:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.414316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.414353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.414354Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:58.414436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486039102175:2050] [node 3] ICS01 socket: 654 reason# EndOfStream
2025-12-24T02:11:58.414448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486039102092:2050] [node 3] ICP30 unregister session Session# [5:7587241486039102175:2050] VirtualId# [5:7587241486039102172:0]
2025-12-24T02:11:58.414454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486039102175:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.414496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.414563Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.414765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:58.414925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483284603216:2050] [node 3] ICS01 socket: 667 reason# EndOfStream
2025-12-24T02:11:58.414933Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483284603124:2050] [node 3] ICP30 unregister session Session# [6:7587241483284603216:2050] VirtualId# [6:7587241483284603213:0]
2025-12-24T02:11:58.414939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483284603216:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.414994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.415023Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.423650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:58.423760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241483284603223:2110] [node 4] ICS07 socket disconnect 675 reason# EndOfStream
2025-12-24T02:11:58.423774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483284603223:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.423833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.423857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241483284603223:2110] [node 4] ICS15 start handshake
2025-12-24T02:11:58.424319Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483284603189:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25468) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.424335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483284603223:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.424341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483284603189:2110] [node 4] ICP30 unregister session Session# [6:7587241483284603223:2110] VirtualId# [6:7587241483284603219:0]
2025-12-24T02:11:58.424404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:58.424412Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241483284603189:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:25468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.424416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:25468) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.432214Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:58.432341Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483284603201:2108] [node 5] ICS01 socket: 607 reason# EndOfStream
2025-12-24T02:11:58.432362Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483284603184:2108] [node 5] ICP30 unregister session Session# [6:7587241483284603201:2108] VirtualId# [6:7587241483284603185:0]
2025-12-24T02:11:58.432371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483284603201:2108] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.432446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:58.432485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:58.455305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241484732774888:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:58.455320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241484732774889:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.456613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486722650554:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.456633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486722650555:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.461192Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483753999446:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.461208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483753999447:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.461213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483753999448:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.458644Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483546628005:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.458653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483546628006:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.462899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485745604227:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.462914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485745604228:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.462924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485745604229:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.466532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483431988499:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.466648Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483431988500:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.466663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483431988501:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.482483Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483431988561:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.483637Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604287:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.483739Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628065:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.486906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241484732774958:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:58.487178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241484732774959:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:58.487288Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628065:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.487787Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241483546628005:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.489692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483546628070:2108] [node 4] ICP01 ready to work
2025-12-24T02:11:58.489835Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483546628071:2109] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.490735Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650615:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.491047Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486722650619:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.491197Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486722650620:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.491326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486722650621:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.492844Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650630:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.493810Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604287:2107] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.494000Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241484732774956:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:58.496222Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483753999515:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:58.496414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241483753999516:2111] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.497024Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650635:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.504696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485745604297:2111] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.505581Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999518:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.505618Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650630:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.506262Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241484732774965:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20789
2025-12-24T02:11:58.508230Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999518:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.508583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485745604289:2108] [node 6] ICP01 ready to work
2025-12-24T02:11:58.508899Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241484732774964:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20789
2025-12-24T02:11:58.509000Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774965:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.507967Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650615:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.508031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241486722650554:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.510781Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774951:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.510804Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241484732774951:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.513636Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241483546628070:2108] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(23728097ns) {Configure:104(23726418ns) {ProcessConfigured:135(23465952ns) {StartInitialHandshake:157(23465417ns) {DropHandshakes:470(369ns) {DropIncomingHandshake:442(43ns) DropOutgoingHandshake:457(43ns)} GenerateSessionVirtualId:988(116ns) PrepareNewSessionHandshake:477(2332ns) {SwitchToState:179(1861ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.514452Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774949:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.514472Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241484732774949:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.514554Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628078:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.514056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483431988569:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:58.514610Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483753999447:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.513848Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650625:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.513866Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486722650625:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.514931Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628078:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.517486Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483753999447:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6162) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241483753999518:2112] held: no
2025-12-24T02:11:58.514797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486722650625:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.515059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486722650619:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.515079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.515086Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486722650619:2107] [node 4] ICP22 created new session: [2:7587241486722650640:2107]
2025-12-24T02:11:58.515107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486722650640:2107] [node 4] ICS09 handshake done sender: [2:7587241486722650625:2111] self: [2:7587241486722650624:0] peer: [4:7587241483753999524:0] socket: 627 qp: -1
2025-12-24T02:11:58.515114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486722650640:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.515125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.519123Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774964:2114] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.519467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241484732774951:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.519496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241483546628079:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.520154Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999527:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.520048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241484732774889:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.520073Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.520082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241484732774889:2049] [node 3] ICP22 created new session: [1:7587241484732774972:2049]
2025-12-24T02:11:58.520091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241484732774972:2049] [node 3] ICS09 handshake done sender: [1:7587241484732774951:2108] self: [1:7587241484732774950:0] peer: [3:7587241483546628066:0] socket: 616 qp: -1
2025-12-24T02:11:58.520101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241484732774972:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.520110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.520654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241484732774949:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.520756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241484732774888:2048] [node 2] ICP20 outgoing handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.521767Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999528:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.520796Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241483431988499:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(21340645ns) {Configure:104(21340133ns) {ProcessConfigured:135(20919681ns) {StartInitialHandshake:157(20919509ns) {DropHandshakes:470(486ns) {DropIncomingHandshake:442(51ns) DropOutgoingHandshake:457(67ns)} GenerateSessionVirtualId:988(258ns) PrepareNewSessionHandshake:477(653ns) {SwitchToState:179(366ns)}}}}}
2025-12-24T02:11:58.521099Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988575:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.521209Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988576:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.520916Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650642:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.520951Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650643:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.521241Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486722650615:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.521307Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486722650644:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:4632
2025-12-24T02:11:58.521315Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241486722650554:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.521327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.521333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486722650554:2048] [node 1] ICP22 created new session: [2:7587241486722650645:2048]
2025-12-24T02:11:58.521340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486722650645:2048] [node 1] ICS09 handshake done sender: [2:7587241486722650615:2106] self: [2:7587241486722650637:0] peer: [1:7587241484732774948:0] socket: 617 qp: -1
2025-12-24T02:11:58.521345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486722650645:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.521353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.522802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.522823Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241484732774888:2048] [node 2] ICP22 created new session: [1:7587241484732774976:2048]
2025-12-24T02:11:58.522830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241484732774976:2048] [node 2] ICS09 handshake done sender: [1:7587241484732774949:2107] self: [1:7587241484732774948:0] peer: [2:7587241486722650637:0] socket: 613 qp: -1
2025-12-24T02:11:58.522837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241484732774976:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:58.522845Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.523077Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241483546628006:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.522350Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483753999518:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.522425Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483753999447:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.522437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.522444Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483753999447:2049] [node 2] ICP22 created new session: [4:7587241483753999529:2049]
2025-12-24T02:11:58.522451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483753999529:2049] [node 2] ICS09 handshake done sender: [4:7587241483753999518:2112] self: [4:7587241483753999524:0] peer: [2:7587241486722650624:0] socket: 628 qp: -1
2025-12-24T02:11:58.522457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483753999529:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.522491Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.524084Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999531:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.523900Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241484732774978:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20789
2025-12-24T02:11:58.525930Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999531:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.525971Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483753999446:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.526605Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999533:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.526115Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774967:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.526127Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241484732774967:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.526557Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241484732774967:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.523208Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628083:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.524218Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650623:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.524231Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486722650623:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.525518Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486722650623:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.526789Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241484732774956:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.526813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.526820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241484732774956:2109] [node 4] ICP22 created new session: [1:7587241484732774981:2109]
2025-12-24T02:11:58.526827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241484732774981:2109] [node 4] ICS09 handshake done sender: [1:7587241484732774967:2116] self: [1:7587241484732774966:0] peer: [4:7587241483753999532:0] socket: 631 qp: -1
2025-12-24T02:11:58.526834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241484732774981:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:58.526842Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.527990Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988579:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.528132Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774978:2117] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.528396Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604303:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.528486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604304:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.528502Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604305:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.529522Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483431988575:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.529590Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483431988569:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.529822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483431988576:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.529852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483431988499:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.530225Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483753999531:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.530506Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774963:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.530517Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241484732774963:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.529951Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486722650555:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.529973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.529982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486722650555:2049] [node 3] ICP22 created new session: [2:7587241486722650649:2049]
2025-12-24T02:11:58.529989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486722650649:2049] [node 3] ICS09 handshake done sender: [2:7587241486722650623:2110] self: [2:7587241486722650622:0] peer: [3:7587241483546628086:0] socket: 626 qp: -1
2025-12-24T02:11:58.529997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486722650649:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.530003Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.528553Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483546628065:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.528668Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241483546628005:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.528682Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.528689Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483546628005:2048] [node 1] ICP22 created new session: [3:7587241483546628088:2048]
2025-12-24T02:11:58.528695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483546628088:2048] [node 1] ICS09 handshake done sender: [3:7587241483546628065:2107] self: [3:7587241483546628066:0] peer: [1:7587241484732774950:0] socket: 620 qp: -1
2025-12-24T02:11:58.528702Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483546628088:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.528720Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.530646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483431988579:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.530685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483431988500:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.534423Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628087:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.534509Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604302:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.534610Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604303:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.531005Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650644:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.534676Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604305:2118] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.534847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485745604227:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.534939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485745604228:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.538853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774962:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.538876Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241484732774962:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.538799Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483753999446:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241483753999531:2118] held: no
2025-12-24T02:11:58.538930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483753999446:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.538947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.538956Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483753999446:2048] [node 1] ICP22 created new session: [4:7587241483753999534:2048]
2025-12-24T02:11:58.538967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483753999534:2048] [node 1] ICS09 handshake done sender: [4:7587241483753999531:2118] self: [4:7587241483753999532:0] peer: [1:7587241484732774966:0] socket: 660 qp: -1
2025-12-24T02:11:58.538977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483753999534:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.538989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.539641Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241484732774984:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:20789
2025-12-24T02:11:58.539720Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241484732774963:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.539737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241484732774962:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.539563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650627:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.539580Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650629:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.539585Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486722650627:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.539591Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486722650629:2113] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.543505Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999522:2114] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:58.543516Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241483753999522:2114] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
::ffff:127.1.0.0
2025-12-24T02:11:58.544321Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604313:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.544379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988584:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.544423Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486722650627:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.544907Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486722650620:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.544921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.544929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486722650620:2108] [node 5] ICP22 created new session: [2:7587241486722650653:2108]
2025-12-24T02:11:58.544936Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486722650653:2108] [node 5] ICS09 handshake done sender: [2:7587241486722650627:2112] self: [2:7587241486722650626:0] peer: [5:7587241485745604312:0] socket: 644 qp: -1
2025-12-24T02:11:58.544943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486722650653:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.544955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.545377Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486722650629:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.545479Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486722650621:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.545524Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.545531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486722650621:2109] [node 6] ICP22 created new session: [2:7587241486722650656:2109]
2025-12-24T02:11:58.545536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486722650656:2109] [node 6] ICS09 handshake done sender: [2:7587241486722650629:2113] self: [2:7587241486722650628:0] peer: [6:7587241483431988582:0] socket: 657 qp: -1
2025-12-24T02:11:58.545542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486722650656:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.545550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.549296Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604314:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.549269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241484732774958:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.549293Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988583:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.549298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.549308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241484732774958:2110] [node 5] ICP22 created new session: [1:7587241484732774987:2110]
2025-12-24T02:11:58.549315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241484732774987:2110] [node 5] ICS09 handshake done sender: [1:7587241484732774962:2112] self: [1:7587241484732774960:0] peer: [5:7587241485745604311:0] socket: 635 qp: -1
2025-12-24T02:11:58.549322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241484732774987:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:58.549324Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241484732774959:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.549333Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.549349Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.549357Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241484732774959:2111] [node 6] ICP22 created new session: [1:7587241484732774989:2111]
2025-12-24T02:11:58.549365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241484732774989:2111] [node 6] ICS09 handshake done sender: [1:7587241484732774963:2113] self: [1:7587241484732774961:0] peer: [6:7587241483431988581:0] socket: 650 qp: -1
2025-12-24T02:11:58.549373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241484732774989:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:58.549382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.549453Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988585:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.549715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483753999522:2114] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.549938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241483753999516:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.549951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.549957Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483753999516:2111] [node 6] ICP22 created new session: [4:7587241483753999538:2111]
2025-12-24T02:11:58.549964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483753999538:2111] [node 6] ICS09 handshake done sender: [4:7587241483753999522:2114] self: [4:7587241483753999521:0] peer: [6:7587241483431988580:0] socket: 624 qp: -1
2025-12-24T02:11:58.549969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483753999538:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:58.549977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.549179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483546628078:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.549242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650642:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.549296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241486722650555:2049] [node 3] ICP18 handshake (actor [2:7587241486722650642:2118]) is too old
2025-12-24T02:11:58.554763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483431988500:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6162) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483431988579:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.555107Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999528:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.555148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241483753999448:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.556488Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483431988576:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.557458Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483431988579:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.557701Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483431988575:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.557850Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483431988499:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13649) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483431988576:2115] held: no
2025-12-24T02:11:58.558001Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483431988499:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.558019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.558032Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483431988499:2048] [node 1] ICP22 created new session: [6:7587241483431988586:2048]
2025-12-24T02:11:58.558039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483431988586:2048] [node 1] ICS09 handshake done sender: [6:7587241483431988576:2115] self: [6:7587241483431988581:0] peer: [1:7587241484732774961:0] socket: 653 qp: -1
2025-12-24T02:11:58.558046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483431988586:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.558056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.557989Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485745604303:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.558070Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604302:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.558160Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485745604305:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.558229Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241484732774984:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.558266Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241484732774958:2110] [node 5] ICP18 handshake (actor [1:7587241484732774984:2118]) is too old
2025-12-24T02:11:58.555000Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628090:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.558243Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486722650635:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.558278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241486722650620:2108] [node 5] ICP18 handshake (actor [2:7587241486722650635:2117]) is too old
2025-12-24T02:11:58.555056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628091:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
2025-12-24T02:11:58.555080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241483546628006:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.555092Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.555097Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483546628006:2049] [node 2] ICP22 created new session: [3:7587241483546628093:2049]
2025-12-24T02:11:58.555103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483546628093:2049] [node 2] ICS09 handshake done sender: [3:7587241483546628078:2114] self: [3:7587241483546628086:0] peer: [2:7587241486722650622:0] socket: 637 qp: -1
2025-12-24T02:11:58.555108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483546628093:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.555114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.555287Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [3:7587241483546628006:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.556082Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628076:2112] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.556087Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483546628076:2112] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.556650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628091:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.557516Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483546628076:2112] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.557664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483546628070:2108] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.557692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.557700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483546628070:2108] [node 4] ICP22 created new session: [3:7587241483546628097:2108]
2025-12-24T02:11:58.557706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483546628097:2108] [node 4] ICS09 handshake done sender: [3:7587241483546628076:2112] self: [3:7587241483546628072:0] peer: [4:7587241483753999540:0] socket: 641 qp: -1
2025-12-24T02:11:58.557713Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483546628097:2108] [node 4] ICS10 traffic start
2025-12-24T02:11:58.557722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.558336Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628090:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.562559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241483753999448:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20195) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241483753999528:2117] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.562979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485745604227:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.563043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.563056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485745604227:2048] [node 1] ICP22 created new session: [5:7587241485745604315:2048]
2025-12-24T02:11:58.563064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485745604315:2048] [node 1] ICS09 handshake done sender: [5:7587241485745604303:2116] self: [5:7587241485745604311:0] peer: [1:7587241484732774960:0] socket: 647 qp: -1
2025-12-24T02:11:58.563073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485745604315:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.563081Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.563261Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999541:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.563407Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483753999528:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.563521Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241483753999448:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.563532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.563539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483753999448:2050] [node 3] ICP22 created new session: [4:7587241483753999542:2050]
2025-12-24T02:11:58.563552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483753999542:2050] [node 3] ICS09 handshake done sender: [4:7587241483753999528:2117] self: [4:7587241483753999540:0] peer: [3:7587241483546628072:0] socket: 648 qp: -1
2025-12-24T02:11:58.563560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483753999542:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.563567Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.563713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483431988500:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.563743Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.563749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483431988500:2049] [node 2] ICP22 created new session: [6:7587241483431988589:2049]
2025-12-24T02:11:58.563754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483431988589:2049] [node 2] ICS09 handshake done sender: [6:7587241483431988579:2117] self: [6:7587241483431988582:0] peer: [2:7587241486722650628:0] socket: 664 qp: -1
2025-12-24T02:11:58.563759Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483431988589:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.563767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.563852Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483431988569:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.563873Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.563882Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483431988569:2110] [node 4] ICP22 created new session: [6:7587241483431988591:2110]
2025-12-24T02:11:58.563891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483431988591:2110] [node 4] ICS09 handshake done sender: [6:7587241483431988575:2114] self: [6:7587241483431988580:0] peer: [4:7587241483753999521:0] socket: 652 qp: -1
2025-12-24T02:11:58.563897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483431988591:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:58.563904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.563250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485745604228:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.563269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.563277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485745604228:2049] [node 2] ICP22 created new session: [5:7587241485745604317:2049]
2025-12-24T02:11:58.563283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485745604317:2049] [node 2] ICS09 handshake done sender: [5:7587241485745604305:2118] self: [5:7587241485745604312:0] peer: [2:7587241486722650626:0] socket: 658 qp: -1
2025-12-24T02:11:58.563288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485745604317:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.563293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.563634Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604304:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.565297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [5:7587241485745604228:2049] [node 2] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.565312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485745604297:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.565389Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485745604229:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.565699Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241483546628100:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:58190
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.566242Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628085:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.566254Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483546628085:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.565532Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988588:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.566009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483431988588:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.566136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483431988501:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.566229Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988595:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.566547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628077:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.566561Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241483546628077:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.566559Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999520:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.566587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241483753999520:2113] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.567547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604323:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.567547Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483431988595:2122] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.567579Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483431988561:2108] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.567532Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483546628077:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.567774Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483546628071:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.567796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.567806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483546628071:2109] [node 5] ICP22 created new session: [3:7587241483546628102:2109]
2025-12-24T02:11:58.567812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483546628102:2109] [node 5] ICS09 handshake done sender: [3:7587241483546628077:2113] self: [3:7587241483546628075:0] peer: [5:7587241485745604322:0] socket: 642 qp: -1
2025-12-24T02:11:58.567819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483546628102:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:58.567833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.568673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241483753999520:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.568833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241483753999544:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:14443
2025-12-24T02:11:58.568927Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241483753999515:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.568942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.568952Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241483753999515:2110] [node 5] ICP22 created new session: [4:7587241483753999546:2110]
2025-12-24T02:11:58.568958Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241483753999546:2110] [node 5] ICS09 handshake done sender: [4:7587241483753999520:2113] self: [4:7587241483753999519:0] peer: [5:7587241485745604321:0] socket: 634 qp: -1
2025-12-24T02:11:58.568964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241483753999546:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:58.568971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.568847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485745604308:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.568861Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241485745604308:2119] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.569520Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485745604304:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.569758Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485745604229:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.569776Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.569783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485745604229:2050] [node 3] ICP22 created new session: [5:7587241485745604326:2050]
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.569790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485745604326:2050] [node 3] ICS09 handshake done sender: [5:7587241485745604304:2117] self: [5:7587241485745604322:0] peer: [3:7587241483546628075:0] socket: 614 qp: -1
2025-12-24T02:11:58.569800Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485745604326:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.569807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.569974Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241483546628100:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.570039Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241483546628085:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.569880Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988598:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.570325Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483431988599:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:55604
2025-12-24T02:11:58.570450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483431988588:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.570845Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241483753999544:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.571183Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483431988501:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.571221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.571229Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483431988501:2050] [node 3] ICP22 created new session: [6:7587241483431988600:2050]
2025-12-24T02:11:58.571235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483431988600:2050] [node 3] ICS09 handshake done sender: [6:7587241483431988588:2121] self: [6:7587241483431988596:0] peer: [3:7587241483546628084:0] socket: 630 qp: -1
2025-12-24T02:11:58.571244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483431988600:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.571251Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.572945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241483546628079:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.572974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.572980Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241483546628079:2110] [node 6] ICP22 created new session: [3:7587241483546628106:2110]
2025-12-24T02:11:58.572985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241483546628106:2110] [node 6] ICS09 handshake done sender: [3:7587241483546628085:2116] self: [3:7587241483546628084:0] peer: [6:7587241483431988596:0] socket: 640 qp: -1
2025-12-24T02:11:58.572992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241483546628106:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.573008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.573274Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485745604308:2119] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.573297Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485745604324:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:21825
2025-12-24T02:11:58.573414Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241485745604289:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.573435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.573440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485745604289:2108] [node 6] ICP22 created new session: [5:7587241485745604328:2108]
2025-12-24T02:11:58.573447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485745604328:2108] [node 6] ICS09 handshake done sender: [5:7587241485745604308:2119] self: [5:7587241485745604307:0] peer: [6:7587241483431988597:0] socket: 656 qp: -1
2025-12-24T02:11:58.573453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485745604328:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:58.573461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.573573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [4:7587241483753999515:2110] [node 5] ICP18 handshake (actor [4:7587241483753999544:2121]) is too old
2025-12-24T02:11:58.573933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485745604302:2115] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.574011Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485745604297:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.574028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.574033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485745604297:2111] [node 4] ICP22 created new session: [5:7587241485745604330:2111]
2025-12-24T02:11:58.574038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485745604330:2111] [node 4] ICS09 handshake done sender: [5:7587241485745604302:2115] self: [5:7587241485745604321:0] peer: [4:7587241483753999519:0] socket: 636 qp: -1
2025-12-24T02:11:58.574044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485745604330:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:58.574051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.574474Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483431988561:2108] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16725) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483431988595:2122] held: no
2025-12-24T02:11:58.574942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483431988595:2122] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.575013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483431988561:2108] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.575021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.575027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483431988561:2108] [node 5] ICP22 created new session: [6:7587241483431988602:2108]
2025-12-24T02:11:58.575032Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483431988602:2108] [node 5] ICS09 handshake done sender: [6:7587241483431988595:2122] self: [6:7587241483431988597:0] peer: [5:7587241485745604307:0] socket: 666 qp: -1
2025-12-24T02:11:58.575037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483431988602:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.575044Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.584781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:58.584866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241483546628088:2048] [node 1] ICS01 socket: 620 reason# EndOfStream
2025-12-24T02:11:58.584873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241483546628005:2048] [node 1] ICP30 unregister session Session# [3:7587241483546628088:2048] VirtualId# [3:7587241483546628066:0]
2025-12-24T02:11:58.584879Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241483546628088:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.584919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.584946Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.584269Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:58.584388Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241486722650645:2048] [node 1] ICS01 socket: 617 reason# EndOfStream
2025-12-24T02:11:58.584398Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241486722650554:2048] [node 1] ICP30 unregister session Session# [2:7587241486722650645:2048] VirtualId# [2:7587241486722650637:0]
2025-12-24T02:11:58.584407Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241486722650645:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.584461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.584481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.587332Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:58.587444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241483753999534:2048] [node 1] ICS01 socket: 660 reason# EndOfStream
2025-12-24T02:11:58.587453Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241483753999446:2048] [node 1] ICP30 unregister session Session# [4:7587241483753999534:2048] VirtualId# [4:7587241483753999532:0]
2025-12-24T02:11:58.587469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241483753999534:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.587530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.587561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.588159Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:58.588240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483431988586:2048] [node 1] ICS01 socket: 653 reason# EndOfStream
2025-12-24T02:11:58.588247Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483431988499:2048] [node 1] ICP30 unregister session Session# [6:7587241483431988586:2048] VirtualId# [6:7587241483431988581:0]
2025-12-24T02:11:58.588253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483431988586:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.588301Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.588327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.587717Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:58.587819Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241485745604315:2048] [node 1] ICS07 socket disconnect 647 reason# EndOfStream
2025-12-24T02:11:58.587826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485745604315:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.587878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.587908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241485745604315:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:58.588551Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241485745604227:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13649) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.588560Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485745604315:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.588565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485745604227:2048] [node 1] ICP30 unregister session Session# [5:7587241485745604315:2048] VirtualId# [5:7587241485745604311:0]
2025-12-24T02:11:58.588633Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.588639Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241485745604227:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:13649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.588643Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:13649) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.598686Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:58.598845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241483546628093:2049] [node 2] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:58.598856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241483546628006:2049] [node 2] ICP30 unregister session Session# [3:7587241483546628093:2049] VirtualId# [3:7587241483546628086:0]
2025-12-24T02:11:58.598863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241483546628093:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.598913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.598942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.598765Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241485745604227:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:58.600709Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:58.600813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241483753999529:2049] [node 2] ICS01 socket: 628 reason# EndOfStream
2025-12-24T02:11:58.600822Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241483753999447:2049] [node 2] ICP30 unregister session Session# [4:7587241483753999529:2049] VirtualId# [4:7587241483753999524:0]
2025-12-24T02:11:58.600829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241483753999529:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.600889Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.600935Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.603492Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:58.603569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241485745604317:2049] [node 2] ICS07 socket disconnect 658 reason# EndOfStream
2025-12-24T02:11:58.603575Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485745604317:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.603605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.603644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241485745604317:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:58.603188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:58.603348Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483431988589:2049] [node 2] ICS01 socket: 664 reason# EndOfStream
2025-12-24T02:11:58.603359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483431988500:2049] [node 2] ICP30 unregister session Session# [6:7587241483431988589:2049] VirtualId# [6:7587241483431988582:0]
2025-12-24T02:11:58.603366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483431988589:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.603422Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.603449Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.604168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241485745604228:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6162) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.604184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485745604317:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.604189Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485745604228:2049] [node 2] ICP30 unregister session Session# [5:7587241485745604317:2049] VirtualId# [5:7587241485745604312:0]
2025-12-24T02:11:58.604256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.604271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241485745604228:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.604275Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6162) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.614191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241485745604228:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:58.625904Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:58.626018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483431988600:2050] [node 3] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:58.626031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483431988501:2050] [node 3] ICP30 unregister session Session# [6:7587241483431988600:2050] VirtualId# [6:7587241483431988596:0]
2025-12-24T02:11:58.626040Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483431988600:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.626107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.626138Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.626803Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:58.626895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241485745604326:2050] [node 3] ICS07 socket disconnect 614 reason# EndOfStream
2025-12-24T02:11:58.626908Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485745604326:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.626968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.627007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241485745604326:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:58.627433Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241485745604229:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20195) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.627451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485745604326:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.627457Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485745604229:2050] [node 3] ICP30 unregister session Session# [5:7587241485745604326:2050] VirtualId# [5:7587241485745604322:0]
2025-12-24T02:11:58.627532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.627544Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241485745604229:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:20195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.627549Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:20195) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.637739Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241485745604229:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:58.643599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:58.643735Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241485745604330:2111] [node 4] ICS07 socket disconnect 636 reason# EndOfStream
2025-12-24T02:11:58.643756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485745604330:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.643830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.643863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241485745604330:2111] [node 4] ICS15 start handshake
2025-12-24T02:11:58.644675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241485745604297:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.644693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485745604330:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.644699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485745604297:2111] [node 4] ICP30 unregister session Session# [5:7587241485745604330:2111] VirtualId# [5:7587241485745604321:0]
2025-12-24T02:11:58.644756Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:58.644770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241485745604297:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.644772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.643844Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:58.643956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241483431988591:2110] [node 4] ICS07 socket disconnect 652 reason# EndOfStream
2025-12-24T02:11:58.643965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483431988591:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.644028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.644051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241483431988591:2110] [node 4] ICS15 start handshake
2025-12-24T02:11:58.644411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483431988569:2110] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.644423Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483431988591:2110] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.644429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483431988569:2110] [node 4] ICP30 unregister session Session# [6:7587241483431988591:2110] VirtualId# [6:7587241483431988580:0]
2025-12-24T02:11:58.644498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:58.644507Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241483431988569:2110] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.644511Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:27448) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.654879Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241483431988569:2110] [node 4] ICP33 wake up from error state
2025-12-24T02:11:58.658757Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:58.679935Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241486957157463:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:58.679950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241486957157464:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.681756Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486354214203:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.681832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486354214204:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.683709Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241485494060932:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.683728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241485494060933:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.685964Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486606370843:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.685976Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486606370845:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.686004Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486606370844:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.690188Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483622868784:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.690201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483622868786:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.690265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483622868785:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.687913Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485811749190:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.687922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485811749191:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.687930Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485811749192:2050] [node 3] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.704370Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486354214262:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5925
2025-12-24T02:11:58.703803Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483622868845:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.705030Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241486957157526:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.705557Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241486957157530:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.704678Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749251:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.706720Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486354214266:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.707112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486354214273:2110] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.707486Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370904:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
2025-12-24T02:11:58.707486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241486957157535:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:58.707769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241486354214279:2113] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.715062Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241486957157537:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3887
2025-12-24T02:11:58.716315Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241486957157536:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3887
2025-12-24T02:11:58.719406Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214262:2106] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.719937Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157536:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.722012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157523:2106] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.722038Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241486957157523:2106] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.721987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241485494060996:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.722563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241485494060997:2108] [node 5] ICP01 ready to work
2025-12-24T02:11:58.723616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486606370911:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:58.724262Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370904:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.724501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486606370844:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.726307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486606370909:2109] [node 5] ICP01 ready to work
2025-12-24T02:11:58.726565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241486354214203:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.726596Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214274:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.726606Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486354214274:2111] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.725172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485811749258:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:58.725206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241485811749259:2110] [node 6] ICP01 ready to work
2025-12-24T02:11:58.731466Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241485494061006:2109] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.731584Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061000:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.732509Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241486957157523:2106] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.732474Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486354214284:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5925
2025-12-24T02:11:58.732629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486354214262:2106] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.732652Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241483622868856:2110] [node 4] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.733061Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241486957157546:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.733271Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370922:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.734381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749251:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.735509Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241486354214203:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.735532Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.735539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486354214203:2048] [node 1] ICP22 created new session: [2:7587241486354214285:2048]
2025-12-24T02:11:58.735546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486354214285:2048] [node 1] ICS09 handshake done sender: [2:7587241486354214262:2106] self: [2:7587241486354214282:0] peer: [1:7587241486957157522:0] socket: 619 qp: -1
2025-12-24T02:11:58.735553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486354214285:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.735564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.736979Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370923:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.738101Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370924:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.738158Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749271:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.738208Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486354214274:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.738375Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486606370904:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.738431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486606370844:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.738448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.738454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486606370844:2049] [node 2] ICP22 created new session: [4:7587241486606370925:2049]
2025-12-24T02:11:58.738460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486606370925:2049] [node 2] ICS09 handshake done sender: [4:7587241486606370904:2108] self: [4:7587241486606370919:0] peer: [2:7587241486354214272:0] socket: 628 qp: -1
2025-12-24T02:11:58.738467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486606370925:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.738573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241486957157463:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.738603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
No RDMA context found for GID: 2025-12-24T02:11:58.738612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241486957157463:2048] [node 2] ICP22 created new session: [1:7587241486957157549:2048]
::ffff:127.1.0.0
2025-12-24T02:11:58.738620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241486957157549:2048] [node 2] ICS09 handshake done sender: [1:7587241486957157523:2106] self: [1:7587241486957157522:0] peer: [2:7587241486354214282:0] socket: 618 qp: -1
2025-12-24T02:11:58.738628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241486957157549:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:58.738635Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.738579Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241483622868785:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(18175058ns) {Configure:104(18174021ns) {ProcessConfigured:135(17711597ns) {StartInitialHandshake:157(17711297ns) {DropHandshakes:470(587ns) {DropIncomingHandshake:442(96ns) DropOutgoingHandshake:457(97ns)} GenerateSessionVirtualId:988(432ns) PrepareNewSessionHandshake:477(1053ns) {SwitchToState:179(423ns)}}}}}
2025-12-24T02:11:58.738485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.738800Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749264:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.738906Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061002:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.739522Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241485494060933:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16894743ns) {Configure:104(16894001ns) {ProcessConfigured:135(16508668ns) {StartInitialHandshake:157(16508279ns) {DropHandshakes:470(509ns) {DropIncomingHandshake:442(79ns) DropOutgoingHandshake:457(71ns)} GenerateSessionVirtualId:988(322ns) PrepareNewSessionHandshake:477(633ns) {SwitchToState:179(273ns)}}}}}
2025-12-24T02:11:58.739572Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061000:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.742802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486354214266:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.742826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.742834Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486354214266:2107] [node 4] ICP22 created new session: [2:7587241486354214289:2107]
2025-12-24T02:11:58.742841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486354214289:2107] [node 4] ICS09 handshake done sender: [2:7587241486354214274:2111] self: [2:7587241486354214272:0] peer: [4:7587241486606370919:0] socket: 627 qp: -1
2025-12-24T02:11:58.742849Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486354214289:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.742867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.743655Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868861:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.743690Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061003:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.743746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241485494060933:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.746914Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241486606370909:2109] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(20470775ns) {Configure:104(20470134ns) {ProcessConfigured:135(20093302ns) {StartInitialHandshake:157(20092746ns) {DropHandshakes:470(451ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(76ns)} GenerateSessionVirtualId:988(360ns) PrepareNewSessionHandshake:477(1154ns) {SwitchToState:179(548ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.747147Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486354214291:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5925
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.747258Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241486957157548:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3887
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.747312Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241486957157551:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:3887
2025-12-24T02:11:58.748490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749264:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.748561Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485811749190:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.749042Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061002:2112] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.749338Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061015:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.749437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241485494060932:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.749813Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157548:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.749847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157537:2113] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.749699Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061015:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.751029Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868864:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.751121Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868863:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.751255Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483622868864:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.751296Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483622868785:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.751303Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483622868863:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.751321Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483622868786:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.751400Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061008:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.751411Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241485494061008:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.751442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214283:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.751455Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486354214283:2115] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.752097Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868867:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.752251Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241485494061008:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.752499Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483622868863:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.752370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241485494061006:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.752393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.752400Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241485494061006:2109] [node 6] ICP22 created new session: [3:7587241485494061019:2109]
2025-12-24T02:11:58.752408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241485494061019:2109] [node 6] ICS09 handshake done sender: [3:7587241485494061008:2115] self: [3:7587241485494061007:0] peer: [6:7587241483622868866:0] socket: 664 qp: -1
2025-12-24T02:11:58.752416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241485494061019:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.752437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.752838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483622868786:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.752856Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.752862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483622868786:2050] [node 3] ICP22 created new session: [6:7587241483622868868:2050]
2025-12-24T02:11:58.752868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483622868868:2050] [node 3] ICS09 handshake done sender: [6:7587241483622868863:2116] self: [6:7587241483622868866:0] peer: [3:7587241485494061007:0] socket: 632 qp: -1
2025-12-24T02:11:58.752873Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483622868868:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.752895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.753048Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [6:7587241483622868786:2050] [node 3] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.754788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749271:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.754786Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214269:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.754805Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486354214269:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.754863Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157541:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.754880Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241486957157541:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.755211Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241486606370844:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(16715570ns) {HandleHandshakeStatus:329(16715060ns) {DropIncomingHandshake:442(200ns) DropOutgoingHandshake:457(993ns) SwitchToState:179(101ns) ProcessPendingSessionEvents:495(11474742ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 11459296ns max# 11398081ns}}}
2025-12-24T02:11:58.755532Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157529:2109] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.755544Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241486957157529:2109] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.756171Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241486957157529:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.756487Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241486957157463:2048] [node 2] ICP35 event processing took too much time PendingConnection:279(23737670ns) {HandleHandshakeStatus:329(23737062ns) {DropIncomingHandshake:442(5978ns) DropOutgoingHandshake:457(156ns) SwitchToState:179(95ns) ProcessPendingSessionEvents:495(9587778ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 9572566ns max# 9503100ns}}}
2025-12-24T02:11:58.756500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241486957157464:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.756514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.756520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241486957157464:2049] [node 3] ICP22 created new session: [1:7587241486957157555:2049]
2025-12-24T02:11:58.756524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241486957157555:2049] [node 3] ICS09 handshake done sender: [1:7587241486957157529:2109] self: [1:7587241486957157528:0] peer: [3:7587241485494061016:0] socket: 634 qp: -1
2025-12-24T02:11:58.756531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241486957157555:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.756537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.756592Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485811749191:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.757079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241486354214293:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:5925
2025-12-24T02:11:58.757277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486354214269:2109] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.757368Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486354214204:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.757382Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.757388Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486354214204:2049] [node 3] ICP22 created new session: [2:7587241486354214295:2049]
2025-12-24T02:11:58.757393Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486354214295:2049] [node 3] ICS09 handshake done sender: [2:7587241486354214269:2109] self: [2:7587241486354214268:0] peer: [3:7587241485494061014:0] socket: 625 qp: -1
2025-12-24T02:11:58.757399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486354214295:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.757405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.763003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370922:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.763172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486606370843:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.765955Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749274:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.766422Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214277:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.766439Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241486354214277:2112] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.766549Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241485494060932:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12047) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241485494061002:2112] held: no
2025-12-24T02:11:58.766652Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241486957157541:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.766858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157540:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.766867Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241486957157540:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.766920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241486957157530:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.766931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.766939Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241486957157530:2108] [node 5] ICP22 created new session: [1:7587241486957157559:2108]
2025-12-24T02:11:58.766945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241486957157559:2108] [node 5] ICS09 handshake done sender: [1:7587241486957157541:2115] self: [1:7587241486957157533:0] peer: [5:7587241485811749272:0] socket: 635 qp: -1
2025-12-24T02:11:58.766951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241486957157559:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.766958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.767039Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241486354214293:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.767113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [2:7587241486354214204:2049] [node 3] ICP18 handshake (actor [2:7587241486354214293:2118]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.769547Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370930:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.769622Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370931:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
2025-12-24T02:11:58.769639Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370932:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.772977Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749275:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.772995Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749276:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.773103Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868871:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.775277Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486354214277:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.775953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486354214273:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.775967Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.775975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486354214273:2110] [node 5] ICP22 created new session: [2:7587241486354214299:2110]
2025-12-24T02:11:58.775994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486354214299:2110] [node 5] ICS09 handshake done sender: [2:7587241486354214277:2112] self: [2:7587241486354214276:0] peer: [5:7587241485811749273:0] socket: 659 qp: -1
2025-12-24T02:11:58.776012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486354214299:2110] [node 5] ICS10 traffic start
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.776360Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868872:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.776535Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483622868864:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.776688Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241486957157540:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.775552Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485811749271:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.777542Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370930:2119] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.777573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370931:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.777641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486606370845:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.778432Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486606370922:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.778536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486606370843:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.778554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.778805Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241485494060933:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [3:7587241485494061000:2111] held: no
2025-12-24T02:11:58.778965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157543:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.778978Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241486957157543:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.779031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241486957157526:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.779041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.779048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241486957157526:2107] [node 4] ICP22 created new session: [1:7587241486957157562:2107]
2025-12-24T02:11:58.779053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241486957157562:2107] [node 4] ICS09 handshake done sender: [1:7587241486957157540:2114] self: [1:7587241486957157532:0] peer: [4:7587241486606370929:0] socket: 647 qp: -1
2025-12-24T02:11:58.779059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241486957157562:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.779067Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.780646Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061005:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.782898Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241486957157543:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.780660Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241485494061005:2114] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.783056Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241486957157535:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.783077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.783082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241486957157535:2110] [node 6] ICP22 created new session: [1:7587241486957157565:2110]
2025-12-24T02:11:58.783088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241486957157565:2110] [node 6] ICS09 handshake done sender: [1:7587241486957157543:2116] self: [1:7587241486957157539:0] peer: [6:7587241483622868875:0] socket: 677 qp: -1
2025-12-24T02:11:58.776020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.776179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241486354214283:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.776678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241486354214279:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.776688Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.776694Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241486354214279:2113] [node 6] ICP22 created new session: [2:7587241486354214301:2113]
2025-12-24T02:11:58.776700Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241486354214301:2113] [node 6] ICS09 handshake done sender: [2:7587241486354214283:2115] self: [2:7587241486354214280:0] peer: [6:7587241483622868865:0] socket: 665 qp: -1
2025-12-24T02:11:58.776706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241486354214301:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:58.776714Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.777338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483622868785:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.777356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.777365Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483622868785:2049] [node 2] ICP22 created new session: [6:7587241483622868873:2049]
2025-12-24T02:11:58.777373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483622868873:2049] [node 2] ICS09 handshake done sender: [6:7587241483622868864:2117] self: [6:7587241483622868865:0] peer: [2:7587241486354214280:0] socket: 666 qp: -1
2025-12-24T02:11:58.777383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483622868873:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.777391Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.778521Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483622868871:2119] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.778552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483622868861:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.778580Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483622868784:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.778591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483622868856:2110] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.783079Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868877:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.783224Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483622868871:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.783281Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483622868784:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.783292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.783297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483622868784:2048] [node 1] ICP22 created new session: [6:7587241483622868878:2048]
2025-12-24T02:11:58.783302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483622868878:2048] [node 1] ICS09 handshake done sender: [6:7587241483622868871:2119] self: [6:7587241483622868875:0] peer: [1:7587241486957157539:0] socket: 679 qp: -1
2025-12-24T02:11:58.783307Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483622868878:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.783313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.775616Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485811749191:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.775634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.775642Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485811749191:2049] [node 2] ICP22 created new session: [5:7587241485811749277:2049]
2025-12-24T02:11:58.775649Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485811749277:2049] [node 2] ICS09 handshake done sender: [5:7587241485811749271:2117] self: [5:7587241485811749273:0] peer: [2:7587241486354214276:0] socket: 661 qp: -1
2025-12-24T02:11:58.775657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485811749277:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.775681Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.795026Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485811749264:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.778560Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486606370843:2048] [node 1] ICP22 created new session: [4:7587241486606370935:2048]
2025-12-24T02:11:58.778566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486606370935:2048] [node 1] ICS09 handshake done sender: [4:7587241486606370922:2115] self: [4:7587241486606370929:0] peer: [1:7587241486957157532:0] socket: 648 qp: -1
2025-12-24T02:11:58.778571Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486606370935:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.778578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.793492Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370915:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.793503Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241486606370915:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.794405Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370923:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.795581Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [4:7587241486606370843:2048] [node 1] ICP35 event processing took too much time PendingConnection:279(16928701ns) {HandleHandshakeStatus:329(16928149ns) {DropIncomingHandshake:442(151ns) DropOutgoingHandshake:457(1414ns) SwitchToState:179(101ns) ProcessPendingSessionEvents:495(16845926ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 99915ns max# 22610ns}}}
2025-12-24T02:11:58.795684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486606370915:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.795856Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241486606370911:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.795874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.795880Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486606370911:2110] [node 6] ICP22 created new session: [4:7587241486606370939:2110]
2025-12-24T02:11:58.795885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486606370939:2110] [node 6] ICS09 handshake done sender: [4:7587241486606370915:2113] self: [4:7587241486606370914:0] peer: [6:7587241483622868876:0] socket: 642 qp: -1
2025-12-24T02:11:58.795891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486606370939:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.795901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.783094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241486957157565:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.783103Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.796898Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061022:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.797027Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061023:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.795782Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749274:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.795800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485811749190:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.795813Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.795820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485811749190:2048] [node 1] ICP22 created new session: [5:7587241485811749279:2048]
2025-12-24T02:11:58.795826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485811749279:2048] [node 1] ICS09 handshake done sender: [5:7587241485811749264:2114] self: [5:7587241485811749272:0] peer: [1:7587241486957157533:0] socket: 636 qp: -1
2025-12-24T02:11:58.795833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485811749279:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.795846Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.797115Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [5:7587241485811749191:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(21394025ns) {HandleHandshakeStatus:329(21393151ns) {DropIncomingHandshake:442(684ns) DropOutgoingHandshake:457(1652ns) SwitchToState:179(272ns) ProcessPendingSessionEvents:495(21273979ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 104351ns max# 25342ns}}}
2025-12-24T02:11:58.797252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485811749258:2109] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.798929Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241485494061000:2111] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.799131Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241486957157530:2108] [node 5] ICP35 event processing took too much time PendingConnection:279(32115316ns) {HandleHandshakeStatus:329(32114653ns) {DropIncomingHandshake:442(124ns) DropOutgoingHandshake:457(152ns) SwitchToState:179(501ns) ProcessPendingSessionEvents:495(32030620ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 24084319ns max# 24008544ns}}}
2025-12-24T02:11:58.800020Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483622868856:2110] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:18423) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241483622868861:2114] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.800021Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241485494061002:2112] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.800136Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241485494061005:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.800224Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241485494060933:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.800236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.800240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241485494060933:2049] [node 2] ICP22 created new session: [3:7587241485494061027:2049]
2025-12-24T02:11:58.800246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241485494061027:2049] [node 2] ICS09 handshake done sender: [3:7587241485494061000:2111] self: [3:7587241485494061014:0] peer: [2:7587241486354214268:0] socket: 630 qp: -1
2025-12-24T02:11:58.800251Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241485494061027:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.800256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.799588Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241486354214273:2110] [node 5] ICP35 event processing took too much time PendingConnection:279(23542809ns) {HandleHandshakeStatus:329(23542320ns) {DropIncomingHandshake:442(62ns) DropOutgoingHandshake:457(158ns) SwitchToState:179(143ns) ProcessPendingSessionEvents:495(23445463ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 85451ns max# 15640ns}}}
2025-12-24T02:11:58.799813Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241486354214273:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.800403Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241486354214279:2113] [node 6] ICP35 event processing took too much time PendingConnection:279(23620932ns) {HandleHandshakeStatus:329(23620425ns) {DropIncomingHandshake:442(75ns) DropOutgoingHandshake:457(161ns) SwitchToState:179(89ns) ProcessPendingSessionEvents:495(23549978ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16119308ns max# 16044592ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.800988Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [1:7587241486957157526:2107] [node 4] ICP35 event processing took too much time PendingConnection:279(21863390ns) {HandleHandshakeStatus:329(21862711ns) {DropIncomingHandshake:442(77ns) DropOutgoingHandshake:457(64ns) SwitchToState:179(90ns) ProcessPendingSessionEvents:495(21784982ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15718818ns max# 15647499ns}}}
2025-12-24T02:11:58.801030Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241486957157551:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.801324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241486957157526:2107] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:58.800343Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486606370927:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.801352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241486957157530:2108] [node 5] ICP18 handshake (actor [1:7587241486957157551:2119]) is too old
2025-12-24T02:11:58.800354Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241486606370927:2118] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.801531Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486606370927:2118] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.800984Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [6:7587241483622868785:2049] [node 2] ICP35 event processing took too much time PendingConnection:279(23544751ns) {HandleHandshakeStatus:329(23543987ns) {DropIncomingHandshake:442(220ns) DropOutgoingHandshake:457(2172ns) SwitchToState:179(246ns) ProcessPendingSessionEvents:495(23438156ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 15625110ns max# 15548321ns}}}
2025-12-24T02:11:58.801791Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749284:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.802924Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241486606370909:2109] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.802942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.802949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486606370909:2109] [node 5] ICP22 created new session: [4:7587241486606370942:2109]
2025-12-24T02:11:58.802955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486606370942:2109] [node 5] ICS09 handshake done sender: [4:7587241486606370927:2118] self: [4:7587241486606370921:0] peer: [5:7587241485811749282:0] socket: 674 qp: -1
2025-12-24T02:11:58.802959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486606370942:2109] [node 5] ICS10 traffic start
2025-12-24T02:11:58.802966Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.803333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241485494060932:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.803347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.803352Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241485494060932:2048] [node 1] ICP22 created new session: [3:7587241485494061029:2048]
2025-12-24T02:11:58.803357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241485494061029:2048] [node 1] ICS09 handshake done sender: [3:7587241485494061002:2112] self: [3:7587241485494061016:0] peer: [1:7587241486957157528:0] socket: 637 qp: -1
2025-12-24T02:11:58.803361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241485494061029:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.803365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.803477Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241485494060996:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.803486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.803489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241485494060996:2107] [node 4] ICP22 created new session: [3:7587241485494061031:2107]
2025-12-24T02:11:58.803492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241485494061031:2107] [node 4] ICS09 handshake done sender: [3:7587241485494061005:2114] self: [3:7587241485494061004:0] peer: [4:7587241486606370933:0] socket: 663 qp: -1
2025-12-24T02:11:58.803495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241485494061031:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:58.803498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.803734Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061003:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.803882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241485494061026:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48202
2025-12-24T02:11:58.804903Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868881:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
2025-12-24T02:11:58.804992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868880:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.805146Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749285:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.805312Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485811749274:2118] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.805397Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241483622868881:2123] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.805431Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483622868861:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.805511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483622868856:2110] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.805526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.805532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483622868856:2110] [node 4] ICP22 created new session: [6:7587241483622868882:2110]
2025-12-24T02:11:58.805538Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483622868882:2110] [node 4] ICS09 handshake done sender: [6:7587241483622868861:2114] self: [6:7587241483622868876:0] peer: [4:7587241486606370914:0] socket: 646 qp: -1
2025-12-24T02:11:58.805544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483622868882:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:58.805552Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.805751Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241483622868845:2107] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.805431Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485811749258:2109] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.805448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.805456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485811749258:2109] [node 4] ICP22 created new session: [5:7587241485811749286:2109]
2025-12-24T02:11:58.805461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485811749286:2109] [node 4] ICS09 handshake done sender: [5:7587241485811749274:2118] self: [5:7587241485811749282:0] peer: [4:7587241486606370921:0] socket: 675 qp: -1
2025-12-24T02:11:58.805468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485811749286:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:58.805475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.807215Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241485494060996:2107] [node 4] ICP18 handshake (actor [3:7587241485494061003:2113]) is too old
2025-12-24T02:11:58.807558Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061026:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.808032Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749269:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.808052Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241485811749269:2116] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.809436Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241483622868885:2124] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:38682
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.809423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241486606370845:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19132) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241486606370930:2119] held: no
2025-12-24T02:11:58.809745Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486606370944:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:63303
2025-12-24T02:11:58.809964Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486606370930:2119] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.810040Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485811749269:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.810272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241483622868881:2123] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.810153Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486606370845:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.810165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.810171Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486606370845:2050] [node 3] ICP22 created new session: [4:7587241486606370945:2050]
2025-12-24T02:11:58.810174Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486606370945:2050] [node 3] ICS09 handshake done sender: [4:7587241486606370930:2119] self: [4:7587241486606370933:0] peer: [3:7587241485494061004:0] socket: 670 qp: -1
2025-12-24T02:11:58.810179Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486606370945:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.810184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.810473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241483622868845:2107] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.810489Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.810495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241483622868845:2107] [node 5] ICP22 created new session: [6:7587241483622868886:2107]
2025-12-24T02:11:58.810501Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241483622868886:2107] [node 5] ICS09 handshake done sender: [6:7587241483622868881:2123] self: [6:7587241483622868884:0] peer: [5:7587241485811749263:0] socket: 688 qp: -1
2025-12-24T02:11:58.810507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241483622868886:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:58.810514Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.810486Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241485811749259:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.810497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.810502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485811749259:2110] [node 6] ICP22 created new session: [5:7587241485811749289:2110]
2025-12-24T02:11:58.810505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485811749289:2110] [node 6] ICS09 handshake done sender: [5:7587241485811749269:2116] self: [5:7587241485811749263:0] peer: [6:7587241483622868884:0] socket: 681 qp: -1
2025-12-24T02:11:58.810508Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485811749289:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.810513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.812512Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241485811749284:2121] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.812565Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241485811749192:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.812701Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241485494061011:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.812710Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241485494061011:2117] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.813343Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241485811749292:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:30075
2025-12-24T02:11:58.813425Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241485494061011:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.813514Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241485811749284:2121] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.813514Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241485494060997:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.813536Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.813553Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241485494060997:2108] [node 5] ICP22 created new session: [3:7587241485494061035:2108]
2025-12-24T02:11:58.813573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241485494061035:2108] [node 5] ICS09 handshake done sender: [3:7587241485494061011:2117] self: [3:7587241485494061009:0] peer: [5:7587241485811749291:0] socket: 655 qp: -1
2025-12-24T02:11:58.813578Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241485494061035:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.813584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.813586Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241485811749192:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.813597Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.813601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241485811749192:2050] [node 3] ICP22 created new session: [5:7587241485811749293:2050]
2025-12-24T02:11:58.813604Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241485811749293:2050] [node 3] ICS09 handshake done sender: [5:7587241485811749284:2121] self: [5:7587241485811749291:0] peer: [3:7587241485494061009:0] socket: 650 qp: -1
2025-12-24T02:11:58.813612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241485811749293:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:58.813619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.823135Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:58.823233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241486354214285:2048] [node 1] ICS01 socket: 619 reason# EndOfStream
2025-12-24T02:11:58.823240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241486354214203:2048] [node 1] ICP30 unregister session Session# [2:7587241486354214285:2048] VirtualId# [2:7587241486354214282:0]
2025-12-24T02:11:58.823247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241486354214285:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.823293Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.823315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.823644Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:58.823729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241485494061029:2048] [node 1] ICS01 socket: 637 reason# EndOfStream
2025-12-24T02:11:58.823746Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241485494060932:2048] [node 1] ICP30 unregister session Session# [3:7587241485494061029:2048] VirtualId# [3:7587241485494061016:0]
2025-12-24T02:11:58.823753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241485494061029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.823817Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.823850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.825070Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:58.825142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241486606370935:2048] [node 1] ICS01 socket: 648 reason# EndOfStream
2025-12-24T02:11:58.825147Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486606370843:2048] [node 1] ICP30 unregister session Session# [4:7587241486606370935:2048] VirtualId# [4:7587241486606370929:0]
2025-12-24T02:11:58.825162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241486606370935:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.825192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.825232Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.825489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:58.825558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485811749279:2048] [node 1] ICS01 socket: 636 reason# EndOfStream
2025-12-24T02:11:58.825563Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485811749190:2048] [node 1] ICP30 unregister session Session# [5:7587241485811749279:2048] VirtualId# [5:7587241485811749272:0]
2025-12-24T02:11:58.825567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485811749279:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.825594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.825612Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.825730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:58.825814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241483622868878:2048] [node 1] ICS07 socket disconnect 679 reason# EndOfStream
2025-12-24T02:11:58.825820Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483622868878:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.825880Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:58.825904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241483622868878:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:58.826222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483622868784:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12047) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.826233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483622868878:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.826238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483622868784:2048] [node 1] ICP30 unregister session Session# [6:7587241483622868878:2048] VirtualId# [6:7587241483622868875:0]
2025-12-24T02:11:58.826278Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:58.826284Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241483622868784:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:12047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.826289Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:12047) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.835393Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:58.835491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241485494061027:2049] [node 2] ICS01 socket: 630 reason# EndOfStream
2025-12-24T02:11:58.835501Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241485494060933:2049] [node 2] ICP30 unregister session Session# [3:7587241485494061027:2049] VirtualId# [3:7587241485494061014:0]
2025-12-24T02:11:58.835507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241485494061027:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.835547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.835568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.837144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241483622868784:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:58.837175Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:58.837235Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241483622868873:2049] [node 2] ICS07 socket disconnect 666 reason# EndOfStream
2025-12-24T02:11:58.837245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483622868873:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.837271Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.837296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241483622868873:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:58.836945Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:58.837019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485811749277:2049] [node 2] ICS01 socket: 661 reason# EndOfStream
2025-12-24T02:11:58.837024Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485811749191:2049] [node 2] ICP30 unregister session Session# [5:7587241485811749277:2049] VirtualId# [5:7587241485811749273:0]
2025-12-24T02:11:58.837029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485811749277:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.837090Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.837113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.837844Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483622868785:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.837856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483622868873:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.837861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483622868785:2049] [node 2] ICP30 unregister session Session# [6:7587241483622868873:2049] VirtualId# [6:7587241483622868865:0]
2025-12-24T02:11:58.837899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.837904Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241483622868785:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.837906Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.838707Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:58.838781Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241486606370925:2049] [node 2] ICS07 socket disconnect 628 reason# EndOfStream
2025-12-24T02:11:58.838785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241486606370925:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.838831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:58.838852Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241486606370925:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:58.839126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241486606370844:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.839141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241486606370925:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.839148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486606370844:2049] [node 2] ICP30 unregister session Session# [4:7587241486606370925:2049] VirtualId# [4:7587241486606370919:0]
2025-12-24T02:11:58.839198Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:58.839205Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241486606370844:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.839207Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:9495) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.847638Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241483622868785:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:58.848649Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241486606370844:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:58.853838Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:58.853948Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241486606370945:2050] [node 3] ICS01 socket: 670 reason# EndOfStream
2025-12-24T02:11:58.853969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486606370845:2050] [node 3] ICP30 unregister session Session# [4:7587241486606370945:2050] VirtualId# [4:7587241486606370933:0]
2025-12-24T02:11:58.853977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241486606370945:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.854051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.854091Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.855059Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:58.855145Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485811749293:2050] [node 3] ICS01 socket: 650 reason# EndOfStream
2025-12-24T02:11:58.855152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485811749192:2050] [node 3] ICP30 unregister session Session# [5:7587241485811749293:2050] VirtualId# [5:7587241485811749291:0]
2025-12-24T02:11:58.855158Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485811749293:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.855213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.855243Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.856044Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:58.856130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483622868868:2050] [node 3] ICS01 socket: 632 reason# EndOfStream
2025-12-24T02:11:58.856136Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483622868786:2050] [node 3] ICP30 unregister session Session# [6:7587241483622868868:2050] VirtualId# [6:7587241483622868866:0]
2025-12-24T02:11:58.856141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483622868868:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.856186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:58.856210Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:58.862812Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:58.862926Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241485811749286:2109] [node 4] ICS01 socket: 675 reason# EndOfStream
2025-12-24T02:11:58.862937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241485811749258:2109] [node 4] ICP30 unregister session Session# [5:7587241485811749286:2109] VirtualId# [5:7587241485811749282:0]
2025-12-24T02:11:58.862944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241485811749286:2109] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.863007Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.863059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:58.863320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:58.863410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483622868882:2110] [node 4] ICS01 socket: 646 reason# EndOfStream
2025-12-24T02:11:58.863424Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483622868856:2110] [node 4] ICP30 unregister session Session# [6:7587241483622868882:2110] VirtualId# [6:7587241483622868876:0]
2025-12-24T02:11:58.863430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483622868882:2110] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.863483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:58.863526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:58.871619Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 5] connection closed by peer
2025-12-24T02:11:58.871720Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241483622868886:2107] [node 5] ICS07 socket disconnect 688 reason# EndOfStream
2025-12-24T02:11:58.871733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241483622868886:2107] [node 5] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:58.871793Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [5] disconnected
2025-12-24T02:11:58.871824Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241483622868886:2107] [node 5] ICS15 start handshake
2025-12-24T02:11:58.872181Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241483622868845:2107] [node 5] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31605) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:58.872198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241483622868886:2107] [node 5] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:58.872206Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241483622868845:2107] [node 5] ICP30 unregister session Session# [6:7587241483622868886:2107] VirtualId# [6:7587241483622868884:0]
2025-12-24T02:11:58.872290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [5] session destroyed
2025-12-24T02:11:58.872302Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241483622868845:2107] [node 5] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:31605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:58.872305Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [5] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:31605) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:58.889076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485560095912:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:58.889137Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485560095913:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.892917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241482909958437:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.892928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241482909958438:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.896678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486530452877:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.896680Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486530452876:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.896692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486530452878:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.891068Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241483166136500:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.891079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241483166136501:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:58.894860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486733693076:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.894868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486733693077:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.894873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486733693078:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.898587Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484757452769:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:58.898602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484757452771:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:58.898637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484757452770:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:58.911268Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484757452831:2106] [node 5] ICP01 ready to work
2025-12-24T02:11:58.911591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485560095973:2107] [node 4] ICP01 ready to work
2025-12-24T02:11:58.912244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485560095975:2108] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.914992Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452935:2106] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.915572Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958500:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.918121Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241485560095987:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:58.918917Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241483166136563:2106] [node 4] ICP01 ready to work
2025-12-24T02:11:58.923527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241483166136571:2110] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.924213Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485560095990:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.924303Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241483166136569:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.925463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241483166136565:2107] [node 5] ICP01 ready to work
2025-12-24T02:11:58.927048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486733693141:2108] [node 6] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.932586Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241483166136579:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44045
2025-12-24T02:11:58.932666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241482909958505:2110] [node 4] ICP01 ready to work
2025-12-24T02:11:58.931082Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452938:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:58.933548Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452835:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.933643Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485560095991:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-12-24T02:11:58.933676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241482909958507:2111] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.934120Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958509:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.934434Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452836:2111] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.936278Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958513:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.941047Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484757452835:2110] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.941536Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484757452770:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.942095Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095990:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.942215Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095991:2115] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.941681Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136573:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.941695Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241483166136573:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.942112Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136579:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.942142Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241483166136500:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.942849Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095982:2111] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.942865Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485560095982:2111] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.941727Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958509:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.941768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241482909958438:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.942876Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136564:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.942893Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241483166136564:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.942998Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241483166136573:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.943085Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241483166136571:2110] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.943100Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.943108Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241483166136571:2110] [node 6] ICP22 created new session: [2:7587241483166136583:2110]
2025-12-24T02:11:58.943115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241483166136583:2110] [node 6] ICS09 handshake done sender: [2:7587241483166136573:2113] self: [2:7587241483166136572:0] peer: [6:7587241484757452837:0] socket: 641 qp: -1
2025-12-24T02:11:58.943123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241483166136583:2110] [node 6] ICS10 traffic start
2025-12-24T02:11:58.943143Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.945018Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958500:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.945048Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241482909958437:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.945676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486530452945:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:58.945824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136569:2111] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.947019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958513:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.947355Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095981:2110] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.947377Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485560095981:2110] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.947207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241482909958515:2115] [node 6] ICP01 ready to work
2025-12-24T02:11:58.950953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241484757452841:2112] [node 4] ICP01 ready to work
2025-12-24T02:11:58.951673Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485560095982:2111] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.952130Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958520:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.951843Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485560095912:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.951871Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.951881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485560095912:2048] [node 2] ICP22 created new session: [1:7587241485560095996:2048]
2025-12-24T02:11:58.951890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485560095996:2048] [node 2] ICS09 handshake done sender: [1:7587241485560095982:2111] self: [1:7587241485560095978:0] peer: [2:7587241483166136581:0] socket: 630 qp: -1
2025-12-24T02:11:58.951898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485560095996:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:58.951907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.952450Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241483166136564:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.952578Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241482909958509:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.952663Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241482909958438:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.952679Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.952685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241482909958438:2049] [node 2] ICP22 created new session: [3:7587241482909958521:2049]
2025-12-24T02:11:58.952691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241482909958521:2049] [node 2] ICS09 handshake done sender: [3:7587241482909958509:2112] self: [3:7587241482909958516:0] peer: [2:7587241483166136560:0] socket: 643 qp: -1
2025-12-24T02:11:58.952697Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241482909958521:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.952704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.953405Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241483166136501:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.953429Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.953436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241483166136501:2049] [node 3] ICP22 created new session: [2:7587241483166136588:2049]
2025-12-24T02:11:58.953442Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241483166136588:2049] [node 3] ICS09 handshake done sender: [2:7587241483166136564:2108] self: [2:7587241483166136560:0] peer: [3:7587241482909958516:0] socket: 639 qp: -1
2025-12-24T02:11:58.953449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241483166136588:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.953458Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.953714Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241483166136500:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [2:7587241483166136579:2115] held: no
2025-12-24T02:11:58.962473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241486733693147:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:58.962409Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241482909958507:2111] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(28580862ns) {Configure:104(28580168ns) {ProcessConfigured:135(28321538ns) {StartInitialHandshake:157(28321097ns) {DropHandshakes:470(384ns) {DropIncomingHandshake:442(92ns) DropOutgoingHandshake:457(46ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(1835ns) {SwitchToState:179(590ns)}}}}}
2025-12-24T02:11:58.963643Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [4:7587241486733693077:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(16770197ns) {Configure:104(16768814ns) {ProcessConfigured:135(16289860ns) {StartInitialHandshake:157(16289031ns) {DropHandshakes:470(892ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(170ns)} GenerateSessionVirtualId:988(851ns) PrepareNewSessionHandshake:477(1278ns) {SwitchToState:179(769ns)}}}}}
2025-12-24T02:11:58.964073Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241486530452950:2113] [node 6] ICP01 ready to work
2025-12-24T02:11:58.965452Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485560095981:2110] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:58.965552Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241486530452877:2049] [node 2] ICP35 event processing took too much time PendingNodeInfo:262(20287081ns) {Configure:104(20286447ns) {ProcessConfigured:135(19929805ns) {StartInitialHandshake:157(19929191ns) {DropHandshakes:470(727ns) {DropIncomingHandshake:442(70ns) DropOutgoingHandshake:457(171ns)} GenerateSessionVirtualId:988(238ns) PrepareNewSessionHandshake:477(909ns) {SwitchToState:179(361ns)}}}}}
2025-12-24T02:11:58.965593Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [5:7587241486530452878:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20158579ns) {Configure:104(20157966ns) {ProcessConfigured:135(19930402ns) {StartInitialHandshake:157(19929890ns) {DropHandshakes:470(510ns) {DropIncomingHandshake:442(55ns) DropOutgoingHandshake:457(33ns)} GenerateSessionVirtualId:988(197ns) PrepareNewSessionHandshake:477(945ns) {SwitchToState:179(408ns)}}}}}
2025-12-24T02:11:58.965593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485560095913:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.965619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:58.965626Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485560095913:2049] [node 3] ICP22 created new session: [1:7587241485560096000:2049]
2025-12-24T02:11:58.965635Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485560096000:2049] [node 3] ICS09 handshake done sender: [1:7587241485560095981:2110] self: [1:7587241485560095977:0] peer: [3:7587241482909958517:0] socket: 627 qp: -1
2025-12-24T02:11:58.965654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485560096000:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:58.965670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:58.966070Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241482909958515:2115] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(18762999ns) {Configure:104(18761987ns) {ProcessConfigured:135(18352065ns) {StartInitialHandshake:157(18351627ns) {DropHandshakes:470(431ns) {DropIncomingHandshake:442(21ns) DropOutgoingHandshake:457(72ns)} GenerateSessionVirtualId:988(160ns) PrepareNewSessionHandshake:477(602ns) {SwitchToState:179(17ns)}}}}}
2025-12-24T02:11:58.966388Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452938:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.966506Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486530452876:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.966940Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958525:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.967266Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241482909958500:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.968207Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095985:2112] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.968228Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485560095985:2112] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.968948Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241484757452771:2050] [node 3] ICP35 event processing took too much time PendingNodeInfo:262(20799338ns) {Configure:104(20798703ns) {ProcessConfigured:135(20411497ns) {StartInitialHandshake:157(20410836ns) {DropHandshakes:470(455ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(80ns)} GenerateSessionVirtualId:988(583ns) PrepareNewSessionHandshake:477(1492ns) {SwitchToState:179(100ns)}}}}}
2025-12-24T02:11:58.969001Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484757452836:2111] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.968963Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485560096004:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-12-24T02:11:58.969039Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485560096005:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-12-24T02:11:58.969464Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485560095985:2112] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.969601Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560096004:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.969661Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485560095975:2108] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.969674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.969685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485560095975:2108] [node 5] ICP22 created new session: [1:7587241485560096007:2108]
2025-12-24T02:11:58.969692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485560096007:2108] [node 5] ICS09 handshake done sender: [1:7587241485560095985:2112] self: [1:7587241485560095979:0] peer: [5:7587241486530452956:0] socket: 633 qp: -1
2025-12-24T02:11:58.969699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485560096007:2108] [node 5] ICS10 traffic start
2025-12-24T02:11:58.969731Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.970027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [1:7587241485560095975:2108] [node 5] ICP18 handshake (actor [1:7587241485560096004:2117]) is too old
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.970356Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241485560096006:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:54082
2025-12-24T02:11:58.970394Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560096005:2118] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.970949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241482909958437:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.970973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.970978Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241482909958437:2048] [node 1] ICP22 created new session: [3:7587241482909958527:2048]
2025-12-24T02:11:58.970985Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241482909958527:2048] [node 1] ICS09 handshake done sender: [3:7587241482909958500:2108] self: [3:7587241482909958517:0] peer: [1:7587241485560095977:0] socket: 629 qp: -1
2025-12-24T02:11:58.970992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241482909958527:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.970998Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.970826Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486733693153:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:58.971538Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693153:2114] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.971603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486733693078:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.971924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693143:2110] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.971931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241486733693143:2110] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.971599Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241484757452841:2112] [node 4] ICP35 event processing took too much time PendingNodeInfo:262(20418508ns) {Configure:104(20417356ns) {ProcessConfigured:135(20001796ns) {StartInitialHandshake:157(20000504ns) {DropHandshakes:470(1080ns) {DropIncomingHandshake:442(143ns) DropOutgoingHandshake:457(161ns)} GenerateSessionVirtualId:988(253ns) PrepareNewSessionHandshake:477(1426ns) {SwitchToState:179(749ns)}}}}}
2025-12-24T02:11:58.971723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484757452841:2112] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.971840Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452847:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:58.972886Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486733693143:2110] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.973248Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241483166136591:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44045
2025-12-24T02:11:58.972987Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241486733693141:2108] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.973001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.973009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486733693141:2108] [node 6] ICP22 created new session: [4:7587241486733693156:2108]
2025-12-24T02:11:58.973015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486733693156:2108] [node 6] ICS09 handshake done sender: [4:7587241486733693143:2110] self: [4:7587241486733693142:0] peer: [6:7587241484757452850:0] socket: 648 qp: -1
2025-12-24T02:11:58.973021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486733693156:2108] [node 6] ICS10 traffic start
2025-12-24T02:11:58.973028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.974260Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241483166136571:2110] [node 6] ICP35 event processing took too much time PendingConnection:279(31043530ns) {HandleHandshakeStatus:329(31042697ns) {DropIncomingHandshake:442(35ns) DropOutgoingHandshake:457(288ns) SwitchToState:179(84ns) ProcessPendingSessionEvents:495(26825383ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 26800404ns max# 26724278ns}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.977882Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452957:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:58.977869Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [6:7587241484757452769:2048] [node 1] ICP35 event processing took too much time PendingNodeInfo:262(17183797ns) {Configure:104(17182899ns) {ProcessConfigured:135(16771093ns) {StartInitialHandshake:157(16770388ns) {DropHandshakes:470(773ns) {DropIncomingHandshake:442(120ns) DropOutgoingHandshake:457(101ns)} GenerateSessionVirtualId:988(677ns) PrepareNewSessionHandshake:477(1765ns) {SwitchToState:179(1084ns)}}}}}
2025-12-24T02:11:58.977969Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958511:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.977988Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241482909958511:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.978972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452935:2106] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.978979Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241486530452876:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486530452938:2108] held: no
2025-12-24T02:11:58.979472Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241483166136592:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44045
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.979679Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241483166136579:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.979754Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241483166136593:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44045
2025-12-24T02:11:58.980025Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241483166136500:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.980043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.980055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241483166136500:2048] [node 1] ICP22 created new session: [2:7587241483166136594:2048]
2025-12-24T02:11:58.980067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241483166136594:2048] [node 1] ICS09 handshake done sender: [2:7587241483166136579:2115] self: [2:7587241483166136581:0] peer: [1:7587241485560095978:0] socket: 636 qp: -1
2025-12-24T02:11:58.980089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241483166136594:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.980261Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452957:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.980317Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486530452877:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.980102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.980439Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486733693161:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:58.980486Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136580:2116] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: 2025-12-24T02:11:58.980496Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136592:2118] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
::ffff:127.1.0.0
2025-12-24T02:11:58.980500Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241483166136580:2116] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.980473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484757452835:2110] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.981122Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452960:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:58.981978Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452848:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:58.982203Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958526:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:58.982354Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486733693160:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:58.982472Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693161:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.982628Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486733693162:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:58.982849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486530452938:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:58.982866Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693162:2118] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.982911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486733693076:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.983020Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560096006:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.982872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241484757452848:2116] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.983135Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095992:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.983111Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958526:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.983149Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485560095992:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.983050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484757452769:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.983204Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484757452770:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.983216Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.984133Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485560095992:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:58.984174Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241483166136580:2116] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:58.984233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485560095987:2109] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.984253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:58.984261Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485560095987:2109] [node 6] ICP22 created new session: [1:7587241485560096012:2109]
2025-12-24T02:11:58.984267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241485560096012:2109] [node 6] ICS09 handshake done sender: [1:7587241485560095992:2116] self: [1:7587241485560095988:0] peer: [6:7587241484757452851:0] socket: 668 qp: -1
2025-12-24T02:11:58.984274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241485560096012:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:58.984285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:58.984358Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693160:2116] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.984409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241486733693077:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.984430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241483166136565:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.983223Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484757452770:2049] [node 2] ICP22 created new session: [6:7587241484757452852:2049]
2025-12-24T02:11:58.984454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:58.983236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241484757452852:2049] [node 2] ICS09 handshake done sender: [6:7587241484757452835:2110] self: [6:7587241484757452837:0] peer: [2:7587241483166136572:0] socket: 642 qp: -1
2025-12-24T02:11:58.983243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241484757452852:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.983252Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.984462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241483166136565:2107] [node 5] ICP22 created new session: [2:7587241483166136598:2107]
2025-12-24T02:11:58.984469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241483166136598:2107] [node 5] ICS09 handshake done sender: [2:7587241483166136580:2116] self: [2:7587241483166136577:0] peer: [5:7587241486530452959:0] socket: 640 qp: -1
2025-12-24T02:11:58.984475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241483166136598:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:58.984482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:58.984747Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241483166136578:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.984759Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241483166136578:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.985511Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241483166136578:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.985660Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241483166136563:2106] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.985675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.985681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241483166136563:2106] [node 4] ICP22 created new session: [2:7587241483166136601:2106]
2025-12-24T02:11:58.985689Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241483166136601:2106] [node 4] ICS09 handshake done sender: [2:7587241483166136578:2114] self: [2:7587241483166136566:0] peer: [4:7587241486733693165:0] socket: 658 qp: -1
2025-12-24T02:11:58.985694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241483166136601:2106] [node 4] ICS10 traffic start
2025-12-24T02:11:58.985703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.985833Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241486733693166:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:58.986179Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486733693160:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.986258Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486733693077:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.986272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.986280Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486733693077:2049] [node 2] ICP22 created new session: [4:7587241486733693167:2049]
2025-12-24T02:11:58.986287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241486733693167:2049] [node 2] ICS09 handshake done sender: [4:7587241486733693160:2116] self: [4:7587241486733693165:0] peer: [2:7587241483166136566:0] socket: 675 qp: -1
2025-12-24T02:11:58.986295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241486733693167:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.986330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.991317Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452961:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.992970Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241482909958511:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:58.993195Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241482909958505:2110] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:58.993220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:58.993228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241482909958505:2110] [node 4] ICP22 created new session: [3:7587241482909958532:2110]
2025-12-24T02:11:58.993234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241482909958532:2110] [node 4] ICS09 handshake done sender: [3:7587241482909958511:2113] self: [3:7587241482909958508:0] peer: [4:7587241486733693154:0] socket: 651 qp: -1
2025-12-24T02:11:58.993243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241482909958532:2110] [node 4] ICS10 traffic start
2025-12-24T02:11:58.993253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:58.994788Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241485560095989:2113] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.994812Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241485560095989:2113] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:58.994974Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484757452769:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241484757452848:2116] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.995341Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452962:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:58.996500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486530452876:2048] [node 1] ICP19 incoming handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.996548Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:58.996578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486530452876:2048] [node 1] ICP22 created new session: [5:7587241486530452963:2048]
2025-12-24T02:11:58.996589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486530452963:2048] [node 1] ICS09 handshake done sender: [5:7587241486530452938:2108] self: [5:7587241486530452956:0] peer: [1:7587241485560095979:0] socket: 635 qp: -1
2025-12-24T02:11:58.996600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486530452963:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:58.996638Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:58.997015Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241486530452877:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486530452957:2117] held: no
2025-12-24T02:11:58.997388Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486530452957:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:58.997391Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452855:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:58.997497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486530452877:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:58.997518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:58.997526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486530452877:2049] [node 2] ICP22 created new session: [5:7587241486530452967:2049]
2025-12-24T02:11:58.997533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241486530452967:2049] [node 2] ICS09 handshake done sender: [5:7587241486530452957:2117] self: [5:7587241486530452959:0] peer: [2:7587241483166136577:0] socket: 665 qp: -1
2025-12-24T02:11:58.997542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241486530452967:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:58.997549Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241486530452965:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:58.997550Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:58.999425Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241486733693078:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241486733693153:2114] held: no
2025-12-24T02:11:58.999622Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452965:2121] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.999676Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486530452945:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:58.999765Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241486733693159:2115] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:58.999776Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241486733693159:2115] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.000869Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486733693159:2115] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.001027Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241486733693147:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.001051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.001059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486733693147:2111] [node 5] ICP22 created new session: [4:7587241491028660467:2111]
2025-12-24T02:11:59.001065Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241491028660467:2111] [node 5] ICS09 handshake done sender: [4:7587241486733693159:2115] self: [4:7587241486733693151:0] peer: [5:7587241486530452969:0] socket: 679 qp: -1
2025-12-24T02:11:59.001071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241491028660467:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:59.001079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:59.001799Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241485560095989:2113] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.001953Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241485560095973:2107] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.001996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.002009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241485560095973:2107] [node 4] ICP22 created new session: [1:7587241489855063311:2107]
2025-12-24T02:11:59.002017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241489855063311:2107] [node 4] ICS09 handshake done sender: [1:7587241485560095989:2113] self: [1:7587241485560095980:0] peer: [4:7587241486733693164:0] socket: 645 qp: -1
2025-12-24T02:11:59.002031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241489855063311:2107] [node 4] ICS10 traffic start
2025-12-24T02:11:59.002046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.002156Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484757452848:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.002198Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241482909958535:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:44823
2025-12-24T02:11:59.003204Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452962:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.003254Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241486530452878:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.006776Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241484757452854:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:59.007927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241483166136563:2106] [node 4] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:59.008172Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484757452769:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.008197Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.008207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484757452769:2048] [node 1] ICP22 created new session: [6:7587241489052420154:2048]
2025-12-24T02:11:59.008215Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241489052420154:2048] [node 1] ICS09 handshake done sender: [6:7587241484757452848:2116] self: [6:7587241484757452851:0] peer: [1:7587241485560095988:0] socket: 676 qp: -1
2025-12-24T02:11:59.008223Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241489052420154:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.008253Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.009131Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241491028660469:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:59.009209Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241484757452836:2111] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.009362Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958523:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.009402Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241482909958523:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.009483Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958535:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.010257Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241486530452945:2111] [node 4] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17755) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486530452965:2121] held: no
2025-12-24T02:11:59.010664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484757452841:2112] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.010686Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.010773Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484757452841:2112] [node 4] ICP22 created new session: [6:7587241489052420156:2112]
2025-12-24T02:11:59.010786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241489052420156:2112] [node 4] ICS09 handshake done sender: [6:7587241484757452836:2111] self: [6:7587241484757452850:0] peer: [4:7587241486733693142:0] socket: 649 qp: -1
2025-12-24T02:11:59.010793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241489052420156:2112] [node 4] ICS10 traffic start
2025-12-24T02:11:59.010799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.012073Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241489052420153:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.012342Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486733693153:2114] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.013519Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241482909958523:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.013901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241482909958507:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.013930Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.013938Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241482909958507:2111] [node 5] ICP22 created new session: [3:7587241487204925834:2111]
2025-12-24T02:11:59.013945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241487204925834:2111] [node 5] ICS09 handshake done sender: [3:7587241482909958523:2117] self: [3:7587241482909958510:0] peer: [5:7587241490825420266:0] socket: 637 qp: -1
2025-12-24T02:11:59.013955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241487204925834:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:59.013963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:59.014084Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241489052420153:2120] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.014226Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484757452771:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.014441Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241490825420268:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:59.014542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486733693078:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.014560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.014566Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486733693078:2050] [node 3] ICP22 created new session: [4:7587241491028660472:2050]
2025-12-24T02:11:59.014573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241491028660472:2050] [node 3] ICS09 handshake done sender: [4:7587241486733693153:2114] self: [4:7587241486733693154:0] peer: [3:7587241482909958508:0] socket: 653 qp: -1
2025-12-24T02:11:59.014579Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241491028660472:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.014585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.015028Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241491028660471:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:59.015201Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486530452965:2121] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.015309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486530452945:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.015320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.015329Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486530452945:2111] [node 4] ICP22 created new session: [5:7587241490825420269:2111]
2025-12-24T02:11:59.015336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241490825420269:2111] [node 4] ICS09 handshake done sender: [5:7587241486530452965:2121] self: [5:7587241486530452969:0] peer: [4:7587241486733693151:0] socket: 650 qp: -1
2025-12-24T02:11:59.015342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241490825420269:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:59.015378Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.015246Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241486733693076:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [4:7587241486733693162:2118] held: no
2025-12-24T02:11:59.015254Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241491028660470:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:23365
2025-12-24T02:11:59.015445Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241486733693162:2118] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.015564Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241482909958524:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.015577Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241482909958524:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.015637Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241486733693141:2108] [node 6] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:59.015651Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241486733693076:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.015659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.015665Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241486733693076:2048] [node 1] ICP22 created new session: [4:7587241491028660474:2048]
2025-12-24T02:11:59.015671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241491028660474:2048] [node 1] ICS09 handshake done sender: [4:7587241486733693162:2118] self: [4:7587241486733693164:0] peer: [1:7587241485560095980:0] socket: 655 qp: -1
2025-12-24T02:11:59.015677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241491028660474:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.015685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.016844Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241482909958524:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.016994Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241489052420159:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:59.017076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241482909958515:2115] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.017095Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.017101Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241482909958515:2115] [node 6] ICP22 created new session: [3:7587241487204925837:2115]
2025-12-24T02:11:59.017112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241487204925837:2115] [node 6] ICS09 handshake done sender: [3:7587241482909958524:2118] self: [3:7587241482909958519:0] peer: [6:7587241489052420158:0] socket: 689 qp: -1
2025-12-24T02:11:59.017118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241487204925837:2115] [node 6] ICS10 traffic start
2025-12-24T02:11:59.017125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.018264Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241489052420159:2121] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.018314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241484757452831:2106] [node 5] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.018171Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241486530452878:2050] [node 3] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6063) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [5:7587241486530452962:2120] held: no
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.018525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241489052420161:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:59.018620Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241489052420153:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.018605Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241486530452955:2116] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.018613Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [5:7587241486530452955:2116] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.018669Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484757452771:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.018703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.018711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484757452771:2050] [node 3] ICP22 created new session: [6:7587241489052420162:2050]
2025-12-24T02:11:59.018717Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241489052420162:2050] [node 3] ICS09 handshake done sender: [6:7587241489052420153:2120] self: [6:7587241489052420158:0] peer: [3:7587241482909958519:0] socket: 646 qp: -1
2025-12-24T02:11:59.018722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241489052420162:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.018730Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.019572Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486530452955:2116] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.019672Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [5:7587241486530452950:2113] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.019691Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.019696Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486530452950:2113] [node 6] ICP22 created new session: [5:7587241490825420273:2113]
2025-12-24T02:11:59.019701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241490825420273:2113] [node 6] ICS09 handshake done sender: [5:7587241486530452955:2116] self: [5:7587241486530452954:0] peer: [6:7587241489052420160:0] socket: 638 qp: -1
2025-12-24T02:11:59.019706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241490825420273:2113] [node 6] ICS10 traffic start
2025-12-24T02:11:59.019722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.020513Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241490825420272:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:48637
2025-12-24T02:11:59.021011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241486530452962:2120] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.021094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241486530452878:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.021108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.021115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241486530452878:2050] [node 3] ICP22 created new session: [5:7587241490825420275:2050]
2025-12-24T02:11:59.021122Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241490825420275:2050] [node 3] ICS09 handshake done sender: [5:7587241486530452962:2120] self: [5:7587241490825420266:0] peer: [3:7587241482909958510:0] socket: 673 qp: -1
2025-12-24T02:11:59.021128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241490825420275:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.021134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.021623Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241489052420164:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:45681
2025-12-24T02:11:59.021892Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484757452831:2106] [node 5] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64957) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241489052420159:2121] held: no
2025-12-24T02:11:59.022043Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241489052420159:2121] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.022213Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241484757452831:2106] [node 5] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.022227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.022233Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241484757452831:2106] [node 5] ICP22 created new session: [6:7587241489052420165:2106]
2025-12-24T02:11:59.022238Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241489052420165:2106] [node 5] ICS09 handshake done sender: [6:7587241489052420159:2121] self: [6:7587241489052420160:0] peer: [5:7587241486530452954:0] socket: 628 qp: -1
2025-12-24T02:11:59.022245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241489052420165:2106] [node 5] ICS10 traffic start
2025-12-24T02:11:59.022250Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:59.038790Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:59.038842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:59.038946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241482909958527:2048] [node 1] ICS07 socket disconnect 629 reason# EndOfStream
2025-12-24T02:11:59.038953Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241482909958527:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.039035Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.039074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241482909958527:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.039501Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241482909958437:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.039511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241482909958527:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.039520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241482909958437:2048] [node 1] ICP30 unregister session Session# [3:7587241482909958527:2048] VirtualId# [3:7587241482909958517:0]
2025-12-24T02:11:59.039574Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.039582Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241482909958437:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.039584Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:17107) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.040456Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:59.040544Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241491028660474:2048] [node 1] ICS01 socket: 655 reason# EndOfStream
2025-12-24T02:11:59.040556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486733693076:2048] [node 1] ICP30 unregister session Session# [4:7587241491028660474:2048] VirtualId# [4:7587241486733693164:0]
2025-12-24T02:11:59.040562Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241491028660474:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.040614Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.040656Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.040902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:59.040983Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241489052420154:2048] [node 1] ICS01 socket: 676 reason# EndOfStream
2025-12-24T02:11:59.040993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241484757452769:2048] [node 1] ICP30 unregister session Session# [6:7587241489052420154:2048] VirtualId# [6:7587241484757452851:0]
2025-12-24T02:11:59.040998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241489052420154:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.041046Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.041082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.041776Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241483166136594:2048] [node 1] ICS01 socket: 636 reason# EndOfStream
2025-12-24T02:11:59.041795Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241483166136500:2048] [node 1] ICP30 unregister session Session# [2:7587241483166136594:2048] VirtualId# [2:7587241483166136581:0]
2025-12-24T02:11:59.041801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241483166136594:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.041861Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.041893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.042337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:59.042430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486530452963:2048] [node 1] ICS01 socket: 635 reason# EndOfStream
2025-12-24T02:11:59.042440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486530452876:2048] [node 1] ICP30 unregister session Session# [5:7587241486530452963:2048] VirtualId# [5:7587241486530452956:0]
2025-12-24T02:11:59.042445Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486530452963:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.042496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.042525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.049346Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241482909958437:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:59.051351Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:59.051521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241482909958521:2049] [node 2] ICS07 socket disconnect 643 reason# EndOfStream
2025-12-24T02:11:59.051533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241482909958521:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.051592Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.051620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241482909958521:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:59.053784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:59.053785Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241482909958438:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Socket error# Connection reset by peer state# SendInitialPacket processed# 0 remain# 52 incoming: [0:0:0] held: no
2025-12-24T02:11:59.053802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241482909958521:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:59.053863Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241486530452967:2049] [node 2] ICS01 socket: 665 reason# EndOfStream
2025-12-24T02:11:59.053875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486530452877:2049] [node 2] ICP30 unregister session Session# [5:7587241486530452967:2049] VirtualId# [5:7587241486530452959:0]
2025-12-24T02:11:59.053882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241486530452967:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.053931Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.053509Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:59.053610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241486733693167:2049] [node 2] ICS07 socket disconnect 675 reason# EndOfStream
2025-12-24T02:11:59.053618Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241486733693167:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.053958Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.053676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.053708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241486733693167:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:59.054183Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241486733693077:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.054195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241486733693167:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.054201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486733693077:2049] [node 2] ICP30 unregister session Session# [4:7587241486733693167:2049] VirtualId# [4:7587241486733693165:0]
2025-12-24T02:11:59.054255Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.054263Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241486733693077:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.054266Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.055083Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241482909958438:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.055101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241482909958521:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.055107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241482909958438:2049] [node 2] ICP30 unregister session Session# [3:7587241482909958521:2049] VirtualId# [3:7587241482909958516:0]
2025-12-24T02:11:59.054856Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:59.055161Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.055174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241482909958438:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.054956Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241484757452852:2049] [node 2] ICS07 socket disconnect 642 reason# EndOfStream
2025-12-24T02:11:59.054964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241484757452852:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.055177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.055024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.055047Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241484757452852:2049] [node 2] ICS15 start handshake
2025-12-24T02:11:59.056438Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484757452770:2049] [node 2] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.056456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241484757452852:2049] [node 2] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.056461Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241484757452770:2049] [node 2] ICP30 unregister session Session# [6:7587241484757452852:2049] VirtualId# [6:7587241484757452837:0]
2025-12-24T02:11:59.056517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.056529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241484757452770:2049] [node 2] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.056531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [2] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:3500) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.063700Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241486733693077:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:59.066651Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 3] connection closed by peer
2025-12-24T02:11:59.066782Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241484757452770:2049] [node 2] ICP33 wake up from error state
2025-12-24T02:11:59.067041Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:59.066805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241491028660472:2050] [node 3] ICS01 socket: 653 reason# EndOfStream
2025-12-24T02:11:59.066837Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241486733693078:2050] [node 3] ICP30 unregister session Session# [4:7587241491028660472:2050] VirtualId# [4:7587241486733693154:0]
2025-12-24T02:11:59.067132Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241489052420162:2050] [node 3] ICS07 socket disconnect 646 reason# EndOfStream
2025-12-24T02:11:59.066864Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:59.067149Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241489052420162:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.066960Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241490825420275:2050] [node 3] ICS01 socket: 673 reason# EndOfStream
2025-12-24T02:11:59.066844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241491028660472:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.067158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:59.067194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:59.066967Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486530452878:2050] [node 3] ICP30 unregister session Session# [5:7587241490825420275:2050] VirtualId# [5:7587241490825420266:0]
2025-12-24T02:11:59.066975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241490825420275:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.067045Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:59.067074Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:59.067227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:59.067269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241489052420162:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:59.067662Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241484757452771:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6063) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.067683Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241489052420162:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.067692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241484757452771:2050] [node 3] ICP30 unregister session Session# [6:7587241489052420162:2050] VirtualId# [6:7587241489052420158:0]
2025-12-24T02:11:59.067767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:59.067792Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241484757452771:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:6063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.067796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:6063) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.078033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241484757452771:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:59.083958Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:59.084071Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241490825420269:2111] [node 4] ICS01 socket: 650 reason# EndOfStream
2025-12-24T02:11:59.084084Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241486530452945:2111] [node 4] ICP30 unregister session Session# [5:7587241490825420269:2111] VirtualId# [5:7587241486530452969:0]
2025-12-24T02:11:59.084092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241490825420269:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.084092Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:59.084195Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241489052420156:2112] [node 4] ICS01 socket: 649 reason# EndOfStream
2025-12-24T02:11:59.084158Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:59.084190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:59.084210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241484757452841:2112] [node 4] ICP30 unregister session Session# [6:7587241489052420156:2112] VirtualId# [6:7587241484757452850:0]
2025-12-24T02:11:59.084216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241489052420156:2112] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.084294Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:59.084330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:11:59.117191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241491325533669:2048] [node 2] ICP01 ready to work
2025-12-24T02:11:59.117207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241491325533670:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:59.119713Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241490433832763:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:59.119726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241490433832764:2049] [node 3] ICP01 ready to work
2025-12-24T02:11:59.121901Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241490043514461:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:59.121911Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241490043514462:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:59.124184Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241487898099471:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:59.124196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241487898099472:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:59.124202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241487898099473:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:59.126480Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241489701736530:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:59.126495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241489701736532:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:59.126535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241489701736531:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:59.129430Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241487174816296:2048] [node 1] ICP01 ready to work
2025-12-24T02:11:59.129446Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241487174816298:2050] [node 3] ICP01 ready to work
2025-12-24T02:11:59.129491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241487174816297:2049] [node 2] ICP01 ready to work
2025-12-24T02:11:59.142307Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241487174816358:2107] [node 5] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.143525Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736591:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.143318Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832823:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.144319Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514521:2107] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.146263Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514521:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.146769Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241490043514461:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.147065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241491325533736:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:59.147119Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241491325533733:2108] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.147136Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241491325533733:2108] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.146905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241489701736591:2108] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.147100Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241489701736593:2109] [node 6] ICP01 ready to work
2025-12-24T02:11:59.147609Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241491325533730:2107] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.147623Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241491325533730:2107] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.146869Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832823:2107] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.147399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241490433832763:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.147778Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241489701736593:2109] [node 6] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.147896Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241487174816360:2109] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.147831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241491325533733:2108] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.147911Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [6:7587241487174816360:2109] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.148322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241491325533670:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.148342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.148350Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241491325533670:2049] [node 3] ICP22 created new session: [1:7587241491325533741:2049]
2025-12-24T02:11:59.148357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241491325533741:2049] [node 3] ICS09 handshake done sender: [1:7587241491325533733:2108] self: [1:7587241491325533732:0] peer: [3:7587241490043514522:0] socket: 636 qp: -1
2025-12-24T02:11:59.148365Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241491325533741:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:59.148386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.148913Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241491325533730:2107] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:59.148932Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832825:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
2025-12-24T02:11:59.149092Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241491325533669:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.149107Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:59.149112Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241491325533669:2048] [node 2] ICP22 created new session: [1:7587241491325533744:2048]
2025-12-24T02:11:59.149118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241491325533744:2048] [node 2] ICS09 handshake done sender: [1:7587241491325533730:2107] self: [1:7587241491325533729:0] peer: [2:7587241490433832824:0] socket: 630 qp: -1
2025-12-24T02:11:59.149124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241491325533744:2048] [node 2] ICS10 traffic start
2025-12-24T02:11:59.149062Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241490433832823:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.149130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.149332Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514523:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.150240Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241490043514521:2107] [node 1] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.150926Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241487174816360:2109] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.150870Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099532:2108] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.150949Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241491325533747:2111] [node 5] ICP01 ready to work
2025-12-24T02:11:59.151801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241490433832763:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.151841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.151858Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241490433832763:2048] [node 1] ICP22 created new session: [2:7587241490433832826:2048]
2025-12-24T02:11:59.151958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241490043514461:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.151870Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241490433832826:2048] [node 1] ICS09 handshake done sender: [2:7587241490433832823:2107] self: [2:7587241490433832824:0] peer: [1:7587241491325533729:0] socket: 632 qp: -1
2025-12-24T02:11:59.151996Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.152007Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241490043514461:2048] [node 1] ICP22 created new session: [3:7587241490043514524:2048]
2025-12-24T02:11:59.152014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241490043514524:2048] [node 1] ICS09 handshake done sender: [3:7587241490043514521:2107] self: [3:7587241490043514522:0] peer: [1:7587241491325533732:0] socket: 637 qp: -1
2025-12-24T02:11:59.152027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241490043514524:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.151884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241490433832826:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.151908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.152056Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.152275Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241491325533749:2112] [node 6] ICP01 ready to work
2025-12-24T02:11:59.152295Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099532:2108] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.152686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241487898099471:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.152827Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241491325533738:2110] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.152877Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241491325533738:2110] [node 4] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.153529Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736595:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.153765Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241489701736591:2108] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.153778Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241491325533738:2110] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.153906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241489701736593:2109] [node 6] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.153919Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.153926Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241489701736593:2109] [node 6] ICP22 created new session: [5:7587241489701736596:2109]
2025-12-24T02:11:59.153934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241489701736596:2109] [node 6] ICS09 handshake done sender: [5:7587241489701736591:2108] self: [5:7587241489701736594:0] peer: [6:7587241487174816359:0] socket: 635 qp: -1
2025-12-24T02:11:59.153940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241489701736596:2109] [node 6] ICS10 traffic start
2025-12-24T02:11:59.153963Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.154089Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099534:2109] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.154207Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241487898099532:2108] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.154631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241490043514528:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:59.156443Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [6:7587241487174816358:2107] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.156466Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.156474Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241487174816358:2107] [node 5] ICP22 created new session: [6:7587241487174816363:2107]
2025-12-24T02:11:59.156481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241487174816363:2107] [node 5] ICS09 handshake done sender: [6:7587241487174816360:2109] self: [6:7587241487174816359:0] peer: [5:7587241489701736594:0] socket: 631 qp: -1
2025-12-24T02:11:59.156489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241487174816363:2107] [node 5] ICS10 traffic start
2025-12-24T02:11:59.156515Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.157379Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514534:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.160295Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241490433832833:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:59.160302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241487898099471:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.160343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.160361Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241487898099471:2048] [node 1] ICP22 created new session: [4:7587241487898099536:2048]
2025-12-24T02:11:59.160294Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241490433832832:2109] [node 4] ICP01 ready to work
2025-12-24T02:11:59.160308Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241490433832834:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:59.160371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241487898099536:2048] [node 1] ICS09 handshake done sender: [4:7587241487898099532:2108] self: [4:7587241487898099533:0] peer: [1:7587241491325533737:0] socket: 640 qp: -1
2025-12-24T02:11:59.160381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241487898099536:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.160415Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.161282Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241490043514532:2110] [node 5] ICP01 ready to work
2025-12-24T02:11:59.161730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [3:7587241490043514533:2111] [node 6] ICP01 ready to work
2025-12-24T02:11:59.160327Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241491325533736:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.160363Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.160377Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241491325533736:2109] [node 4] ICP22 created new session: [1:7587241491325533755:2109]
2025-12-24T02:11:59.160387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241491325533755:2109] [node 4] ICS09 handshake done sender: [1:7587241491325533738:2110] self: [1:7587241491325533737:0] peer: [4:7587241487898099533:0] socket: 639 qp: -1
2025-12-24T02:11:59.160395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241491325533755:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:59.160426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.162647Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816365:2110] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.162661Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514534:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.162789Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832840:2113] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
2025-12-24T02:11:59.162732Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514541:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.162762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [3:7587241490043514462:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.162935Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832835:2112] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.162948Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241490433832835:2112] [node 3] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.163202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241487898099544:2112] [node 5] ICP01 ready to work
No RDMA context found for GID: 2025-12-24T02:11:59.165172Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241490433832835:2112] [node 3] ICH04 handshake succeeded
::ffff:127.1.0.0
2025-12-24T02:11:59.165241Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514543:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.165429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241490043514534:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:59.165503Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [3:7587241490043514462:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.165516Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:59.165524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241490043514462:2049] [node 2] ICP22 created new session: [3:7587241490043514544:2049]
2025-12-24T02:11:59.165530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241490043514544:2049] [node 2] ICS09 handshake done sender: [3:7587241490043514534:2112] self: [3:7587241490043514542:0] peer: [2:7587241490433832829:0] socket: 652 qp: -1
2025-12-24T02:11:59.165537Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241490043514544:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:59.165545Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:59.165559Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [4:7587241487898099549:2114] [node 6] ICP01 ready to work
2025-12-24T02:11:59.166203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241490433832764:2049] [node 3] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.166223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.166230Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241490433832764:2049] [node 3] ICP22 created new session: [2:7587241490433832847:2049]
2025-12-24T02:11:59.166237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241490433832847:2049] [node 3] ICS09 handshake done sender: [2:7587241490433832835:2112] self: [2:7587241490433832829:0] peer: [3:7587241490043514542:0] socket: 650 qp: -1
2025-12-24T02:11:59.166245Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241490433832847:2049] [node 3] ICS10 traffic start
2025-12-24T02:11:59.166276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:59.165588Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241487174816365:2110] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.166104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241487174816296:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.166393Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241491325533752:2113] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.166406Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241491325533752:2113] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.167119Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241491325533752:2113] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.167781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241491325533749:2112] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.167840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.167853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241491325533749:2112] [node 6] ICP22 created new session: [1:7587241491325533760:2112]
2025-12-24T02:11:59.167861Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241491325533760:2112] [node 6] ICS09 handshake done sender: [1:7587241491325533752:2113] self: [1:7587241491325533751:0] peer: [6:7587241487174816366:0] socket: 646 qp: -1
2025-12-24T02:11:59.167872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241491325533760:2112] [node 6] ICS10 traffic start
2025-12-24T02:11:59.167895Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.169228Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832841:2114] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.177160Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [1:7587241491325533762:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:7271
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.178124Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099552:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.178153Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099553:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.178583Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241490043514533:2111] [node 6] ICP35 event processing took too much time PendingNodeInfo:262(16717646ns) {Configure:104(16717234ns) {ProcessConfigured:135(16461180ns) {StartInitialHandshake:157(16460612ns) {DropHandshakes:470(339ns) {DropIncomingHandshake:442(28ns) DropOutgoingHandshake:457(16ns)} GenerateSessionVirtualId:988(417ns) PrepareNewSessionHandshake:477(1331ns) {SwitchToState:179(548ns)}}}}}
2025-12-24T02:11:59.178909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099553:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.179117Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514546:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.183215Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832851:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
2025-12-24T02:11:59.184706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832851:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.186949Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:262: Proxy [3:7587241490043514532:2110] [node 5] ICP35 event processing took too much time PendingNodeInfo:262(25511314ns) {Configure:104(25510632ns) {ProcessConfigured:135(25184374ns) {StartInitialHandshake:157(25183777ns) {DropHandshakes:470(404ns) {DropIncomingHandshake:442(67ns) DropOutgoingHandshake:457(63ns)} GenerateSessionVirtualId:988(502ns) PrepareNewSessionHandshake:477(1146ns) {SwitchToState:179(551ns)}}}}}
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.187030Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736610:2116] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.187056Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736609:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.187065Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736611:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.187232Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736612:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.187340Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241489701736609:2115] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.187357Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241489701736611:2117] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.187369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241489701736530:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.187395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241489701736531:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.187520Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [5:7587241489701736601:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:59.190757Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832843:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.190782Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241490433832843:2116] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.187531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [6:7587241487174816369:2111] [node 4] ICP01 ready to work
2025-12-24T02:11:59.190728Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816371:2112] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.191570Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099555:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.192877Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241487898099472:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.192888Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832849:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.192905Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241491325533757:2114] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.192918Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241490433832849:2117] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.192931Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241491325533757:2114] [node 5] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.194016Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241490433832854:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:32831
2025-12-24T02:11:59.194355Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [3:7587241490043514551:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:19452
2025-12-24T02:11:59.193009Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241487174816371:2112] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.193050Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241487174816297:2049] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.194274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816377:2115] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.194878Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241490433832843:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.194944Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241490433832849:2117] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.194981Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241490433832842:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.194990Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [2:7587241490433832842:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.194951Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241491325533757:2114] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.195055Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241491325533747:2111] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.195075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.195091Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241491325533747:2111] [node 5] ICP22 created new session: [1:7587241491325533764:2111]
2025-12-24T02:11:59.195099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241491325533764:2111] [node 5] ICS09 handshake done sender: [1:7587241491325533757:2114] self: [1:7587241491325533750:0] peer: [5:7587241489701736613:0] socket: 656 qp: -1
2025-12-24T02:11:59.195111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241491325533764:2111] [node 5] ICS10 traffic start
2025-12-24T02:11:59.195130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.195841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241490433832832:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.195846Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241490433832842:2115] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.195862Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.195870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241490433832832:2109] [node 4] ICP22 created new session: [2:7587241490433832857:2109]
2025-12-24T02:11:59.195878Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241490433832857:2109] [node 4] ICS09 handshake done sender: [2:7587241490433832843:2116] self: [2:7587241490433832837:0] peer: [4:7587241487898099554:0] socket: 665 qp: -1
2025-12-24T02:11:59.195892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241490433832857:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:59.195909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.195929Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241490433832833:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.195953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.195969Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241490433832833:2111] [node 6] ICP22 created new session: [2:7587241490433832859:2111]
2025-12-24T02:11:59.195977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241490433832859:2111] [node 6] ICS09 handshake done sender: [2:7587241490433832842:2115] self: [2:7587241490433832838:0] peer: [6:7587241487174816378:0] socket: 664 qp: -1
2025-12-24T02:11:59.195999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241490433832859:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:59.196012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.196399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241490433832834:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.196414Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.196420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241490433832834:2110] [node 5] ICP22 created new session: [2:7587241490433832861:2110]
2025-12-24T02:11:59.196425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241490433832861:2110] [node 5] ICS09 handshake done sender: [2:7587241490433832849:2117] self: [2:7587241490433832839:0] peer: [5:7587241489701736614:0] socket: 655 qp: -1
2025-12-24T02:11:59.196431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241490433832861:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:59.196438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:59.197592Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514551:2120] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
No RDMA context found for GID: ::ffff:127.1.0.0
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.198890Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099556:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.199020Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736617:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.199149Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099556:2119] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.199150Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241487898099553:2117] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:59.199208Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241487898099472:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.199221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:59.199228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241487898099472:2049] [node 2] ICP22 created new session: [4:7587241487898099558:2049]
2025-12-24T02:11:59.199240Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241487898099558:2049] [node 2] ICS09 handshake done sender: [4:7587241487898099553:2117] self: [4:7587241487898099554:0] peer: [2:7587241490433832837:0] socket: 675 qp: -1
2025-12-24T02:11:59.199247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241487898099558:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:59.199256Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:59.197543Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241487174816297:2049] [node 2] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:16256) Socket error# connection unexpectedly closed state# ReceiveExReplyHeader processed# 0 remain# 8 incoming: [6:7587241487174816371:2112] held: no
2025-12-24T02:11:59.197618Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241487174816365:2110] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.198411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241487174816296:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.198421Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.198429Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241487174816296:2048] [node 1] ICP22 created new session: [6:7587241487174816382:2048]
2025-12-24T02:11:59.198436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241487174816382:2048] [node 1] ICS09 handshake done sender: [6:7587241487174816365:2110] self: [6:7587241487174816366:0] peer: [1:7587241491325533751:0] socket: 647 qp: -1
2025-12-24T02:11:59.198443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241487174816382:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.198450Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.199274Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816381:2117] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.202836Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241489701736611:2117] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.206841Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241489701736531:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.206866Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:59.206873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241489701736531:2049] [node 2] ICP22 created new session: [5:7587241489701736619:2049]
2025-12-24T02:11:59.206882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241489701736619:2049] [node 2] ICS09 handshake done sender: [5:7587241489701736611:2117] self: [5:7587241489701736614:0] peer: [2:7587241490433832839:0] socket: 684 qp: -1
2025-12-24T02:11:59.206889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241489701736619:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:59.206897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:59.207490Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736618:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.211035Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241489701736609:2115] [node 1] ICH04 handshake succeeded
2025-12-24T02:11:59.211500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241489701736530:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.211522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:11:59.211534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241489701736530:2048] [node 1] ICP22 created new session: [5:7587241489701736622:2048]
2025-12-24T02:11:59.211540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241489701736622:2048] [node 1] ICS09 handshake done sender: [5:7587241489701736609:2115] self: [5:7587241489701736613:0] peer: [1:7587241491325533750:0] socket: 659 qp: -1
2025-12-24T02:11:59.211549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241489701736622:2048] [node 1] ICS10 traffic start
2025-12-24T02:11:59.211555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:11:59.212319Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514546:2117] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.212359Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099552:2116] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.212395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [4:7587241487898099473:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.214526Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514537:2114] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.214538Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241490043514537:2114] [node 4] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.216104Z :INTERCONNECT ERROR: interconnect_tcp_proxy.h:279: Proxy [2:7587241490433832832:2109] [node 4] ICP35 event processing took too much time PendingConnection:279(20128092ns) {HandleHandshakeStatus:329(20127032ns) {DropIncomingHandshake:442(447ns) DropOutgoingHandshake:457(289ns) SwitchToState:179(257ns) ProcessPendingSessionEvents:495(16465782ns) {unordered ForwardSessionEventToSession:612 num# 100 sum# 16438809ns max# 16359827ns}}}
2025-12-24T02:11:59.218055Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241490433832834:2110] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:59.215841Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816384:2118] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.216140Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241487174816381:2117] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.216192Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241487174816371:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:11:59.216291Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241487174816297:2049] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.216308Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:11:59.216316Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241487174816297:2049] [node 2] ICP22 created new session: [6:7587241487174816385:2049]
2025-12-24T02:11:59.216345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241487174816385:2049] [node 2] ICS09 handshake done sender: [6:7587241487174816371:2112] self: [6:7587241487174816378:0] peer: [2:7587241490433832838:0] socket: 666 qp: -1
2025-12-24T02:11:59.216354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241487174816385:2049] [node 2] ICS10 traffic start
2025-12-24T02:11:59.216367Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:11:59.216532Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241487174816298:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.218897Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241489701736610:2116] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.218944Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241489701736601:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.219101Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099546:2113] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.219117Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241487898099546:2113] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.219561Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241490043514537:2114] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.219581Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514547:2118] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.219587Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241490043514547:2118] [node 6] ICRDMA Non success qp response from remote side
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.219579Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [5:7587241489701736612:2118] [node 3] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.219608Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [5:7587241489701736532:2050] [node 3] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:11:59.219958Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241490043514528:2109] [node 4] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.219973Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.219982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241490043514528:2109] [node 4] ICP22 created new session: [3:7587241490043514555:2109]
2025-12-24T02:11:59.219999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241490043514555:2109] [node 4] ICS09 handshake done sender: [3:7587241490043514537:2114] self: [3:7587241490043514530:0] peer: [4:7587241487898099561:0] socket: 657 qp: -1
2025-12-24T02:11:59.220006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241490043514555:2109] [node 4] ICS10 traffic start
2025-12-24T02:11:59.220031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.221067Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514549:2119] [node 5] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.221079Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [3:7587241490043514549:2119] [node 5] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.222429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241487898099546:2113] [node 5] ICH04 handshake succeeded
2025-12-24T02:11:59.222554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241487898099544:2112] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.222580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
2025-12-24T02:11:59.222591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241487898099544:2112] [node 5] ICP22 created new session: [4:7587241487898099563:2112]
2025-12-24T02:11:59.222608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241487898099563:2112] [node 5] ICS09 handshake done sender: [4:7587241487898099546:2113] self: [4:7587241487898099545:0] peer: [5:7587241489701736624:0] socket: 668 qp: -1
2025-12-24T02:11:59.222617Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241487898099563:2112] [node 5] ICS10 traffic start
2025-12-24T02:11:59.222637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
2025-12-24T02:11:59.221381Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816388:2119] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.221744Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [6:7587241487174816388:2119] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.221818Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [6:7587241487174816369:2111] [node 4] ICP08 No active sessions, becoming PendingConnection
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.223191Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241490043514547:2118] [node 6] ICH04 handshake succeeded
2025-12-24T02:11:59.223223Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [4:7587241487898099551:2115] [node 6] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.223241Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [4:7587241487898099551:2115] [node 6] ICRDMA Non success qp response from remote side
2025-12-24T02:11:59.223311Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241490043514533:2111] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.223329Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.223335Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241490043514533:2111] [node 6] ICP22 created new session: [3:7587241490043514558:2111]
2025-12-24T02:11:59.223340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241490043514558:2111] [node 6] ICS09 handshake done sender: [3:7587241490043514547:2118] self: [3:7587241490043514540:0] peer: [6:7587241487174816387:0] socket: 677 qp: -1
2025-12-24T02:11:59.223345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241490043514558:2111] [node 6] ICS10 traffic start
2025-12-24T02:11:59.223352Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.224019Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [3:7587241490043514541:2115] [node 4] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:11:59.224539Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:272: Proxy [3:7587241490043514528:2109] [node 4] ICP18 handshake (actor [3:7587241490043514541:2115]) is too old
2025-12-24T02:11:59.224758Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [3:7587241490043514549:2119] [node 5] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.225066Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [1:7587241491325533747:2111] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:59.225037Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736626:2122] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.224873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [3:7587241490043514532:2110] [node 5] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.224891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [5] session created
No RDMA context found for GID: ::ffff:127.1.0.02025-12-24T02:11:59.224898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [3:7587241490043514532:2110] [node 5] ICP22 created new session: [3:7587241490043514561:2110]
2025-12-24T02:11:59.224904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [3:7587241490043514561:2110] [node 5] ICS09 handshake done sender: [3:7587241490043514549:2119] self: [3:7587241490043514539:0] peer: [5:7587241489701736625:0] socket: 685 qp: -1
2025-12-24T02:11:59.224910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [3:7587241490043514561:2110] [node 5] ICS10 traffic start
2025-12-24T02:11:59.224928Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [5] connected
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.225732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241489701736610:2116] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.225827Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241489701736601:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.225837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.225845Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241489701736601:2111] [node 4] ICP22 created new session: [5:7587241489701736628:2111]
2025-12-24T02:11:59.225851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241489701736628:2111] [node 4] ICS09 handshake done sender: [5:7587241489701736610:2116] self: [5:7587241489701736624:0] peer: [4:7587241487898099545:0] socket: 683 qp: -1
2025-12-24T02:11:59.225856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241489701736628:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:59.225863Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.225917Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099565:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.225935Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [5:7587241489701736627:2123] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:34431
2025-12-24T02:11:59.226087Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241487898099552:2116] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.226190Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [4:7587241487898099551:2115] [node 6] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.226273Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [4:7587241487898099473:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.226295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.226310Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241487898099473:2050] [node 3] ICP22 created new session: [4:7587241487898099567:2050]
2025-12-24T02:11:59.226319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241487898099567:2050] [node 3] ICS09 handshake done sender: [4:7587241487898099552:2116] self: [4:7587241487898099561:0] peer: [3:7587241490043514530:0] socket: 660 qp: -1
2025-12-24T02:11:59.226332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241487898099567:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.226341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:59.223224Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816390:2120] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.223341Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241487174816381:2117] [node 3] ICH04 handshake succeeded
2025-12-24T02:11:59.223435Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241487174816298:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.223455Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.223462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241487174816298:2050] [node 3] ICP22 created new session: [6:7587241487174816391:2050]
2025-12-24T02:11:59.223469Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241487174816391:2050] [node 3] ICS09 handshake done sender: [6:7587241487174816381:2117] self: [6:7587241487174816387:0] peer: [3:7587241490043514540:0] socket: 691 qp: -1
2025-12-24T02:11:59.223476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241487174816391:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.223503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:59.226276Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [6:7587241487174816393:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47213
2025-12-24T02:11:59.226763Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [5:7587241489701736612:2118] [node 3] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:11:59.226914Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [6:7587241487174816388:2119] [node 4] ICH04 handshake succeeded
2025-12-24T02:11:59.227005Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [6:7587241487174816369:2111] [node 4] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.227027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [4] session created
2025-12-24T02:11:59.227034Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [6:7587241487174816369:2111] [node 4] ICP22 created new session: [6:7587241487174816394:2111]
2025-12-24T02:11:59.227041Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [6:7587241487174816394:2111] [node 4] ICS09 handshake done sender: [6:7587241487174816388:2119] self: [6:7587241487174816389:0] peer: [4:7587241487898099550:0] socket: 658 qp: -1
2025-12-24T02:11:59.227048Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [6:7587241487174816394:2111] [node 4] ICS10 traffic start
2025-12-24T02:11:59.227064Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [4] connected
2025-12-24T02:11:59.228173Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [5:7587241489701736532:2050] [node 3] ICP19 incoming handshake succeeded
2025-12-24T02:11:59.228203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [3] session created
2025-12-24T02:11:59.228212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [5:7587241489701736532:2050] [node 3] ICP22 created new session: [5:7587241489701736630:2050]
2025-12-24T02:11:59.228219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [5:7587241489701736630:2050] [node 3] ICS09 handshake done sender: [5:7587241489701736612:2118] self: [5:7587241489701736625:0] peer: [3:7587241490043514539:0] socket: 686 qp: -1
2025-12-24T02:11:59.228227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [5:7587241489701736630:2050] [node 3] ICS10 traffic start
2025-12-24T02:11:59.228236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [3] connected
2025-12-24T02:11:59.228556Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [4:7587241487898099569:2121] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:47463
2025-12-24T02:11:59.231899Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [4:7587241487898099549:2114] [node 6] ICP20 outgoing handshake succeeded
2025-12-24T02:11:59.231942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [6] session created
2025-12-24T02:11:59.231954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [4:7587241487898099549:2114] [node 6] ICP22 created new session: [4:7587241487898099570:2114]
2025-12-24T02:11:59.231962Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [4:7587241487898099570:2114] [node 6] ICS09 handshake done sender: [4:7587241487898099551:2115] self: [4:7587241487898099550:0] peer: [6:7587241487174816389:0] socket: 680 qp: -1
2025-12-24T02:11:59.231974Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [4:7587241487898099570:2114] [node 6] ICS10 traffic start
2025-12-24T02:11:59.232001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [6] connected
2025-12-24T02:11:59.234820Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [4:7587241487898099544:2112] [node 5] ICP27 obsolete handshake fail ignored
2025-12-24T02:11:59.252823Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:11:59.252733Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 1] connection closed by peer
2025-12-24T02:11:59.252828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [3:7587241490043514524:2048] [node 1] ICS07 socket disconnect 637 reason# EndOfStream
2025-12-24T02:11:59.252833Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241490043514524:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.252916Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241490433832826:2048] [node 1] ICS07 socket disconnect 632 reason# EndOfStream
2025-12-24T02:11:59.252922Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241490433832826:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.252994Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.253025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241490433832826:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.252901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.252929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [3:7587241490043514524:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.253324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [3:7587241490043514461:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.253354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241490043514524:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.253363Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241490043514461:2048] [node 1] ICP30 unregister session Session# [3:7587241490043514524:2048] VirtualId# [3:7587241490043514522:0]
2025-12-24T02:11:59.253438Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.253450Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [3:7587241490043514461:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.253452Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.253418Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241490433832763:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.253446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241490433832826:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.253455Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241490433832763:2048] [node 1] ICP30 unregister session Session# [2:7587241490433832826:2048] VirtualId# [2:7587241490433832824:0]
2025-12-24T02:11:59.253513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.253520Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241490433832763:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.253522Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.254243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 1] connection closed by peer
2025-12-24T02:11:59.254347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241489701736622:2048] [node 1] ICS07 socket disconnect 659 reason# EndOfStream
2025-12-24T02:11:59.254356Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241489701736622:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.254410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.254443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241489701736622:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.254782Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 1] connection closed by peer
2025-12-24T02:11:59.254843Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241489701736530:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.254891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [4:7587241487898099536:2048] [node 1] ICS07 socket disconnect 640 reason# EndOfStream
2025-12-24T02:11:59.254904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241487898099536:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.254862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241489701736622:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.254869Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241489701736530:2048] [node 1] ICP30 unregister session Session# [5:7587241489701736622:2048] VirtualId# [5:7587241489701736613:0]
2025-12-24T02:11:59.254929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.254957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.254982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [4:7587241487898099536:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.254937Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241489701736530:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.254791Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 1] connection closed by peer
2025-12-24T02:11:59.254939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.254887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241487174816382:2048] [node 1] ICS07 socket disconnect 647 reason# EndOfStream
2025-12-24T02:11:59.254898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241487174816382:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.254957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:11:59.254982Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241487174816382:2048] [node 1] ICS15 start handshake
2025-12-24T02:11:59.255385Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [4:7587241487898099471:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.255400Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241487898099536:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.255406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241487898099471:2048] [node 1] ICP30 unregister session Session# [4:7587241487898099536:2048] VirtualId# [4:7587241487898099533:0]
2025-12-24T02:11:59.255470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.255482Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [4:7587241487898099471:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.255485Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.255811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241487174816296:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.255829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241487174816382:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.255836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241487174816296:2048] [node 1] ICP30 unregister session Session# [6:7587241487174816382:2048] VirtualId# [6:7587241487174816366:0]
2025-12-24T02:11:59.255884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:11:59.255894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241487174816296:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.255897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26396) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.263360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [3:7587241490043514461:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:59.264367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241489701736530:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:59.266799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [4:7587241487898099471:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:59.266740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241487174816296:2048] [node 1] ICP33 wake up from error state
2025-12-24T02:11:59.269405Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [3 <-> 2] connection closed by peer
2025-12-24T02:11:59.269531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [3:7587241490043514544:2049] [node 2] ICS01 socket: 652 reason# EndOfStream
2025-12-24T02:11:59.269542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [3:7587241490043514462:2049] [node 2] ICP30 unregister session Session# [3:7587241490043514544:2049] VirtualId# [3:7587241490043514542:0]
2025-12-24T02:11:59.269552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [3:7587241490043514544:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.269623Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.269650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.273061Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 2] connection closed by peer
2025-12-24T02:11:59.273171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241489701736619:2049] [node 2] ICS01 socket: 684 reason# EndOfStream
2025-12-24T02:11:59.273185Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241489701736531:2049] [node 2] ICP30 unregister session Session# [5:7587241489701736619:2049] VirtualId# [5:7587241489701736614:0]
2025-12-24T02:11:59.273193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241489701736619:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.273259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.273320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.274743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 2] connection closed by peer
2025-12-24T02:11:59.274822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241487174816385:2049] [node 2] ICS01 socket: 666 reason# EndOfStream
2025-12-24T02:11:59.274830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241487174816297:2049] [node 2] ICP30 unregister session Session# [6:7587241487174816385:2049] VirtualId# [6:7587241487174816378:0]
2025-12-24T02:11:59.274835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241487174816385:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.274884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.274910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.275164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [4 <-> 2] connection closed by peer
2025-12-24T02:11:59.275257Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [4:7587241487898099558:2049] [node 2] ICS01 socket: 675 reason# EndOfStream
2025-12-24T02:11:59.275264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [4:7587241487898099472:2049] [node 2] ICP30 unregister session Session# [4:7587241487898099558:2049] VirtualId# [4:7587241487898099554:0]
2025-12-24T02:11:59.275271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [4:7587241487898099558:2049] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.275343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:11:59.275372Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:11:59.282802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 3] connection closed by peer
2025-12-24T02:11:59.282840Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 3] connection closed by peer
2025-12-24T02:11:59.282945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241489701736630:2050] [node 3] ICS07 socket disconnect 686 reason# EndOfStream
2025-12-24T02:11:59.282951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241489701736630:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.282999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:59.283022Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241489701736630:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:59.282920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [6:7587241487174816391:2050] [node 3] ICS07 socket disconnect 691 reason# EndOfStream
2025-12-24T02:11:59.282929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241487174816391:2050] [node 3] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.282976Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [3] disconnected
2025-12-24T02:11:59.282997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [6:7587241487174816391:2050] [node 3] ICS15 start handshake
2025-12-24T02:11:59.283440Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [6:7587241487174816298:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.283453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241487174816391:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.283458Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241487174816298:2050] [node 3] ICP30 unregister session Session# [6:7587241487174816391:2050] VirtualId# [6:7587241487174816387:0]
2025-12-24T02:11:59.283498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:59.283503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [6:7587241487174816298:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.283506Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.287139Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241489701736532:2050] [node 3] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.287160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241489701736630:2050] [node 3] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.287166Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241489701736532:2050] [node 3] ICP30 unregister session Session# [5:7587241489701736630:2050] VirtualId# [5:7587241489701736625:0]
2025-12-24T02:11:59.287213Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [3] session destroyed
2025-12-24T02:11:59.287219Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241489701736532:2050] [node 3] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.287221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [3] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:64587) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.294749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [6:7587241487174816298:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:59.297356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:801: Proxy [5:7587241489701736532:2050] [node 3] ICP33 wake up from error state
2025-12-24T02:11:59.299801Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [5 <-> 4] connection closed by peer
2025-12-24T02:11:59.299943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [5:7587241489701736628:2111] [node 4] ICS07 socket disconnect 683 reason# EndOfStream
2025-12-24T02:11:59.299955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [5:7587241489701736628:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.300024Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:59.300057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [5:7587241489701736628:2111] [node 4] ICS15 start handshake
2025-12-24T02:11:59.300477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [5:7587241489701736601:2111] [node 4] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26553) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:11:59.300485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [5:7587241489701736628:2111] [node 4] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:11:59.300489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [5:7587241489701736601:2111] [node 4] ICP30 unregister session Session# [5:7587241489701736628:2111] VirtualId# [5:7587241489701736624:0]
2025-12-24T02:11:59.300525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
2025-12-24T02:11:59.300529Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [5:7587241489701736601:2111] [node 4] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:26553) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.300531Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [4] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:26553) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:11:59.302774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [6 <-> 4] connection closed by peer
2025-12-24T02:11:59.302867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [6:7587241487174816394:2111] [node 4] ICS01 socket: 658 reason# EndOfStream
2025-12-24T02:11:59.302875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [6:7587241487174816369:2111] [node 4] ICP30 unregister session Session# [6:7587241487174816394:2111] VirtualId# [6:7587241487174816389:0]
2025-12-24T02:11:59.302883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [6:7587241487174816394:2111] [node 4] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:11:59.302940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [4] disconnected
2025-12-24T02:11:59.302969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [4] session destroyed
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
Error.size# 10
2025-12-24T02:12:01.065013Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241498620650951:2048] [node 2] ICP01 ready to work
nodeId# 1 peerNodeId# 2 TEvPoisonSession
Subscribe
2025-12-24T02:12:01.068723Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241498244783284:2048] [node 1] ICP01 ready to work
2025-12-24T02:12:01.073245Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241498620650960:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:01.073777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:01.074034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241498244783294:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:01.076294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241498620650960:2056] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:01.076399Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:01.076426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:01.076437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241498620650963:2048]
2025-12-24T02:12:01.076448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241498620650963:2048] [node 2] ICS09 handshake done sender: [1:7587241498620650960:2056] self: [1:7587241498620650961:0] peer: [2:7587241498244783293:0] socket: 17 qp: 22138
2025-12-24T02:12:01.076457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241498620650963:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:01.076509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeConnected
2025-12-24T02:12:01.076333Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241498244783294:2056] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:01.076441Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:01.076454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:01.076462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241498244783297:2048]
2025-12-24T02:12:01.076468Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241498244783297:2048] [node 1] ICS09 handshake done sender: [2:7587241498244783294:2056] self: [2:7587241498244783293:0] peer: [1:7587241498620650961:0] socket: 16 qp: 22137
2025-12-24T02:12:01.076474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241498244783297:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:01.076484Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:01.659401Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:01.659629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241498620650963:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:01.659642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241498620650963:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:01.659650Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:01.659671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241498620650963:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:01.661160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241498620650963:2048] [node 2] ICS08 incoming handshake Self# [2:7587241498244783293:0] Peer# [1:7587241498620650961:0] Counter# 183 LastInputSerial# 7383
2025-12-24T02:12:01.658843Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:01.658943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241498244783297:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:01.658949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241498244783297:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:01.658981Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:01.658997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241498244783297:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:01.662465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241498244783297:2048] [node 1] ICS08 incoming handshake Self# [1:7587241498620650961:0] Peer# [2:7587241498244783293:0] Counter# 7383 LastInputSerial# 183
2025-12-24T02:12:01.662817Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241498620650965:2058] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:01.662983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:01.663004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241498620650963:2048] [node 2] ICS09 handshake done sender: [1:7587241498620650965:2058] self: [1:7587241498620650961:0] peer: [2:7587241498244783293:0] socket: 24 qp: 22608
2025-12-24T02:12:01.663343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241498620650963:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:01.663365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:01.665168Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241498244783301:2058] held: no
2025-12-24T02:12:01.665202Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241498244783301:2058] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:01.665265Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:01.665278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241498244783297:2048] [node 1] ICS09 handshake done sender: [2:7587241498244783301:2058] self: [2:7587241498244783293:0] peer: [1:7587241498620650961:0] socket: 25 qp: 22607
2025-12-24T02:12:01.665573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241498244783297:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:01.665596Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:01.759597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:01.759744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241498244783297:2048] [node 1] ICS07 socket disconnect 25 reason# Debug
2025-12-24T02:12:01.759754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241498244783297:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:01.759777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:01.759813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241498244783297:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:01.759842Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:01.759950Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241498620650963:2048] [node 2] ICS01 socket: 24 reason# EndOfStream
2025-12-24T02:12:01.759991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241498620650963:2048] VirtualId# [1:7587241498620650961:0]
2025-12-24T02:12:01.759997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241498620650963:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:01.760043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:01.760076Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:01.762389Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241498620650970:2060]) peer tries to resume nonexistent session Self# [2:7587241498244783293:0] Peer# [1:7587241498620650961:0]
2025-12-24T02:12:01.762428Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241498620650970:2060] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:01.762516Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241498244783304:2060] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:01.763456Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:01.763498Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:01.763505Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241498244783297:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:01.763531Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241498244783297:2048] VirtualId# [2:7587241498244783293:0]
2025-12-24T02:12:01.763578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 2097
TEvUndelivered Cookie# 2098
TEvUndelivered Cookie# 2099
TEvUndelivered Cookie# 2100
TEvUndelivered Cookie# 2101
2025-12-24T02:12:01.766524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241498244783308:2062] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:01.766617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:01.766645Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241498620650974:2062] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:01.766939Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241498620650972:2061] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:01.769016Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241498244783308:2062] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:01.769073Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241498620650972:2061] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:01.769106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:01.769130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:01.769169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241498244783312:2048]
2025-12-24T02:12:01.769182Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241498244783312:2048] [node 1] ICS09 handshake done sender: [2:7587241498244783308:2062] self: [2:7587241498244783310:0] peer: [1:7587241498620650971:0] socket: 17 qp: 22730
2025-12-24T02:12:01.769192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241498244783312:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:01.769206Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:01.769253Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:01.769277Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:01.769289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241498620650977:2048]
2025-12-24T02:12:01.769301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241498620650977:2048] [node 2] ICS09 handshake done sender: [1:7587241498620650972:2061] self: [1:7587241498620650971:0] peer: [2:7587241498244783310:0] socket: 16 qp: 22728
2025-12-24T02:12:01.769312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241498620650977:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:01.769324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:02.067038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241498244783312:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:02.067090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241498244783312:2048] VirtualId# [2:7587241498244783310:0]
2025-12-24T02:12:02.067099Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241498244783312:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:02.067145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:02.067183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:02.067202Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:02.067315Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241498620650977:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:02.067328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241498620650977:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:02.067386Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:02.067408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241498620650977:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 2743
TEvUndelivered Cookie# 2744
TEvUndelivered Cookie# 2745
TEvUndelivered Cookie# 2746
TEvUndelivered Cookie# 2747
2025-12-24T02:12:02.070460Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241502539750612:2065]) peer tries to resume nonexistent session Self# [1:7587241498620650971:0] Peer# [2:7587241498244783310:0]
2025-12-24T02:12:02.070486Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241502539750612:2065] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:02.070663Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241502915618277:2064] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:02.070719Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:02.070725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241498620650977:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:02.070762Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241498620650977:2048] VirtualId# [1:7587241498620650971:0]
2025-12-24T02:12:02.070791Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:02.071336Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241502915618275:2063] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:02.071821Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [1:7587241502915618277:2064] held: yes
2025-12-24T02:12:02.071983Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241502539750611:2064] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:02.074719Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241502915618277:2064] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:02.074836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:02.074859Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:02.074822Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241502539750611:2064] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:02.074870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241502915618280:2048]
2025-12-24T02:12:02.074882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241502915618280:2048] [node 2] ICS09 handshake done sender: [1:7587241502915618277:2064] self: [1:7587241502915618278:0] peer: [2:7587241502539750610:0] socket: 23 qp: 23008
2025-12-24T02:12:02.074892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241502915618280:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:02.074909Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:02.075110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:02.075130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:02.075138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241502539750615:2048]
2025-12-24T02:12:02.075144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241502539750615:2048] [node 1] ICS09 handshake done sender: [2:7587241502539750611:2064] self: [2:7587241502539750610:0] peer: [1:7587241502915618278:0] socket: 21 qp: 23007
2025-12-24T02:12:02.075151Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241502539750615:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:02.075163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:02.459314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241502915618280:2048] [node 2] ICS01 socket: 23 reason# Debug
2025-12-24T02:12:02.459382Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241502915618280:2048] VirtualId# [1:7587241502915618278:0]
2025-12-24T02:12:02.459392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241502915618280:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:02.459416Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:02.459448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:02.461261Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:02.461293Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241502539750615:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:02.461306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241502539750615:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:02.461318Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:02.461421Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:02.461527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241502539750615:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:02.461535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241502539750615:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:02.492423Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241502915618282:2066]) peer tries to resume nonexistent session Self# [2:7587241502539750610:0] Peer# [1:7587241502915618278:0]
2025-12-24T02:12:02.492452Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241502915618282:2066] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:02.492973Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241502539750617:2066] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:02.493477Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:02.493525Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:02.493532Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241502539750615:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:02.493583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241502539750615:2048] VirtualId# [2:7587241502539750610:0]
2025-12-24T02:12:02.493616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:02.504050Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241502915618283:2067] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:02.504144Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:02.506853Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241502539750620:2067] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:02.531928Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241502915618283:2067] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:02.532195Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241502539750620:2067] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:02.532115Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:02.532145Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:02.532155Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241502915618286:2048]
2025-12-24T02:12:02.532167Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241502915618286:2048] [node 2] ICS09 handshake done sender: [1:7587241502915618283:2067] self: [1:7587241502915618284:0] peer: [2:7587241502539750619:0] socket: 17 qp: 23345
2025-12-24T02:12:02.532178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241502915618286:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:02.532196Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:02.532360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:02.532376Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:02.532384Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241502539750623:2048]
2025-12-24T02:12:02.532392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241502539750623:2048] [node 1] ICS09 handshake done sender: [2:7587241502539750620:2067] self: [2:7587241502539750619:0] peer: [1:7587241502915618284:0] socket: 16 qp: 23346
2025-12-24T02:12:02.532399Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241502539750623:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:02.532410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:02.958932Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:02.959743Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:02.959933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241502915618286:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:02.960084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241502915618286:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:02.960135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:02.960164Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241502915618286:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:02.959585Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241502539750623:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:02.959610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241502539750623:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:02.959647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:02.959670Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241502539750623:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:02.976278Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241502915618286:2048] [node 2] ICS08 incoming handshake Self# [2:7587241502539750619:0] Peer# [1:7587241502915618284:0] Counter# 136 LastInputSerial# 3561
2025-12-24T02:12:03.021236Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241502539750623:2048] [node 1] ICS08 incoming handshake Self# [1:7587241502915618284:0] Peer# [2:7587241502539750619:0] Counter# 3561 LastInputSerial# 136
2025-12-24T02:12:03.022655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241502915618288:2069] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:03.022813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:03.022834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241502915618286:2048] [node 2] ICS09 handshake done sender: [1:7587241502915618288:2069] self: [1:7587241502915618284:0] peer: [2:7587241502539750619:0] socket: 24 qp: 23584
2025-12-24T02:12:03.023067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241502915618286:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:03.023088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:03.027664Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241502539750627:2069] held: no
2025-12-24T02:12:03.029641Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241502539750627:2069] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:03.029734Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:03.029749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241502539750623:2048] [node 1] ICS09 handshake done sender: [2:7587241502539750627:2069] self: [2:7587241502539750619:0] peer: [1:7587241502915618284:0] socket: 25 qp: 23577
2025-12-24T02:12:03.030094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241502539750623:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:03.030109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:03.510884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:03.511489Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:03.511525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241502539750623:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:03.511542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241502539750623:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:03.511554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:03.511608Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:03.511690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241502539750623:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:03.511698Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241502539750623:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:03.514981Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-12-24T02:12:03.515080Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241502915618286:2048] [node 2] ICS07 socket disconnect 24 reason# ECONNRESET
2025-12-24T02:12:03.515111Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241502915618286:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-12-24T02:12:03.515125Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:03.515162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241502915618286:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:03.576373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241502539750623:2048] [node 1] ICS08 incoming handshake Self# [1:7587241502915618284:0] Peer# [2:7587241502539750619:0] Counter# 10049 LastInputSerial# 488
2025-12-24T02:12:03.576275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241502915618286:2048] [node 2] ICS08 incoming handshake Self# [2:7587241502539750619:0] Peer# [1:7587241502915618284:0] Counter# 488 LastInputSerial# 10049
2025-12-24T02:12:03.577309Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241507210585589:2071] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:03.577515Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:03.577559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241502915618286:2048] [node 2] ICS09 handshake done sender: [1:7587241507210585589:2071] self: [1:7587241502915618284:0] peer: [2:7587241502539750619:0] socket: 21 qp: 23895
2025-12-24T02:12:03.577963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241502915618286:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:03.577993Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:03.580794Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241506834717928:2072] held: no
2025-12-24T02:12:03.580831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241506834717928:2072] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:03.580881Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:03.580892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241502539750623:2048] [node 1] ICS09 handshake done sender: [2:7587241506834717928:2072] self: [2:7587241502539750619:0] peer: [1:7587241502915618284:0] socket: 22 qp: 23901
2025-12-24T02:12:03.581162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241502539750623:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:03.581177Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:03.808830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241502539750623:2048] [node 1] ICS01 socket: 22 reason# Debug
2025-12-24T02:12:03.808870Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241502539750623:2048] VirtualId# [2:7587241502539750619:0]
2025-12-24T02:12:03.808877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241502539750623:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:03.808899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:03.808948Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:03.809447Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:03.809545Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241502915618286:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:03.809553Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241502915618286:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:03.809581Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:03.809592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241502915618286:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:03.812214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241506834717931:2074]) peer tries to resume nonexistent session Self# [1:7587241502915618284:0] Peer# [2:7587241502539750619:0]
2025-12-24T02:12:03.812233Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241506834717931:2074] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:03.812575Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241507210585594:2073] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:03.813131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:03.813190Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:03.813200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241502915618286:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:03.813228Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241502915618286:2048] VirtualId# [1:7587241502915618284:0]
2025-12-24T02:12:03.813272Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:03.827711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241507210585596:2074] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:03.830375Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241506834717935:2076] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:03.830418Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:03.830650Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241507210585598:2075] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:03.861655Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241506834717935:2076] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:03.861732Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241507210585598:2075] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:03.861806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:03.861835Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:03.861874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241506834717938:2048]
2025-12-24T02:12:03.861883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241506834717938:2048] [node 1] ICS09 handshake done sender: [2:7587241506834717935:2076] self: [2:7587241506834717936:0] peer: [1:7587241507210585597:0] socket: 21 qp: 24057
2025-12-24T02:12:03.861893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241506834717938:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:03.861907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:03.861884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:03.861910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:03.861936Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241507210585602:2048]
2025-12-24T02:12:03.861944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241507210585602:2048] [node 2] ICS09 handshake done sender: [1:7587241507210585598:2075] self: [1:7587241507210585597:0] peer: [2:7587241506834717936:0] socket: 16 qp: 24058
2025-12-24T02:12:03.861951Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241507210585602:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:03.861960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:04.263112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241507210585602:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:04.263162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241507210585602:2048] VirtualId# [1:7587241507210585597:0]
2025-12-24T02:12:04.263218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241507210585602:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:04.263259Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:04.263287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:04.263907Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:04.264003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241506834717938:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:04.264028Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241506834717938:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:04.264039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:04.264082Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241506834717938:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:04.291642Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241511505552900:2076]) peer tries to resume nonexistent session Self# [2:7587241506834717936:0] Peer# [1:7587241507210585597:0]
2025-12-24T02:12:04.291678Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241511505552900:2076] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:04.292117Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241511129685236:2078] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:04.292719Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:04.292790Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:04.292796Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241506834717938:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:04.292851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241506834717938:2048] VirtualId# [2:7587241506834717936:0]
2025-12-24T02:12:04.292881Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:04.330291Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241511505552901:2077] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:04.330358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:04.330524Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241511129685239:2079] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:04.347297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241511505552901:2077] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:04.347352Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241511129685239:2079] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:04.347422Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:04.347441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:04.347449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241511505552904:2048]
2025-12-24T02:12:04.347460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241511505552904:2048] [node 2] ICS09 handshake done sender: [1:7587241511505552901:2077] self: [1:7587241511505552902:0] peer: [2:7587241511129685238:0] socket: 17 qp: 24312
2025-12-24T02:12:04.347472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241511505552904:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:04.347483Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:04.347482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:04.347537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:04.347548Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241511129685242:2048]
2025-12-24T02:12:04.347559Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241511129685242:2048] [node 1] ICS09 handshake done sender: [2:7587241511129685239:2079] self: [2:7587241511129685238:0] peer: [1:7587241511505552902:0] socket: 16 qp: 24309
2025-12-24T02:12:04.347569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241511129685242:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:04.347602Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:04.826374Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241511505552904:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:04.826436Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241511505552904:2048] VirtualId# [1:7587241511505552902:0]
2025-12-24T02:12:04.826447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241511505552904:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:04.826482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:04.826526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:04.826546Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:04.826755Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:04.826770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241511129685242:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:04.826775Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241511129685242:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:04.826788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:04.826816Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241511129685242:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:04.826826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241511129685242:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:04.829559Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241511505552906:2079]) peer tries to resume nonexistent session Self# [2:7587241511129685238:0] Peer# [1:7587241511505552902:0]
2025-12-24T02:12:04.829589Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241511505552906:2079] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:04.829684Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241511129685244:2080] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:04.830326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:04.830362Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:04.830375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241511129685242:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:04.830409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241511129685242:2048] VirtualId# [2:7587241511129685238:0]
2025-12-24T02:12:04.830448Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 9026
TEvUndelivered Cookie# 9027
TEvUndelivered Cookie# 9028
2025-12-24T02:12:04.835370Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241511505552907:2080] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:04.835440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:04.836118Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241511129685247:2081] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:04.847130Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241511505552907:2080] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:04.847238Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:04.847261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:04.847277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241511505552910:2048]
2025-12-24T02:12:04.847289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241511505552910:2048] [node 2] ICS09 handshake done sender: [1:7587241511505552907:2080] self: [1:7587241511505552908:0] peer: [2:7587241511129685246:0] socket: 17 qp: 24620
2025-12-24T02:12:04.847229Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241511129685247:2081] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:04.847322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241511505552910:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:04.847342Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:04.847467Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:04.847487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:04.847497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241511129685250:2048]
2025-12-24T02:12:04.847504Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241511129685250:2048] [node 1] ICS09 handshake done sender: [2:7587241511129685247:2081] self: [2:7587241511129685246:0] peer: [1:7587241511505552908:0] socket: 16 qp: 24619
2025-12-24T02:12:04.847512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241511129685250:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:04.847520Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:05.104197Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:05.104290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241511129685250:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:05.104295Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241511129685250:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:05.104330Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:05.104349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241511129685250:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:05.103384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241511505552910:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:05.103454Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241511505552910:2048] VirtualId# [1:7587241511505552908:0]
2025-12-24T02:12:05.103464Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241511505552910:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:05.103499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:05.103526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:05.148763Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241515800520211:2083]) peer tries to resume nonexistent session Self# [2:7587241511129685246:0] Peer# [1:7587241511505552908:0]
2025-12-24T02:12:05.148790Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241515800520211:2083] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:05.150909Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241515424652549:2083] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:05.150958Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241515424652548:2082] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:05.151806Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:05.151857Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:05.151867Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241511129685250:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:05.151906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241511129685250:2048] VirtualId# [2:7587241511129685246:0]
2025-12-24T02:12:05.151947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:05.152736Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:05.152972Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241515800520209:2082] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:05.156606Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241515800520212:2084] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:05.156979Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241515424652549:2083] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:05.157087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:05.157109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:05.157063Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241515800520209:2082] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:05.157118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241515424652556:2048]
2025-12-24T02:12:05.157130Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241515424652556:2048] [node 1] ICS09 handshake done sender: [2:7587241515424652549:2083] self: [2:7587241515424652553:0] peer: [1:7587241515800520208:0] socket: 21 qp: 24806
2025-12-24T02:12:05.157146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241515424652556:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:05.157163Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:05.157211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:05.157240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:05.157250Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241515800520215:2048]
2025-12-24T02:12:05.157259Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241515800520215:2048] [node 2] ICS09 handshake done sender: [1:7587241515800520209:2082] self: [1:7587241515800520208:0] peer: [2:7587241515424652553:0] socket: 17 qp: 24807
2025-12-24T02:12:05.157267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241515800520215:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:05.157279Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:05.681695Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:05.681858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241515424652556:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-12-24T02:12:05.681871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241515424652556:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:05.681897Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:05.681915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241515424652556:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:05.681941Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:05.682037Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241515800520215:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:05.682051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241515800520215:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:05.682086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:05.682115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241515800520215:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:05.684039Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241515800520215:2048] [node 2] ICS08 incoming handshake Self# [2:7587241515424652553:0] Peer# [1:7587241515800520208:0] Counter# 199 LastInputSerial# 6167
2025-12-24T02:12:05.685431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241515424652556:2048] [node 1] ICS08 incoming handshake Self# [1:7587241515800520208:0] Peer# [2:7587241515424652553:0] Counter# 6167 LastInputSerial# 199
2025-12-24T02:12:05.685660Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241515800520217:2085] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:05.685851Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:05.685892Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241515800520215:2048] [node 2] ICS09 handshake done sender: [1:7587241515800520217:2085] self: [1:7587241515800520208:0] peer: [2:7587241515424652553:0] socket: 24 qp: 25314
2025-12-24T02:12:05.686567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241515800520215:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:05.686591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:05.689079Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241515424652560:2087] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:05.689223Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241515424652560:2087] held: no
2025-12-24T02:12:05.689243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:05.689254Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241515424652556:2048] [node 1] ICS09 handshake done sender: [2:7587241515424652560:2087] self: [2:7587241515424652553:0] peer: [1:7587241515800520208:0] socket: 25 qp: 25313
2025-12-24T02:12:05.689557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241515424652556:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:05.689580Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:06.199356Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:06.199462Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:06.199475Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241515424652556:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:06.199484Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241515424652556:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:06.199497Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:06.199466Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:06.199533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:06.199589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241515800520215:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-12-24T02:12:06.199603Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241515800520215:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:06.199626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241515424652556:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:06.199634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241515424652556:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:06.199654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:06.199694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241515800520215:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:06.202522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241515424652556:2048] [node 1] ICS08 incoming handshake Self# [1:7587241515800520208:0] Peer# [2:7587241515424652553:0] Counter# 13025 LastInputSerial# 383
2025-12-24T02:12:06.202753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241515800520215:2048] [node 2] ICS08 incoming handshake Self# [2:7587241515424652553:0] Peer# [1:7587241515800520208:0] Counter# 383 LastInputSerial# 13025
2025-12-24T02:12:06.202967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241520095487518:2087] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:06.203102Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:06.203124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241515800520215:2048] [node 2] ICS09 handshake done sender: [1:7587241520095487518:2087] self: [1:7587241515800520208:0] peer: [2:7587241515424652553:0] socket: 17 qp: 25706
2025-12-24T02:12:06.203623Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241515800520215:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:06.203649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:06.205607Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241519719619861:2090] held: no
2025-12-24T02:12:06.205646Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241519719619861:2090] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:06.205705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:06.205726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241515424652556:2048] [node 1] ICS09 handshake done sender: [2:7587241519719619861:2090] self: [2:7587241515424652553:0] peer: [1:7587241515800520208:0] socket: 21 qp: 25708
2025-12-24T02:12:06.206165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241515424652556:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:06.206190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:06.386274Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241515800520215:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:06.386320Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241515800520215:2048] VirtualId# [1:7587241515800520208:0]
2025-12-24T02:12:06.386327Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241515800520215:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:06.386353Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:06.386402Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:06.386400Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:06.386522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241515424652556:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:06.386539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241515424652556:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:06.386590Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:06.386632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241515424652556:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:06.389054Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241520095487523:2089]) peer tries to resume nonexistent session Self# [2:7587241515424652553:0] Peer# [1:7587241515800520208:0]
2025-12-24T02:12:06.389154Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241520095487523:2089] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:06.389483Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241519719619864:2092] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:06.390086Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:06.390123Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:06.390133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241515424652556:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:06.390162Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241515424652556:2048] VirtualId# [2:7587241515424652553:0]
2025-12-24T02:12:06.390193Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:06.394057Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241519719619866:2093] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:06.394111Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:06.394267Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241520095487525:2090] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:06.394416Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241520095487527:2091] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:06.396666Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241519719619866:2093] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:06.396727Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241520095487525:2090] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:06.396750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:06.396772Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:06.396781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241519719619872:2048]
2025-12-24T02:12:06.396789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241519719619872:2048] [node 1] ICS09 handshake done sender: [2:7587241519719619866:2093] self: [2:7587241519719619870:0] peer: [1:7587241520095487524:0] socket: 22 qp: 25877
2025-12-24T02:12:06.396798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241519719619872:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:06.396810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:06.396853Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:06.396876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:06.396884Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241520095487530:2048]
2025-12-24T02:12:06.396897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241520095487530:2048] [node 2] ICS09 handshake done sender: [1:7587241520095487525:2090] self: [1:7587241520095487524:0] peer: [2:7587241519719619870:0] socket: 20 qp: 25875
2025-12-24T02:12:06.396904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241520095487530:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:06.396913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:06.888570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:06.888675Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:06.888687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241519719619872:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:06.888693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241519719619872:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:06.888705Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:06.888672Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:06.888742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:06.888826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241519719619872:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:06.888834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241519719619872:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:06.888795Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241520095487530:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:06.888810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241520095487530:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:06.888857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:06.888886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241520095487530:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:06.891422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241519719619872:2048] [node 1] ICS08 incoming handshake Self# [1:7587241520095487524:0] Peer# [2:7587241519719619870:0] Counter# 7292 LastInputSerial# 294
2025-12-24T02:12:06.891638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241520095487530:2048] [node 2] ICS08 incoming handshake Self# [2:7587241519719619870:0] Peer# [1:7587241520095487524:0] Counter# 294 LastInputSerial# 7292
2025-12-24T02:12:06.891808Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241520095487532:2092] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:06.891934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:06.891955Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241520095487530:2048] [node 2] ICS09 handshake done sender: [1:7587241520095487532:2092] self: [1:7587241520095487524:0] peer: [2:7587241519719619870:0] socket: 17 qp: 26514
2025-12-24T02:12:06.892470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241520095487530:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:06.892498Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:06.894446Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241519719619876:2097] held: no
2025-12-24T02:12:06.894473Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241519719619876:2097] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:06.894542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:06.894558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241519719619872:2048] [node 1] ICS09 handshake done sender: [2:7587241519719619876:2097] self: [2:7587241519719619870:0] peer: [1:7587241520095487524:0] socket: 25 qp: 26516
2025-12-24T02:12:06.895024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241519719619872:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:06.895043Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:07.201428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241520095487530:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:07.201502Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241520095487530:2048] VirtualId# [1:7587241520095487524:0]
2025-12-24T02:12:07.201511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241520095487530:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:07.201542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:07.201566Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:07.202164Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:07.202320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241519719619872:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:07.202329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241519719619872:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:07.202338Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:07.202361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241519719619872:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:07.211222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241524390454836:2095]) peer tries to resume nonexistent session Self# [2:7587241519719619870:0] Peer# [1:7587241520095487524:0]
2025-12-24T02:12:07.211254Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241524390454836:2095] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:07.215947Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241524014587175:2099] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:07.216411Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:07.216452Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:07.216461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241519719619872:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:07.216482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241519719619872:2048] VirtualId# [2:7587241519719619870:0]
2025-12-24T02:12:07.216527Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:07.217156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241524014587177:2100] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
TEvUndelivered Cookie# 16030
TEvUndelivered Cookie# 16031
TEvUndelivered Cookie# 16032
TEvUndelivered Cookie# 16033
2025-12-24T02:12:07.217259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:07.217485Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241524390454834:2094] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:07.221193Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241524390454837:2096] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:07.221272Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241524014587177:2100] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:07.221297Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241524390454834:2094] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:07.221340Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:07.221365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:07.221373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241524014587183:2048]
2025-12-24T02:12:07.221381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241524014587183:2048] [node 1] ICS09 handshake done sender: [2:7587241524014587177:2100] self: [2:7587241524014587181:0] peer: [1:7587241524390454833:0] socket: 20 qp: 26904
2025-12-24T02:12:07.221389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241524014587183:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:07.221406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:07.221420Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:07.221439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:07.221447Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241524390454840:2048]
2025-12-24T02:12:07.221457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241524390454840:2048] [node 2] ICS09 handshake done sender: [1:7587241524390454834:2094] self: [1:7587241524390454833:0] peer: [2:7587241524014587181:0] socket: 17 qp: 26898
2025-12-24T02:12:07.221463Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241524390454840:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:07.221475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:07.714575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:07.714677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241524390454840:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-12-24T02:12:07.714687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241524390454840:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:07.714740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:07.714763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241524390454840:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:07.714778Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:07.714853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241524014587183:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:07.714862Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241524014587183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:07.714921Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:07.714947Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241524014587183:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:07.718882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241524014587183:2048] [node 1] ICS08 incoming handshake Self# [1:7587241524390454833:0] Peer# [2:7587241524014587181:0] Counter# 7018 LastInputSerial# 431
2025-12-24T02:12:07.720021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241524390454840:2048] [node 2] ICS08 incoming handshake Self# [2:7587241524014587181:0] Peer# [1:7587241524390454833:0] Counter# 431 LastInputSerial# 7018
2025-12-24T02:12:07.720220Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241524390454842:2097] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:07.720358Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:07.720378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241524390454840:2048] [node 2] ICS09 handshake done sender: [1:7587241524390454842:2097] self: [1:7587241524390454833:0] peer: [2:7587241524014587181:0] socket: 16 qp: 27378
2025-12-24T02:12:07.720802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241524390454840:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:07.720825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:07.730538Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241524014587186:2104] held: no
2025-12-24T02:12:07.730617Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241524014587186:2104] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:07.730681Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:07.730712Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241524014587183:2048] [node 1] ICS09 handshake done sender: [2:7587241524014587186:2104] self: [2:7587241524014587181:0] peer: [1:7587241524390454833:0] socket: 21 qp: 27379
2025-12-24T02:12:07.731098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241524014587183:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:07.731124Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:07.845692Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:07.845756Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-12-24T02:12:07.845764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241524390454840:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:07.845769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241524390454840:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:07.845780Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:07.845815Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:07.845907Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241524390454840:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:07.845915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241524390454840:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:07.847326Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:07.847413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241524014587183:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:07.847425Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241524014587183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:07.847465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:07.847495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241524014587183:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:07.851771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241524014587183:2048] [node 1] ICS08 incoming handshake Self# [1:7587241524390454833:0] Peer# [2:7587241524014587181:0] Counter# 9247 LastInputSerial# 535
2025-12-24T02:12:07.853922Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241524390454847:2099] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:07.854065Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:07.854081Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241524390454840:2048] [node 2] ICS09 handshake done sender: [1:7587241524390454847:2099] self: [1:7587241524390454833:0] peer: [2:7587241524014587181:0] socket: 17 qp: 27542
2025-12-24T02:12:07.854451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241524390454840:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:07.854474Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:07.858123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241524390454840:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:07.858139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241524390454840:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:07.858170Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:07.858189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241524390454840:2048] [node 2] ICS08 incoming handshake Self# [2:7587241524014587181:0] Peer# [1:7587241524390454833:0] Counter# 535 LastInputSerial# 9247
2025-12-24T02:12:07.858205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:07.858282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241524390454840:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:07.863991Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241524014587191:2107] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:07.864164Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:07.864183Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241524014587183:2048] [node 1] ICS09 handshake done sender: [2:7587241524014587191:2107] self: [2:7587241524014587181:0] peer: [1:7587241524390454833:0] socket: 20 qp: 27546
2025-12-24T02:12:07.864565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241524014587183:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:07.864583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:07.864716Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:07.864724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241524014587183:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:07.864728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241524014587183:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:07.864737Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:07.864805Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:07.864850Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241524014587183:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:07.864856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241524014587183:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:07.868200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241524390454840:2048] [node 2] ICS08 incoming handshake Self# [2:7587241524014587181:0] Peer# [1:7587241524390454833:0] Counter# 536 LastInputSerial# 9247
2025-12-24T02:12:07.868814Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241524014587196:2109] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:07.868906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:07.868919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241524014587183:2048] [node 1] ICS09 handshake done sender: [2:7587241524014587196:2109] self: [2:7587241524014587181:0] peer: [1:7587241524390454833:0] socket: 20 qp: 27565
2025-12-24T02:12:07.869323Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241524014587183:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:07.869343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:07.870715Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241524390454853:2102] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:07.870784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:07.870802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241524390454840:2048] [node 2] ICS09 handshake done sender: [1:7587241524390454853:2102] self: [1:7587241524390454833:0] peer: [2:7587241524014587181:0] socket: 22 qp: 27568
2025-12-24T02:12:07.871095Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241524390454840:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:07.871109Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:08.295702Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:08.295797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241524390454840:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-12-24T02:12:08.295810Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241524390454840:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:08.295840Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:08.295859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241524390454840:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:08.297822Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:08.297932Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241524014587183:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:08.297944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241524014587183:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:08.297989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:08.298021Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241524014587183:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:08.309512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241524014587183:2048] [node 1] ICS08 incoming handshake Self# [1:7587241524390454833:0] Peer# [2:7587241524014587181:0] Counter# 16342 LastInputSerial# 955
2025-12-24T02:12:08.318927Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241528685422152:2104] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:08.319104Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:08.319120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241524390454840:2048] [node 2] ICS09 handshake done sender: [1:7587241528685422152:2104] self: [1:7587241524390454833:0] peer: [2:7587241524014587181:0] socket: 16 qp: 27981
2025-12-24T02:12:08.319500Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241524390454840:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:08.319525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:08.319570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241524390454840:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:08.319582Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241524390454840:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:08.319632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:08.319654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241524390454840:2048] [node 2] ICS08 incoming handshake Self# [2:7587241524014587181:0] Peer# [1:7587241524390454833:0] Counter# 955 LastInputSerial# 16342
2025-12-24T02:12:08.319683Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:08.319752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241524390454840:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:08.319821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241524390454840:2048] VirtualId# [1:7587241524390454833:0]
2025-12-24T02:12:08.319834Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:08.346116Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241528309554497:2111] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:08.346322Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:08.346339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241524014587183:2048] [node 1] ICS09 handshake done sender: [2:7587241528309554497:2111] self: [2:7587241524014587181:0] peer: [1:7587241524390454833:0] socket: 17 qp: 27984
2025-12-24T02:12:08.346721Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241524014587183:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:08.346736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:08.346866Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:08.346874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241524014587183:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:08.346880Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241524014587183:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:08.346888Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:08.346927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:08.346989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241524014587183:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:08.346992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241524014587183:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:08.354675Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241528685422158:2107]) peer tries to resume nonexistent session Self# [2:7587241524014587181:0] Peer# [1:7587241524390454833:0]
2025-12-24T02:12:08.354728Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241528685422158:2107] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:08.354812Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241528309554502:2113] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:08.355377Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:08.355421Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:08.355433Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241524014587183:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:08.355462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241524014587183:2048] VirtualId# [2:7587241524014587181:0]
2025-12-24T02:12:08.355513Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 19999
TEvUndelivered Cookie# 20000
TEvUndelivered Cookie# 20001
TEvUndelivered Cookie# 20002
TEvUndelivered Cookie# 20003
TEvUndelivered Cookie# 20004
TEvUndelivered Cookie# 20005
TEvUndelivered Cookie# 20006
TEvUndelivered Cookie# 20007
2025-12-24T02:12:08.367012Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241528685422159:2108] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:08.367071Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:08.367718Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241528309554505:2114] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:08.375238Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241528685422159:2108] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:08.375319Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241528309554505:2114] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:08.375359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:08.375380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:08.375390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241528685422162:2048]
2025-12-24T02:12:08.375401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241528685422159:2108] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 17 qp: 28034
2025-12-24T02:12:08.375411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:08.375430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:08.375462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:08.375475Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:08.375482Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241528309554508:2048]
2025-12-24T02:12:08.375489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241528309554505:2114] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 16 qp: 28033
2025-12-24T02:12:08.375495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:08.375505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:08.659819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:08.660285Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-12-24T02:12:08.660389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 17 reason# ECONNRESET
2025-12-24T02:12:08.660395Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-12-24T02:12:08.660403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:08.660437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:08.660201Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:08.660217Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241528309554508:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:08.660224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:08.660235Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:08.660291Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:08.660379Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:08.660381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:08.708677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 2458 LastInputSerial# 119
2025-12-24T02:12:08.714609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 119 LastInputSerial# 2458
2025-12-24T02:12:08.714857Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241528685422164:2110] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:08.714991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:08.715015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241528685422164:2110] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 23 qp: 28224
2025-12-24T02:12:08.715362Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:08.715393Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:08.718574Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241528309554512:2116] held: no
2025-12-24T02:12:08.718607Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241528309554512:2116] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:08.718712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:08.718729Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241528309554512:2116] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 25 qp: 28228
2025-12-24T02:12:08.719178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:08.719199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:08.864378Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:08.864455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:08.864485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241528309554508:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:08.864491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:08.864503Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:08.864537Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:08.864631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:08.864638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:08.865180Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:08.865264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:08.865270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:08.865327Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:08.865359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:08.870199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 4788 LastInputSerial# 261
2025-12-24T02:12:08.871456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 261 LastInputSerial# 4788
2025-12-24T02:12:08.871663Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241528685422169:2112] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:08.871802Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:08.871825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241528685422169:2112] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 20 qp: 28354
2025-12-24T02:12:08.872474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:08.872500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:08.880408Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241528309554517:2119] held: no
2025-12-24T02:12:08.880442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241528309554517:2119] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:08.880495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:08.880511Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241528309554517:2119] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 21 qp: 28355
2025-12-24T02:12:08.880815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:08.880837Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:09.160034Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:09.160056Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241528309554508:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:09.160063Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:09.160079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:09.160109Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:09.160216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:09.160227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:09.159019Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:09.164914Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-12-24T02:12:09.165026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 20 reason# ECONNRESET
2025-12-24T02:12:09.165038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-12-24T02:12:09.165054Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:09.165104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:09.184415Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 378 LastInputSerial# 7827
2025-12-24T02:12:09.190609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 7827 LastInputSerial# 378
2025-12-24T02:12:09.194310Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241532980389470:2114] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:09.194914Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:09.194943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241532980389470:2114] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 23 qp: 28534
2025-12-24T02:12:09.195302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:09.195326Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:09.197577Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241532604521818:2122] held: no
2025-12-24T02:12:09.197609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241532604521818:2122] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:09.197685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:09.197701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241532604521818:2122] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 25 qp: 28531
2025-12-24T02:12:09.198589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:09.198617Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:09.352366Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:09.352439Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:09.352448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:09.352481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:09.352503Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:09.352223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:09.352324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 23 reason# Debug
2025-12-24T02:12:09.352330Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:09.352360Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:09.352373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:09.368630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 442 LastInputSerial# 9576
2025-12-24T02:12:09.385289Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 9576 LastInputSerial# 442
2025-12-24T02:12:09.385645Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241532980389475:2116] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:09.391434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:09.391466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241532980389475:2116] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 16 qp: 28605
2025-12-24T02:12:09.391911Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:09.391939Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:09.399490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241532604521823:2125] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:09.399601Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241532604521823:2125] held: no
2025-12-24T02:12:09.399617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:09.399627Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241532604521823:2125] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 21 qp: 28615
2025-12-24T02:12:09.399927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:09.399952Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:09.822534Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:09.822582Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:09.822681Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:09.822715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241528309554508:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:09.822722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:09.822733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:09.822767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:09.822770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:09.822881Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:09.823064Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:09.823077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:09.823088Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:09.823117Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:09.830551Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 760 LastInputSerial# 15306
2025-12-24T02:12:09.835358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 15306 LastInputSerial# 760
2025-12-24T02:12:09.835654Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241532980389480:2118] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:09.835784Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:09.835806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241532980389480:2118] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 23 qp: 28860
2025-12-24T02:12:09.836299Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:09.836323Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:09.838754Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241532604521828:2128] held: no
2025-12-24T02:12:09.838806Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241532604521828:2128] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:09.838945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:09.838966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241532604521828:2128] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 25 qp: 28856
2025-12-24T02:12:09.839518Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:09.839539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:10.072057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:10.072194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:10.072206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241528309554508:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:10.072214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:10.072224Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:10.072194Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:10.072314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:10.072325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:10.072263Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:10.072340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241528309554508:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:10.072343Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241528309554508:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:10.072380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:10.072418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:10.075112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241528685422162:2048] [node 2] ICS08 incoming handshake Self# [2:7587241528309554504:0] Peer# [1:7587241528685422160:0] Counter# 923 LastInputSerial# 18599
2025-12-24T02:12:10.075377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241528309554508:2048] [node 1] ICS08 incoming handshake Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0] Counter# 18599 LastInputSerial# 923
2025-12-24T02:12:10.075604Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241537275356781:2120] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:10.075770Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:10.075787Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241528685422162:2048] [node 2] ICS09 handshake done sender: [1:7587241537275356781:2120] self: [1:7587241528685422160:0] peer: [2:7587241528309554504:0] socket: 17 qp: 29028
2025-12-24T02:12:10.076050Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241528685422162:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:10.076075Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:10.078514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241536899489129:2131] held: no
2025-12-24T02:12:10.078549Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241536899489129:2131] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:10.078732Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:10.078753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241528309554508:2048] [node 1] ICS09 handshake done sender: [2:7587241536899489129:2131] self: [2:7587241528309554504:0] peer: [1:7587241528685422160:0] socket: 21 qp: 29029
2025-12-24T02:12:10.079381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241528309554508:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:10.079408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:10.382592Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:10.382434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:10.382501Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:10.382581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241528309554508:2048] [node 1] ICS01 socket: 21 reason# EndOfStream
2025-12-24T02:12:10.382621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241528309554508:2048] VirtualId# [2:7587241528309554504:0]
2025-12-24T02:12:10.382629Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241528309554508:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:10.382642Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:10.382673Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:10.382939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241528685422162:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:10.382957Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241528685422162:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:10.382968Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:10.382992Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241528685422162:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:10.385210Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241536899489132:2133]) peer tries to resume nonexistent session Self# [1:7587241528685422160:0] Peer# [2:7587241528309554504:0]
2025-12-24T02:12:10.385243Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241536899489132:2133] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:10.385321Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241537275356786:2122] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:10.386071Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:10.386127Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:10.386139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241528685422162:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:10.386169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241528685422162:2048] VirtualId# [1:7587241528685422160:0]
2025-12-24T02:12:10.386199Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:10.388919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241536899489133:2134] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:10.388982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:10.389156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241537275356789:2123] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:10.393576Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241536899489133:2134] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:10.393698Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:10.393718Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:10.393662Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241537275356789:2123] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:10.393728Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241536899489136:2048]
2025-12-24T02:12:10.393739Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241536899489136:2048] [node 1] ICS09 handshake done sender: [2:7587241536899489133:2134] self: [2:7587241536899489134:0] peer: [1:7587241537275356788:0] socket: 17 qp: 29286
2025-12-24T02:12:10.393749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241536899489136:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:10.393762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:10.393805Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:10.393827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:10.393836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241537275356792:2048]
2025-12-24T02:12:10.393851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241537275356792:2048] [node 2] ICS09 handshake done sender: [1:7587241537275356789:2123] self: [1:7587241537275356788:0] peer: [2:7587241536899489134:0] socket: 16 qp: 29285
2025-12-24T02:12:10.393859Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241537275356792:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:10.393869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:10.806857Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:10.806963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241537275356792:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:10.807014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241537275356792:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:10.807048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:10.807079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241537275356792:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:10.807064Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:10.807154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241536899489136:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:10.807211Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241536899489136:2048] VirtualId# [2:7587241536899489134:0]
2025-12-24T02:12:10.807226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241536899489136:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:10.807276Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:10.807310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:10.809644Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241536899489138:2136]) peer tries to resume nonexistent session Self# [1:7587241537275356788:0] Peer# [2:7587241536899489134:0]
2025-12-24T02:12:10.809669Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241536899489138:2136] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:10.809745Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241537275356794:2124] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:10.810459Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:10.810503Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:10.810514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241537275356792:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:10.810556Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241537275356792:2048] VirtualId# [1:7587241537275356788:0]
2025-12-24T02:12:10.810594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:10.814552Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241536899489139:2137] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:10.814613Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:10.814840Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241537275356797:2125] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:10.817330Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241536899489139:2137] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:10.817408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:10.817426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:10.817398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241537275356797:2125] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:10.817434Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241536899489142:2048]
2025-12-24T02:12:10.817440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241536899489142:2048] [node 1] ICS09 handshake done sender: [2:7587241536899489139:2137] self: [2:7587241536899489140:0] peer: [1:7587241537275356796:0] socket: 17 qp: 29597
2025-12-24T02:12:10.817448Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241536899489142:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:10.817460Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:10.817593Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:10.817615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:10.817623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241537275356800:2048]
2025-12-24T02:12:10.817630Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241537275356800:2048] [node 2] ICS09 handshake done sender: [1:7587241537275356797:2125] self: [1:7587241537275356796:0] peer: [2:7587241536899489140:0] socket: 16 qp: 29596
2025-12-24T02:12:10.817638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241537275356800:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:10.817647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:11.146868Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:11.146923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:11.147023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241536899489142:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:11.147074Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241536899489142:2048] VirtualId# [2:7587241536899489140:0]
2025-12-24T02:12:11.147087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241536899489142:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:11.147097Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:11.147121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:11.147294Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:11.147459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241537275356800:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:11.147471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241537275356800:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:11.147482Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:11.147519Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241537275356800:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:11.149614Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241541194456440:2139]) peer tries to resume nonexistent session Self# [1:7587241537275356796:0] Peer# [2:7587241536899489140:0]
2025-12-24T02:12:11.149638Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241541194456440:2139] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:11.149718Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241541570324098:2126] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:11.150386Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:11.150421Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:11.150431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241537275356800:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:11.150462Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241537275356800:2048] VirtualId# [1:7587241537275356796:0]
2025-12-24T02:12:11.150481Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:11.154159Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241541194456441:2140] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.154219Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:11.154516Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241541570324101:2127] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.184187Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241541194456441:2140] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:11.184291Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241541570324101:2127] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:11.184354Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:11.184379Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:11.184396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241541194456444:2048]
2025-12-24T02:12:11.184410Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241541194456444:2048] [node 1] ICS09 handshake done sender: [2:7587241541194456441:2140] self: [2:7587241541194456442:0] peer: [1:7587241541570324100:0] socket: 17 qp: 29885
2025-12-24T02:12:11.184421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241541194456444:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:11.184441Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:11.184497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:11.184525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:11.184533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241541570324104:2048]
2025-12-24T02:12:11.184540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241541570324104:2048] [node 2] ICS09 handshake done sender: [1:7587241541570324101:2127] self: [1:7587241541570324100:0] peer: [2:7587241541194456442:0] socket: 16 qp: 29886
2025-12-24T02:12:11.184548Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241541570324104:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:11.184558Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:11.626385Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241541194456444:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:11.626450Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241541194456444:2048] VirtualId# [2:7587241541194456442:0]
2025-12-24T02:12:11.626459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241541194456444:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:11.626494Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:11.626530Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:11.626617Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:11.626738Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241541570324104:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:11.626752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241541570324104:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:11.626799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:11.626829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241541570324104:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 29300
2025-12-24T02:12:11.645766Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241541194456446:2142]) peer tries to resume nonexistent session Self# [1:7587241541570324100:0] Peer# [2:7587241541194456442:0]
2025-12-24T02:12:11.645803Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241541194456446:2142] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:11.645902Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241541570324106:2128] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:11.646611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:11.646683Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:11.646716Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241541570324104:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:11.646776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241541570324104:2048] VirtualId# [1:7587241541570324100:0]
2025-12-24T02:12:11.646822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:11.708687Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241541570324110:2130] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.708714Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241541194456450:2144] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.708763Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:11.708990Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241541570324109:2129] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
Subscribe
2025-12-24T02:12:11.755680Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241541194456450:2144] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:11.755736Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241541570324109:2129] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:11.755790Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:11.755810Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:11.755820Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241541194456453:2048]
2025-12-24T02:12:11.755828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241541194456453:2048] [node 1] ICS09 handshake done sender: [2:7587241541194456450:2144] self: [2:7587241541194456451:0] peer: [1:7587241541570324108:0] socket: 21 qp: 30377
2025-12-24T02:12:11.755837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241541194456453:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:11.755852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:11.755867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:11.755886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:11.755920Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241541570324114:2048]
2025-12-24T02:12:11.755931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241541570324114:2048] [node 2] ICS09 handshake done sender: [1:7587241541570324109:2129] self: [1:7587241541570324108:0] peer: [2:7587241541194456451:0] socket: 20 qp: 30376
2025-12-24T02:12:11.755941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241541570324114:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:11.755947Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:11.945109Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241541570324114:2048] [node 2] ICS01 socket: 20 reason# Debug
2025-12-24T02:12:11.945165Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241541570324114:2048] VirtualId# [1:7587241541570324108:0]
2025-12-24T02:12:11.945175Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241541570324114:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:11.945234Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:11.945267Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:11.945283Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:11.945418Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:11.945430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241541194456453:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:11.945437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241541194456453:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:11.945454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:11.945502Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241541194456453:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:11.945510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241541194456453:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:11.947846Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241541194456455:2146] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:11.947735Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241541570324116:2131]) peer tries to resume nonexistent session Self# [2:7587241541194456451:0] Peer# [1:7587241541570324108:0]
2025-12-24T02:12:11.947760Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241541570324116:2131] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:11.948514Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:11.948557Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:11.948566Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241541194456453:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:11.948597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241541194456453:2048] VirtualId# [2:7587241541194456451:0]
2025-12-24T02:12:11.948634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 30201
TEvUndelivered Cookie# 30202
TEvUndelivered Cookie# 30203
TEvUndelivered Cookie# 30204
TEvUndelivered Cookie# 30205
TEvUndelivered Cookie# 30206
2025-12-24T02:12:11.950870Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241541570324117:2132] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.950918Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:11.951051Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241541194456458:2147] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:11.957747Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241541570324117:2132] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:11.957834Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241541194456458:2147] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:11.957873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:11.957896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:11.957905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241541570324120:2048]
2025-12-24T02:12:11.957913Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241541570324120:2048] [node 2] ICS09 handshake done sender: [1:7587241541570324117:2132] self: [1:7587241541570324118:0] peer: [2:7587241541194456457:0] socket: 17 qp: 30600
2025-12-24T02:12:11.957923Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241541570324120:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:11.957934Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:11.958018Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:11.958041Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:11.958049Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241541194456461:2048]
2025-12-24T02:12:11.958055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241541194456461:2048] [node 1] ICS09 handshake done sender: [2:7587241541194456458:2147] self: [2:7587241541194456457:0] peer: [1:7587241541570324118:0] socket: 16 qp: 30599
2025-12-24T02:12:11.958062Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241541194456461:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:11.958077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:12.450218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241541194456461:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:12.450289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241541194456461:2048] VirtualId# [2:7587241541194456457:0]
2025-12-24T02:12:12.450301Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241541194456461:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:12.450319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:12.450343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:12.450575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:12.450657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241541570324120:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:12.450674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241541570324120:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:12.450742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:12.450765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241541570324120:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:12.453466Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241545489423759:2148]) peer tries to resume nonexistent session Self# [1:7587241541570324118:0] Peer# [2:7587241541194456457:0]
2025-12-24T02:12:12.453489Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241545489423759:2148] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:12.453580Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241545865291418:2134] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:12.454326Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:12.454369Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:12.454376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241541570324120:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:12.454408Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241541570324120:2048] VirtualId# [1:7587241541570324118:0]
2025-12-24T02:12:12.454451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:12.478816Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241545489423760:2149] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:12.478887Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:12.479087Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241545865291421:2135] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:12.490280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241545489423760:2149] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:12.490406Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:12.490410Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241545865291421:2135] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:12.490453Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:12.490469Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241545489423763:2048]
2025-12-24T02:12:12.490480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241545489423763:2048] [node 1] ICS09 handshake done sender: [2:7587241545489423760:2149] self: [2:7587241545489423761:0] peer: [1:7587241545865291420:0] socket: 17 qp: 31042
2025-12-24T02:12:12.490492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241545489423763:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:12.490505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:12.490597Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:12.490619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:12.490631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241545865291424:2048]
2025-12-24T02:12:12.490639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241545865291424:2048] [node 2] ICS09 handshake done sender: [1:7587241545865291421:2135] self: [1:7587241545865291420:0] peer: [2:7587241545489423761:0] socket: 16 qp: 31041
2025-12-24T02:12:12.490657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241545865291424:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:12.490675Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:12.764331Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:12.764432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241545865291424:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:12.764444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241545865291424:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:12.764477Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:12.764493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241545865291424:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:12.764726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:12.764811Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241545489423763:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:12.764822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241545489423763:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:12.764869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:12.764904Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241545489423763:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:12.790187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241545489423763:2048] [node 1] ICS08 incoming handshake Self# [1:7587241545865291420:0] Peer# [2:7587241545489423761:0] Counter# 3106 LastInputSerial# 151
2025-12-24T02:12:12.798477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241545865291424:2048] [node 2] ICS08 incoming handshake Self# [2:7587241545489423761:0] Peer# [1:7587241545865291420:0] Counter# 151 LastInputSerial# 3106
2025-12-24T02:12:12.798716Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241545865291426:2136] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:12.798836Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:12.798858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241545865291424:2048] [node 2] ICS09 handshake done sender: [1:7587241545865291426:2136] self: [1:7587241545865291420:0] peer: [2:7587241545489423761:0] socket: 22 qp: 31339
2025-12-24T02:12:12.799205Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241545865291424:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:12.799220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:12.806062Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241545489423766:2152] held: no
2025-12-24T02:12:12.806097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241545489423766:2152] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:12.806169Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:12.806187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241545489423763:2048] [node 1] ICS09 handshake done sender: [2:7587241545489423766:2152] self: [2:7587241545489423761:0] peer: [1:7587241545865291420:0] socket: 23 qp: 31342
2025-12-24T02:12:12.806529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241545489423763:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:12.806554Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:13.141440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:13.141968Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:13.142090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241545865291424:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-12-24T02:12:13.142103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241545865291424:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:13.142118Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:13.142148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241545865291424:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:13.142212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:13.142473Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241545489423763:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:13.142482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241545489423763:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:13.142496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:13.142529Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241545489423763:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:13.146549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241545489423763:2048] [node 1] ICS08 incoming handshake Self# [1:7587241545865291420:0] Peer# [2:7587241545489423761:0] Counter# 7609 LastInputSerial# 428
2025-12-24T02:12:13.155493Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241545865291424:2048] [node 2] ICS08 incoming handshake Self# [2:7587241545489423761:0] Peer# [1:7587241545865291420:0] Counter# 428 LastInputSerial# 7609
2025-12-24T02:12:13.155714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241550160258727:2138] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:13.155832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:13.155853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241545865291424:2048] [node 2] ICS09 handshake done sender: [1:7587241550160258727:2138] self: [1:7587241545865291420:0] peer: [2:7587241545489423761:0] socket: 16 qp: 31638
2025-12-24T02:12:13.156203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241545865291424:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:13.156230Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:13.159131Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241549784391067:2155] held: no
2025-12-24T02:12:13.159165Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241549784391067:2155] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:13.159274Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:13.159290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241545489423763:2048] [node 1] ICS09 handshake done sender: [2:7587241549784391067:2155] self: [2:7587241545489423761:0] peer: [1:7587241545865291420:0] socket: 17 qp: 31639
2025-12-24T02:12:13.159920Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241545489423763:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:13.159951Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:13.596530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241545489423763:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:13.596578Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241545489423763:2048] VirtualId# [2:7587241545489423761:0]
2025-12-24T02:12:13.596587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241545489423763:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:13.596628Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:13.596654Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:13.596712Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:13.596889Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241545865291424:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:13.596900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241545865291424:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:13.596910Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:13.596941Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241545865291424:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:13.598880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241549784391071:2157]) peer tries to resume nonexistent session Self# [1:7587241545865291420:0] Peer# [2:7587241545489423761:0]
2025-12-24T02:12:13.598902Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241549784391071:2157] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:13.598990Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241550160258732:2140] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:13.599660Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:13.599697Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:13.599705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241545865291424:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:13.599726Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241545865291424:2048] VirtualId# [1:7587241545865291420:0]
2025-12-24T02:12:13.599749Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:13.601683Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241549784391072:2158] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:13.601742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:13.601867Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241550160258735:2141] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:13.604184Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241549784391072:2158] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:13.604255Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241550160258735:2141] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:13.604319Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:13.604335Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:13.604341Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241549784391075:2048]
2025-12-24T02:12:13.604346Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241549784391075:2048] [node 1] ICS09 handshake done sender: [2:7587241549784391072:2158] self: [2:7587241549784391073:0] peer: [1:7587241550160258734:0] socket: 17 qp: 32084
2025-12-24T02:12:13.604352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241549784391075:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:13.604362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:13.604410Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:13.604430Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:13.604437Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241550160258738:2048]
2025-12-24T02:12:13.604444Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241550160258738:2048] [node 2] ICS09 handshake done sender: [1:7587241550160258735:2141] self: [1:7587241550160258734:0] peer: [2:7587241549784391073:0] socket: 16 qp: 32082
2025-12-24T02:12:13.604451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241550160258738:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:13.604464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:13.964138Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:13.964185Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:13.964192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241549784391075:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:13.964207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241549784391075:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:13.964218Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:13.964239Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:13.964257Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:13.964333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241549784391075:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:13.964339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241549784391075:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:13.964339Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241550160258738:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:13.964350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241550160258738:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:13.964394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:13.964422Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241550160258738:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:13.990420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241549784391075:2048] [node 1] ICS08 incoming handshake Self# [1:7587241550160258734:0] Peer# [2:7587241549784391073:0] Counter# 3929 LastInputSerial# 231
2025-12-24T02:12:14.002105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241550160258738:2048] [node 2] ICS08 incoming handshake Self# [2:7587241549784391073:0] Peer# [1:7587241550160258734:0] Counter# 231 LastInputSerial# 3929
2025-12-24T02:12:14.002276Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241550160258740:2142] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:14.002387Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:14.002403Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241550160258738:2048] [node 2] ICS09 handshake done sender: [1:7587241550160258740:2142] self: [1:7587241550160258734:0] peer: [2:7587241549784391073:0] socket: 22 qp: 32426
2025-12-24T02:12:14.002666Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241550160258738:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:14.002685Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:14.004811Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241549784391078:2161] held: no
2025-12-24T02:12:14.004843Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241549784391078:2161] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:14.004905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:14.004919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241549784391075:2048] [node 1] ICS09 handshake done sender: [2:7587241549784391078:2161] self: [2:7587241549784391073:0] peer: [1:7587241550160258734:0] socket: 23 qp: 32427
2025-12-24T02:12:14.005294Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241549784391075:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:14.005314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:14.135205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:14.135305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241550160258738:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-12-24T02:12:14.135318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241550160258738:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:14.135345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:14.135370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241550160258738:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:14.136585Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:14.136813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241549784391075:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:14.136825Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241549784391075:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:14.136836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:14.136933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241549784391075:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:14.139187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241549784391075:2048] [node 1] ICS08 incoming handshake Self# [1:7587241550160258734:0] Peer# [2:7587241549784391073:0] Counter# 6266 LastInputSerial# 344
2025-12-24T02:12:14.140078Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241554455226041:2144] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:14.140241Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:14.140260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241550160258738:2048] [node 2] ICS09 handshake done sender: [1:7587241554455226041:2144] self: [1:7587241550160258734:0] peer: [2:7587241549784391073:0] socket: 16 qp: 32581
2025-12-24T02:12:14.140678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241550160258738:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:14.140703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:14.141077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241550160258738:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:14.141091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241550160258738:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:14.141144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:14.141172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241550160258738:2048] [node 2] ICS08 incoming handshake Self# [2:7587241549784391073:0] Peer# [1:7587241550160258734:0] Counter# 344 LastInputSerial# 6266
2025-12-24T02:12:14.141188Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:14.141270Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241550160258738:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:14.141299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241550160258738:2048] VirtualId# [1:7587241550160258734:0]
2025-12-24T02:12:14.141314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:14.142639Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241554079358379:2164] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:14.142832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:14.142853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241549784391075:2048] [node 1] ICS09 handshake done sender: [2:7587241554079358379:2164] self: [2:7587241549784391073:0] peer: [1:7587241550160258734:0] socket: 17 qp: 32582
2025-12-24T02:12:14.143303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241549784391075:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:14.143319Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:14.143455Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:14.143462Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241549784391075:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:14.143466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241549784391075:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:14.143471Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:14.143496Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:14.143536Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241549784391075:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:14.143542Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241549784391075:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:14.146124Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241554455226047:2147]) peer tries to resume nonexistent session Self# [2:7587241549784391073:0] Peer# [1:7587241550160258734:0]
2025-12-24T02:12:14.146160Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241554455226047:2147] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:14.146259Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241554079358384:2166] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:14.147252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:14.147305Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:14.147310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241549784391075:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:14.147333Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241549784391075:2048] VirtualId# [2:7587241549784391073:0]
2025-12-24T02:12:14.147380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 36283
TEvUndelivered Cookie# 36284
TEvUndelivered Cookie# 36285
TEvUndelivered Cookie# 36286
TEvUndelivered Cookie# 36287
2025-12-24T02:12:14.154993Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241554455226048:2148] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:14.155060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:14.155269Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241554079358387:2167] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:14.159031Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241554455226048:2148] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:14.159090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241554079358387:2167] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:14.159134Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:14.159155Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:14.159168Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241554455226051:2048]
2025-12-24T02:12:14.159178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241554455226051:2048] [node 2] ICS09 handshake done sender: [1:7587241554455226048:2148] self: [1:7587241554455226049:0] peer: [2:7587241554079358386:0] socket: 17 qp: 32597
2025-12-24T02:12:14.159197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241554455226051:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:14.159212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:14.159243Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:14.159262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:14.159272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241554079358390:2048]
2025-12-24T02:12:14.159279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241554079358390:2048] [node 1] ICS09 handshake done sender: [2:7587241554079358387:2167] self: [2:7587241554079358386:0] peer: [1:7587241554455226049:0] socket: 16 qp: 32596
2025-12-24T02:12:14.159287Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241554079358390:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:14.159295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:14.264853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241554455226051:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:14.265021Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241554455226051:2048] VirtualId# [1:7587241554455226049:0]
2025-12-24T02:12:14.265036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241554455226051:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:14.265079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:14.265116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:14.265110Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:14.265221Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241554079358390:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:14.265264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241554079358390:2048] VirtualId# [2:7587241554079358386:0]
2025-12-24T02:12:14.265275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241554079358390:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:14.265331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:14.265365Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:14.268325Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241554079358392:2168] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:14.268383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:14.268563Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241554455226054:2150] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:14.271030Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241554079358392:2168] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:14.271097Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241554455226054:2150] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:14.271198Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:14.271222Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:14.271231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241554079358395:2048]
2025-12-24T02:12:14.271237Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241554079358395:2048] [node 1] ICS09 handshake done sender: [2:7587241554079358392:2168] self: [2:7587241554079358393:0] peer: [1:7587241554455226053:0] socket: 16 qp: 32706
2025-12-24T02:12:14.271247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241554079358395:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:14.271258Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:14.271314Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:14.271331Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:14.271339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241554455226057:2048]
2025-12-24T02:12:14.271350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241554455226057:2048] [node 2] ICS09 handshake done sender: [1:7587241554455226054:2150] self: [1:7587241554455226053:0] peer: [2:7587241554079358393:0] socket: 17 qp: 32705
2025-12-24T02:12:14.271357Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241554455226057:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:14.271371Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:14.497279Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:14.497358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241554455226057:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-12-24T02:12:14.497367Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241554455226057:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:14.497403Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:14.497428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241554455226057:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:14.499789Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:14.499912Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241554079358395:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:14.499924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241554079358395:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:14.499969Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:14.500004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241554079358395:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:14.503228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241554079358395:2048] [node 1] ICS08 incoming handshake Self# [1:7587241554455226053:0] Peer# [2:7587241554079358393:0] Counter# 1899 LastInputSerial# 46
2025-12-24T02:12:14.503244Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241554455226057:2048] [node 2] ICS08 incoming handshake Self# [2:7587241554079358393:0] Peer# [1:7587241554455226053:0] Counter# 46 LastInputSerial# 1899
2025-12-24T02:12:14.503436Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241554455226059:2151] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:14.503533Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:14.503554Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241554455226057:2048] [node 2] ICS09 handshake done sender: [1:7587241554455226059:2151] self: [1:7587241554455226053:0] peer: [2:7587241554079358393:0] socket: 22 qp: 32967
2025-12-24T02:12:14.503881Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241554455226057:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:14.503898Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:14.505930Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241554079358398:2171] held: no
2025-12-24T02:12:14.505959Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241554079358398:2171] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:14.506009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:14.506025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241554079358395:2048] [node 1] ICS09 handshake done sender: [2:7587241554079358398:2171] self: [2:7587241554079358393:0] peer: [1:7587241554455226053:0] socket: 23 qp: 32976
2025-12-24T02:12:14.506482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241554079358395:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:14.506507Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:15.010004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241554079358395:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-12-24T02:12:15.010046Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241554079358395:2048] VirtualId# [2:7587241554079358393:0]
2025-12-24T02:12:15.010055Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241554079358395:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:15.010082Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.010116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:15.010130Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:15.010246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241554455226057:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-12-24T02:12:15.010258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241554455226057:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.010310Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.010354Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241554455226057:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:15.013012Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241558374325698:2173]) peer tries to resume nonexistent session Self# [1:7587241554455226053:0] Peer# [2:7587241554079358393:0]
2025-12-24T02:12:15.013040Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241558374325698:2173] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:15.013121Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241558750193360:2153] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:15.013795Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:15.013836Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:15.013841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241554455226057:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:15.013871Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241554455226057:2048] VirtualId# [1:7587241554455226053:0]
2025-12-24T02:12:15.013916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.017828Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325699:2174] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.017903Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.018117Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193363:2154] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.028594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325699:2174] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.028664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193363:2154] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.028742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.028766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.028780Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325702:2048]
2025-12-24T02:12:15.028792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325702:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325699:2174] self: [2:7587241558374325700:0] peer: [1:7587241558750193362:0] socket: 17 qp: 33620
2025-12-24T02:12:15.028803Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325702:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.028821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:15.028850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.028864Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.028876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193366:2048]
2025-12-24T02:12:15.028885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193366:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193363:2154] self: [1:7587241558750193362:0] peer: [2:7587241558374325700:0] socket: 16 qp: 33619
2025-12-24T02:12:15.028896Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193366:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.028907Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:15.225680Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:15.225764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241558750193366:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:15.225773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193366:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:15.225798Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.225826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241558750193366:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:15.227510Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.227608Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325702:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:15.227621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325702:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.227662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.227693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325702:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.228774Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241558374325702:2048] [node 1] ICS08 incoming handshake Self# [1:7587241558750193362:0] Peer# [2:7587241558374325700:0] Counter# 1752 LastInputSerial# 45
2025-12-24T02:12:15.229093Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193368:2155] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.229249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.229269Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193366:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193368:2155] self: [1:7587241558750193362:0] peer: [2:7587241558374325700:0] socket: 22 qp: 33870
2025-12-24T02:12:15.229737Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193366:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.229762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.230964Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241558750193366:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:15.230976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193366:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:15.230997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.231012Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241558750193366:2048] [node 2] ICS08 incoming handshake Self# [2:7587241558374325700:0] Peer# [1:7587241558750193362:0] Counter# 45 LastInputSerial# 1752
2025-12-24T02:12:15.231025Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:15.231168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193366:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:15.231193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193366:2048] VirtualId# [1:7587241558750193362:0]
2025-12-24T02:12:15.231208Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.231893Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325704:2176] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.231962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.231978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325702:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325704:2176] self: [2:7587241558374325700:0] peer: [1:7587241558750193362:0] socket: 23 qp: 33874
2025-12-24T02:12:15.232350Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325702:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.232364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:15.232470Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:15.232477Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241558374325702:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:15.232481Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325702:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:15.232487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.232511Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.232552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325702:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:15.232557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325702:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.234463Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241558750193373:2157]) peer tries to resume nonexistent session Self# [2:7587241558374325700:0] Peer# [1:7587241558750193362:0]
2025-12-24T02:12:15.234488Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241558750193373:2157] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:15.234559Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241558374325709:2179] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:15.234992Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:15.235017Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:15.235024Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325702:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:15.235042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325702:2048] VirtualId# [2:7587241558374325700:0]
2025-12-24T02:12:15.235084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 39193
TEvUndelivered Cookie# 39194
TEvUndelivered Cookie# 39195
TEvUndelivered Cookie# 39196
TEvUndelivered Cookie# 39197
TEvUndelivered Cookie# 39198
TEvUndelivered Cookie# 39199
TEvUndelivered Cookie# 39200
2025-12-24T02:12:15.238172Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193374:2158] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.238210Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.238872Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325712:2180] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.240613Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193374:2158] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.240664Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325712:2180] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.240684Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.240704Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.240711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193377:2048]
2025-12-24T02:12:15.240719Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193377:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193374:2158] self: [1:7587241558750193375:0] peer: [2:7587241558374325711:0] socket: 17 qp: 33890
2025-12-24T02:12:15.240726Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193377:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.240736Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.240786Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.240806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.240814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325715:2048]
2025-12-24T02:12:15.240821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325715:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325712:2180] self: [2:7587241558374325711:0] peer: [1:7587241558750193375:0] socket: 16 qp: 33889
2025-12-24T02:12:15.240828Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325715:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.240839Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:15.356832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193377:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:15.356889Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193377:2048] VirtualId# [1:7587241558750193375:0]
2025-12-24T02:12:15.356898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193377:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:15.356932Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.356996Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.356962Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.357107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325715:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:15.357148Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325715:2048] VirtualId# [2:7587241558374325711:0]
2025-12-24T02:12:15.357155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325715:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.357203Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.357240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:15.359707Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325717:2181] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.359766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.359917Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193380:2160] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.362835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325717:2181] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.362882Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193380:2160] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.362945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.362974Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.362984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325720:2048]
2025-12-24T02:12:15.362991Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325720:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325717:2181] self: [2:7587241558374325718:0] peer: [1:7587241558750193379:0] socket: 17 qp: 34045
2025-12-24T02:12:15.362999Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325720:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.362989Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.363004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.363009Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193383:2048]
2025-12-24T02:12:15.363012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:15.363015Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193383:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193380:2160] self: [1:7587241558750193379:0] peer: [2:7587241558374325718:0] socket: 16 qp: 34044
2025-12-24T02:12:15.363020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193383:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.363027Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:15.621312Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193383:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:15.621369Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193383:2048] VirtualId# [1:7587241558750193379:0]
2025-12-24T02:12:15.621378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193383:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:15.621418Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.621451Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.622688Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.622921Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325720:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:15.622933Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325720:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.622944Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.622973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325720:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.624503Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241558750193385:2161]) peer tries to resume nonexistent session Self# [2:7587241558374325718:0] Peer# [1:7587241558750193379:0]
2025-12-24T02:12:15.624611Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241558374325722:2183] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:15.624530Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241558750193385:2161] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:15.625271Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:15.625309Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:15.625314Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325720:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:15.625367Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325720:2048] VirtualId# [2:7587241558374325718:0]
2025-12-24T02:12:15.625410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 39842
TEvUndelivered Cookie# 39843
TEvUndelivered Cookie# 39844
TEvUndelivered Cookie# 39845
TEvUndelivered Cookie# 39846
TEvUndelivered Cookie# 39847
TEvUndelivered Cookie# 39848
TEvUndelivered Cookie# 39849
TEvUndelivered Cookie# 39850
2025-12-24T02:12:15.629270Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193386:2162] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.629339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.629487Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325725:2184] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.633569Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193386:2162] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.633640Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325725:2184] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.633678Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.633698Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.633711Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193389:2048]
2025-12-24T02:12:15.633722Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193389:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193386:2162] self: [1:7587241558750193387:0] peer: [2:7587241558374325724:0] socket: 17 qp: 34359
2025-12-24T02:12:15.633736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193389:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.633754Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.633813Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.633830Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.633838Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325728:2048]
2025-12-24T02:12:15.633845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325728:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325725:2184] self: [2:7587241558374325724:0] peer: [1:7587241558750193387:0] socket: 16 qp: 34358
2025-12-24T02:12:15.633856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325728:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.633865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:15.734533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193389:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:15.734583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193389:2048] VirtualId# [1:7587241558750193387:0]
2025-12-24T02:12:15.734591Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193389:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:15.734624Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.734649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.734835Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.735078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325728:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:15.735096Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325728:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.735113Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.735146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325728:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.737380Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241558750193391:2164]) peer tries to resume nonexistent session Self# [2:7587241558374325724:0] Peer# [1:7587241558750193387:0]
2025-12-24T02:12:15.737407Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241558750193391:2164] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:15.737498Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241558374325730:2185] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:15.738174Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:15.738214Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:15.738220Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325728:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:15.738255Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325728:2048] VirtualId# [2:7587241558374325724:0]
2025-12-24T02:12:15.738311Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 39873
TEvUndelivered Cookie# 39874
TEvUndelivered Cookie# 39875
TEvUndelivered Cookie# 39876
TEvUndelivered Cookie# 39877
TEvUndelivered Cookie# 39878
TEvUndelivered Cookie# 39879
TEvUndelivered Cookie# 39880
2025-12-24T02:12:15.742442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193392:2165] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.742511Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.742754Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325733:2186] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.746554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193392:2165] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.746685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.746672Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325733:2186] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.746721Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.746737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193395:2048]
2025-12-24T02:12:15.746745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193395:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193392:2165] self: [1:7587241558750193393:0] peer: [2:7587241558374325732:0] socket: 17 qp: 34520
2025-12-24T02:12:15.746755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193395:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.746767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.746946Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.746971Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.746982Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325736:2048]
2025-12-24T02:12:15.746997Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325736:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325733:2186] self: [2:7587241558374325732:0] peer: [1:7587241558750193393:0] socket: 16 qp: 34519
2025-12-24T02:12:15.747006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325736:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.747036Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:15.862623Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:15.862720Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:15.862801Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241558750193395:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:15.862813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193395:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.862825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.862844Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241558750193395:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:15.863381Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.863620Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325736:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:15.863638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325736:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:15.863653Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.863691Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325736:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.865460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241558374325736:2048] [node 1] ICS08 incoming handshake Self# [1:7587241558750193393:0] Peer# [2:7587241558374325732:0] Counter# 341 LastInputSerial# 8
2025-12-24T02:12:15.865775Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193397:2167] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.865921Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.865935Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193395:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193397:2167] self: [1:7587241558750193393:0] peer: [2:7587241558374325732:0] socket: 22 qp: 34716
2025-12-24T02:12:15.866540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193395:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.866582Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.866919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241558750193395:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:15.866929Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193395:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:15.866957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:15.866972Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241558750193395:2048] [node 2] ICS08 incoming handshake Self# [2:7587241558374325732:0] Peer# [1:7587241558750193393:0] Counter# 8 LastInputSerial# 341
2025-12-24T02:12:15.867007Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:15.867073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193395:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:15.867093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193395:2048] VirtualId# [1:7587241558750193393:0]
2025-12-24T02:12:15.867121Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:15.868911Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325739:2188] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.869113Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.869134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325736:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325739:2188] self: [2:7587241558374325732:0] peer: [1:7587241558750193393:0] socket: 23 qp: 34720
2025-12-24T02:12:15.869799Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325736:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.869821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:15.870004Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:15.870018Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241558374325736:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:15.870023Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325736:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:15.870033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:15.870075Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:15.870135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325736:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:15.870146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325736:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:15.872515Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241558750193403:2170]) peer tries to resume nonexistent session Self# [2:7587241558374325732:0] Peer# [1:7587241558750193393:0]
2025-12-24T02:12:15.872539Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241558750193403:2170] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:15.872615Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241558374325744:2190] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:15.873612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:15.873649Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:15.873653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325736:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:15.873675Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325736:2048] VirtualId# [2:7587241558374325732:0]
2025-12-24T02:12:15.873715Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 39964
TEvUndelivered Cookie# 39965
TEvUndelivered Cookie# 39966
TEvUndelivered Cookie# 39967
TEvUndelivered Cookie# 39968
TEvUndelivered Cookie# 39969
TEvUndelivered Cookie# 39970
2025-12-24T02:12:15.876037Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241558750193404:2171] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.876079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:15.876649Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241558374325747:2191] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:15.878512Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241558750193404:2171] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:15.878599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:15.878616Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:15.878621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241558750193407:2048]
2025-12-24T02:12:15.878626Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241558750193407:2048] [node 2] ICS09 handshake done sender: [1:7587241558750193404:2171] self: [1:7587241558750193405:0] peer: [2:7587241558374325746:0] socket: 17 qp: 34749
2025-12-24T02:12:15.878634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241558750193407:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:15.878602Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241558374325747:2191] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:15.878641Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:15.878799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:15.878821Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:15.878830Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241558374325750:2048]
2025-12-24T02:12:15.878838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241558374325750:2048] [node 1] ICS09 handshake done sender: [2:7587241558374325747:2191] self: [2:7587241558374325746:0] peer: [1:7587241558750193405:0] socket: 16 qp: 34748
2025-12-24T02:12:15.878877Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241558374325750:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:15.878894Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:16.330744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241558750193407:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:16.330799Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241558750193407:2048] VirtualId# [1:7587241558750193405:0]
2025-12-24T02:12:16.330806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241558750193407:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:16.330852Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:16.330884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:16.331650Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:16.331755Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241558374325750:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:16.331769Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241558374325750:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:16.331825Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:16.331855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241558374325750:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:16.352283Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241563045160708:2174]) peer tries to resume nonexistent session Self# [2:7587241558374325746:0] Peer# [1:7587241558750193405:0]
2025-12-24T02:12:16.352294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241562669293049:2193] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:16.352315Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241563045160708:2174] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:16.352339Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:16.352345Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241558374325750:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:16.352380Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241558374325750:2048] VirtualId# [2:7587241558374325746:0]
2025-12-24T02:12:16.352437Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:16.352456Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:16.353811Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241562669293048:2192] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 41164
TEvUndelivered Cookie# 41165
TEvUndelivered Cookie# 41166
TEvUndelivered Cookie# 41167
TEvUndelivered Cookie# 41168
TEvUndelivered Cookie# 41169
TEvUndelivered Cookie# 41170
TEvUndelivered Cookie# 41171
TEvUndelivered Cookie# 41172
2025-12-24T02:12:16.354452Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [2:7587241562669293049:2193] held: no
2025-12-24T02:12:16.354569Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241563045160706:2173] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:16.369633Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241562669293049:2193] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:16.369698Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241563045160706:2173] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:16.369747Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:16.369767Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:16.369781Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241562669293053:2048]
2025-12-24T02:12:16.369798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241562669293053:2048] [node 1] ICS09 handshake done sender: [2:7587241562669293049:2193] self: [2:7587241562669293051:0] peer: [1:7587241563045160705:0] socket: 21 qp: 35263
2025-12-24T02:12:16.369814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241562669293053:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:16.369827Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:16.369811Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:16.369826Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:16.369832Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241563045160710:2048]
2025-12-24T02:12:16.369837Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241563045160710:2048] [node 2] ICS09 handshake done sender: [1:7587241563045160706:2173] self: [1:7587241563045160705:0] peer: [2:7587241562669293051:0] socket: 17 qp: 35259
2025-12-24T02:12:16.369842Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241563045160710:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:16.369850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:16.790874Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:16.790935Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-12-24T02:12:16.790943Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241563045160710:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:16.790949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241563045160710:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:16.790961Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:16.790995Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:16.791087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241563045160710:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:16.791129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241563045160710:2048] VirtualId# [1:7587241563045160705:0]
2025-12-24T02:12:16.791146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:16.791142Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:16.791516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241562669293053:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:16.791527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241562669293053:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:16.791539Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:16.791569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241562669293053:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:16.793770Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241563045160712:2175]) peer tries to resume nonexistent session Self# [2:7587241562669293051:0] Peer# [1:7587241563045160705:0]
2025-12-24T02:12:16.793798Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241563045160712:2175] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:16.793862Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241562669293055:2195] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:16.794458Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:16.794484Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:16.794488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241562669293053:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:16.794513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241562669293053:2048] VirtualId# [2:7587241562669293051:0]
2025-12-24T02:12:16.794535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 42363
TEvUndelivered Cookie# 42364
2025-12-24T02:12:16.801154Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241562669293059:2197] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:16.805862Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:16.805900Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241563045160716:2177] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:16.806034Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241563045160714:2176] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:16.812390Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241562669293059:2197] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:16.812461Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241563045160714:2176] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:16.812489Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:16.812509Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:16.812516Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241562669293063:2048]
2025-12-24T02:12:16.812521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241562669293063:2048] [node 1] ICS09 handshake done sender: [2:7587241562669293059:2197] self: [2:7587241562669293061:0] peer: [1:7587241563045160713:0] socket: 17 qp: 35714
2025-12-24T02:12:16.812527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241562669293063:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:16.812537Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:16.812591Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:16.812613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:16.812620Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241563045160719:2048]
2025-12-24T02:12:16.812631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241563045160719:2048] [node 2] ICS09 handshake done sender: [1:7587241563045160714:2176] self: [1:7587241563045160713:0] peer: [2:7587241562669293061:0] socket: 16 qp: 35713
2025-12-24T02:12:16.812638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241563045160719:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:16.812647Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:17.306945Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:17.307036Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:17.307106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241563045160719:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:17.307129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241563045160719:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:17.307153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:17.307187Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241563045160719:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:17.307309Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:17.307326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241562669293063:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:17.307333Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241562669293063:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:17.307344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:17.307391Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:17.307471Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241562669293063:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:17.307482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241562669293063:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:17.314451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241562669293063:2048] [node 1] ICS08 incoming handshake Self# [1:7587241563045160713:0] Peer# [2:7587241562669293061:0] Counter# 5490 LastInputSerial# 346
2025-12-24T02:12:17.323640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241563045160719:2048] [node 2] ICS08 incoming handshake Self# [2:7587241562669293061:0] Peer# [1:7587241563045160713:0] Counter# 346 LastInputSerial# 5490
2025-12-24T02:12:17.323873Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241567340128017:2178] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:17.324057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:17.324083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241563045160719:2048] [node 2] ICS09 handshake done sender: [1:7587241567340128017:2178] self: [1:7587241563045160713:0] peer: [2:7587241562669293061:0] socket: 20 qp: 36157
2025-12-24T02:12:17.324707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241563045160719:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:17.324733Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:17.325336Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241566964260362:2200] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:17.325934Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:17.325949Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241562669293063:2048] [node 1] ICS09 handshake done sender: [2:7587241566964260362:2200] self: [2:7587241562669293061:0] peer: [1:7587241563045160713:0] socket: 21 qp: 36156
2025-12-24T02:12:17.326286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241562669293063:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:17.326304Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:17.481898Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:17.481974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:17.481987Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241562669293063:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:17.481993Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241562669293063:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:17.481999Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:17.481974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:17.482078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241563045160719:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:17.482089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241563045160719:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:17.482129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:17.482152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241563045160719:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:17.482320Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:17.482398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241562669293063:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:17.482405Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241562669293063:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:17.588523Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241562669293063:2048] [node 1] ICS08 incoming handshake Self# [1:7587241563045160713:0] Peer# [2:7587241562669293061:0] Counter# 7199 LastInputSerial# 482
2025-12-24T02:12:17.605779Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241567340128022:2180] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:17.605955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:17.605973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241563045160719:2048] [node 2] ICS09 handshake done sender: [1:7587241567340128022:2180] self: [1:7587241563045160713:0] peer: [2:7587241562669293061:0] socket: 16 qp: 36258
2025-12-24T02:12:17.607212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241563045160719:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:17.607229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:17.647506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241563045160719:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:17.647527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241563045160719:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:17.647555Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:17.647577Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241563045160719:2048] [node 2] ICS08 incoming handshake Self# [2:7587241562669293061:0] Peer# [1:7587241563045160713:0] Counter# 482 LastInputSerial# 7199
2025-12-24T02:12:17.647597Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:17.647699Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241563045160719:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:17.647733Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241563045160719:2048] VirtualId# [1:7587241563045160713:0]
2025-12-24T02:12:17.647748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:17.667176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241566964260367:2203] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:17.667491Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:17.667522Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241562669293063:2048] [node 1] ICS09 handshake done sender: [2:7587241566964260367:2203] self: [2:7587241562669293061:0] peer: [1:7587241563045160713:0] socket: 17 qp: 36259
2025-12-24T02:12:17.668025Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241562669293063:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:17.668051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:17.668221Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:17.668227Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241562669293063:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:17.668233Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241562669293063:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:17.668240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:17.668301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:17.668363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241562669293063:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:17.668366Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241562669293063:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:17.694077Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241567340128028:2183]) peer tries to resume nonexistent session Self# [2:7587241562669293061:0] Peer# [1:7587241563045160713:0]
2025-12-24T02:12:17.694105Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241567340128028:2183] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:17.694187Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241566964260372:2205] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:17.695119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:17.695169Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:17.695178Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241562669293063:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:17.695203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241562669293063:2048] VirtualId# [2:7587241562669293061:0]
2025-12-24T02:12:17.695248Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 44084
TEvUndelivered Cookie# 44085
TEvUndelivered Cookie# 44086
TEvUndelivered Cookie# 44087
TEvUndelivered Cookie# 44088
TEvUndelivered Cookie# 44089
2025-12-24T02:12:17.712368Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241567340128029:2184] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:17.712433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:17.713157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241566964260375:2206] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:17.716563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241567340128029:2184] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:17.716628Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241566964260375:2206] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:17.716653Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:17.716676Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:17.716685Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241567340128032:2048]
2025-12-24T02:12:17.716693Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241567340128032:2048] [node 2] ICS09 handshake done sender: [1:7587241567340128029:2184] self: [1:7587241567340128030:0] peer: [2:7587241566964260374:0] socket: 17 qp: 36336
2025-12-24T02:12:17.716707Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241567340128032:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:17.716722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:17.716759Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:17.716809Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:17.716821Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241566964260378:2048]
2025-12-24T02:12:17.716831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241566964260378:2048] [node 1] ICS09 handshake done sender: [2:7587241566964260375:2206] self: [2:7587241566964260374:0] peer: [1:7587241567340128030:0] socket: 16 qp: 36335
2025-12-24T02:12:17.716840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241566964260378:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:17.716850Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:17.778225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241566964260378:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:17.778287Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241566964260378:2048] VirtualId# [2:7587241566964260374:0]
2025-12-24T02:12:17.778304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241566964260378:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:17.778358Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:17.778410Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:17.778448Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:17.778539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241567340128032:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:17.778573Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241567340128032:2048] VirtualId# [1:7587241567340128030:0]
2025-12-24T02:12:17.778583Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241567340128032:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:17.778625Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:17.778657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
Subscribe
2025-12-24T02:12:17.817322Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241567340128034:2186] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:17.817396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:17.817590Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241566964260381:2207] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:17.820361Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241567340128034:2186] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:17.820463Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:17.820487Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:17.820500Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241567340128037:2048]
2025-12-24T02:12:17.820513Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241567340128037:2048] [node 2] ICS09 handshake done sender: [1:7587241567340128034:2186] self: [1:7587241567340128035:0] peer: [2:7587241566964260380:0] socket: 17 qp: 36420
2025-12-24T02:12:17.820496Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241566964260381:2207] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:17.820527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241567340128037:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:17.820542Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:17.820621Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:17.820639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:17.820646Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241566964260384:2048]
2025-12-24T02:12:17.820653Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241566964260384:2048] [node 1] ICS09 handshake done sender: [2:7587241566964260381:2207] self: [2:7587241566964260380:0] peer: [1:7587241567340128035:0] socket: 16 qp: 36419
2025-12-24T02:12:17.820661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241566964260384:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:17.820670Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:18.312297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:18.312359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:18.312437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241567340128037:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:18.312447Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241567340128037:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:18.312459Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:18.312480Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241567340128037:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:18.312977Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:18.313162Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241566964260384:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:18.313172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241566964260384:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:18.313183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:18.313213Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241566964260384:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:18.314416Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241566964260384:2048] [node 1] ICS08 incoming handshake Self# [1:7587241567340128035:0] Peer# [2:7587241566964260380:0] Counter# 7769 LastInputSerial# 277
2025-12-24T02:12:18.315713Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241571635095335:2188] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:18.315829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241567340128037:2048] [node 2] ICS08 incoming handshake Self# [2:7587241566964260380:0] Peer# [1:7587241567340128035:0] Counter# 277 LastInputSerial# 7769
2025-12-24T02:12:18.315839Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:18.315848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241567340128037:2048] [node 2] ICS09 handshake done sender: [1:7587241571635095335:2188] self: [1:7587241567340128035:0] peer: [2:7587241566964260380:0] socket: 22 qp: 36856
2025-12-24T02:12:18.316114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241567340128037:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:18.316130Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:18.317375Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241571259227682:2208] held: no
2025-12-24T02:12:18.317396Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241571259227682:2208] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:18.317442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:18.317453Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241566964260384:2048] [node 1] ICS09 handshake done sender: [2:7587241571259227682:2208] self: [2:7587241566964260380:0] peer: [1:7587241567340128035:0] socket: 23 qp: 36857
2025-12-24T02:12:18.317773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241566964260384:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:18.317788Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:18.745409Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:18.745503Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:18.745527Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241566964260384:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:18.745533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241566964260384:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:18.745547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:18.745577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:18.745591Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-12-24T02:12:18.745668Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241566964260384:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:18.745677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241566964260384:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:18.745695Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241567340128037:2048] [node 2] ICS07 socket disconnect 22 reason# ECONNRESET
2025-12-24T02:12:18.745706Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241567340128037:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-12-24T02:12:18.745719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:18.745761Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241567340128037:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:18.748708Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241567340128037:2048] [node 2] ICS08 incoming handshake Self# [2:7587241566964260380:0] Peer# [1:7587241567340128035:0] Counter# 441 LastInputSerial# 14060
2025-12-24T02:12:18.748752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241566964260384:2048] [node 1] ICS08 incoming handshake Self# [1:7587241567340128035:0] Peer# [2:7587241566964260380:0] Counter# 14060 LastInputSerial# 441
2025-12-24T02:12:18.749018Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241571635095340:2190] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:18.749193Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:18.749216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241567340128037:2048] [node 2] ICS09 handshake done sender: [1:7587241571635095340:2190] self: [1:7587241567340128035:0] peer: [2:7587241566964260380:0] socket: 20 qp: 37260
2025-12-24T02:12:18.749636Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241567340128037:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:18.749660Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:18.751797Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241571259227689:2212] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:18.751865Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241571259227689:2212] held: no
2025-12-24T02:12:18.751875Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:18.751882Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241566964260384:2048] [node 1] ICS09 handshake done sender: [2:7587241571259227689:2212] self: [2:7587241566964260380:0] peer: [1:7587241567340128035:0] socket: 21 qp: 37263
2025-12-24T02:12:18.752172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241566964260384:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:18.752186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:18.928534Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:18.928610Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241567340128037:2048] [node 2] ICS07 socket disconnect 20 reason# Debug
2025-12-24T02:12:18.928619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241567340128037:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:18.928639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:18.928659Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241567340128037:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:18.929974Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:18.930126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241566964260384:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:18.930134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241566964260384:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:18.930144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:18.930170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241566964260384:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:18.931203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241566964260384:2048] [node 1] ICS08 incoming handshake Self# [1:7587241567340128035:0] Peer# [2:7587241566964260380:0] Counter# 16505 LastInputSerial# 504
2025-12-24T02:12:18.931685Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241571635095345:2192] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:18.931806Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:18.931822Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241567340128037:2048] [node 2] ICS09 handshake done sender: [1:7587241571635095345:2192] self: [1:7587241567340128035:0] peer: [2:7587241566964260380:0] socket: 16 qp: 37451
2025-12-24T02:12:18.932243Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241567340128037:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:18.932260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:18.932451Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241567340128037:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:18.932465Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241567340128037:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:18.932499Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:18.932525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241567340128037:2048] [node 2] ICS08 incoming handshake Self# [2:7587241566964260380:0] Peer# [1:7587241567340128035:0] Counter# 504 LastInputSerial# 16505
2025-12-24T02:12:18.932533Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:18.932602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241567340128037:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:18.932630Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241567340128037:2048] VirtualId# [1:7587241567340128035:0]
2025-12-24T02:12:18.932646Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:18.933859Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241571259227693:2215] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:18.934031Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:18.934049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241566964260384:2048] [node 1] ICS09 handshake done sender: [2:7587241571259227693:2215] self: [2:7587241566964260380:0] peer: [1:7587241567340128035:0] socket: 17 qp: 37452
2025-12-24T02:12:18.934606Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241566964260384:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:18.934621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:18.934781Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:18.934793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241566964260384:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:18.934798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241566964260384:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:18.934804Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:18.934834Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:18.934891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241566964260384:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:18.934898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241566964260384:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:18.937297Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241571635095351:2195]) peer tries to resume nonexistent session Self# [2:7587241566964260380:0] Peer# [1:7587241567340128035:0]
2025-12-24T02:12:18.937320Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241571635095351:2195] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:18.937400Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241571259227698:2217] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:18.938943Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:18.938999Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:18.939013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241566964260384:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:18.939054Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241566964260384:2048] VirtualId# [2:7587241566964260380:0]
2025-12-24T02:12:18.939110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 48079
TEvUndelivered Cookie# 48080
TEvUndelivered Cookie# 48081
TEvUndelivered Cookie# 48082
TEvUndelivered Cookie# 48083
2025-12-24T02:12:18.941573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241571635095352:2196] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:18.941617Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:18.941758Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241571259227701:2218] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:18.943933Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241571635095352:2196] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:18.944007Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241571259227701:2218] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:18.944051Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:18.944071Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:18.944080Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241571635095355:2048]
2025-12-24T02:12:18.944093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241571635095355:2048] [node 2] ICS09 handshake done sender: [1:7587241571635095352:2196] self: [1:7587241571635095353:0] peer: [2:7587241571259227700:0] socket: 17 qp: 37466
2025-12-24T02:12:18.944101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241571635095355:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:18.944115Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:18.944203Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:18.944262Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:18.944289Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241571259227704:2048]
2025-12-24T02:12:18.944303Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241571259227704:2048] [node 1] ICS09 handshake done sender: [2:7587241571259227701:2218] self: [2:7587241571259227700:0] peer: [1:7587241571635095353:0] socket: 16 qp: 37465
2025-12-24T02:12:18.944320Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241571259227704:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:18.944341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:19.112602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:19.112682Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-12-24T02:12:19.112688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241571635095355:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:19.112694Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241571635095355:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:19.112706Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:19.112734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:19.112714Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:19.112814Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241571259227704:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:19.112812Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241571635095355:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:19.112827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241571259227704:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:19.112860Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241571635095355:2048] VirtualId# [1:7587241571635095353:0]
2025-12-24T02:12:19.112870Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:19.112876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:19.112891Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241571259227704:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:19.115880Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241575930062653:2198]) peer tries to resume nonexistent session Self# [2:7587241571259227700:0] Peer# [1:7587241571635095353:0]
2025-12-24T02:12:19.115986Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241575930062653:2198] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:19.116080Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241575554195002:2219] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:19.116978Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:19.117009Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:19.117013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241571259227704:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:19.117042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241571259227704:2048] VirtualId# [2:7587241571259227700:0]
2025-12-24T02:12:19.117065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:19.118965Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241575554195004:2220] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:19.120554Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:19.120706Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241575930062655:2199] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:19.120802Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241575930062657:2200] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:19.123014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241575554195004:2220] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:19.123079Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:19.123096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:19.123103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241575554195010:2048]
2025-12-24T02:12:19.123112Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241575554195010:2048] [node 1] ICS09 handshake done sender: [2:7587241575554195004:2220] self: [2:7587241575554195008:0] peer: [1:7587241575930062654:0] socket: 23 qp: 37689
2025-12-24T02:12:19.123120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241575554195010:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:19.123102Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241575930062655:2199] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:19.123134Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:19.123272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:19.123296Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:19.123305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241575930062660:2048]
2025-12-24T02:12:19.123318Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241575930062660:2048] [node 2] ICS09 handshake done sender: [1:7587241575930062655:2199] self: [1:7587241575930062654:0] peer: [2:7587241575554195008:0] socket: 21 qp: 37687
2025-12-24T02:12:19.123329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241575930062660:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:19.123341Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:19.223586Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:19.223733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241575554195010:2048] [node 1] ICS07 socket disconnect 23 reason# Debug
2025-12-24T02:12:19.223746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241575554195010:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:19.223769Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:19.223790Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241575554195010:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:19.223779Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:19.223872Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241575930062660:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:19.223885Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241575930062660:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:19.223938Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:19.223973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241575930062660:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:19.226430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241575930062660:2048] [node 2] ICS08 incoming handshake Self# [2:7587241575554195008:0] Peer# [1:7587241575930062654:0] Counter# 6 LastInputSerial# 115
2025-12-24T02:12:19.228290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241575554195010:2048] [node 1] ICS08 incoming handshake Self# [1:7587241575930062654:0] Peer# [2:7587241575554195008:0] Counter# 115 LastInputSerial# 6
2025-12-24T02:12:19.228594Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241575930062662:2201] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:19.228710Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:19.228728Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241575930062660:2048] [node 2] ICS09 handshake done sender: [1:7587241575930062662:2201] self: [1:7587241575930062654:0] peer: [2:7587241575554195008:0] socket: 24 qp: 37832
2025-12-24T02:12:19.229148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241575930062660:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:19.229168Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:19.231603Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241575554195014:2224] held: no
2025-12-24T02:12:19.231629Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241575554195014:2224] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:19.231737Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:19.231753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241575554195010:2048] [node 1] ICS09 handshake done sender: [2:7587241575554195014:2224] self: [2:7587241575554195008:0] peer: [1:7587241575930062654:0] socket: 25 qp: 37831
2025-12-24T02:12:19.232290Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241575554195010:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:19.232313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:19.639314Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:19.639408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241575930062660:2048] [node 2] ICS07 socket disconnect 24 reason# Debug
2025-12-24T02:12:19.639421Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241575930062660:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:19.639446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:19.639467Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241575930062660:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:19.640396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:19.640496Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241575554195010:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:19.640509Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241575554195010:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:19.640561Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:19.640595Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241575554195010:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:19.642375Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241575554195010:2048] [node 1] ICS08 incoming handshake Self# [1:7587241575930062654:0] Peer# [2:7587241575554195008:0] Counter# 6875 LastInputSerial# 273
2025-12-24T02:12:19.642714Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241575930062667:2203] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:19.642855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:19.642874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241575930062660:2048] [node 2] ICS09 handshake done sender: [1:7587241575930062667:2203] self: [1:7587241575930062654:0] peer: [2:7587241575554195008:0] socket: 16 qp: 38185
2025-12-24T02:12:19.643387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241575930062660:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:19.643412Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:19.643692Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241575930062660:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:19.643703Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241575930062660:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:19.643728Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:19.643754Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241575930062660:2048] [node 2] ICS08 incoming handshake Self# [2:7587241575554195008:0] Peer# [1:7587241575930062654:0] Counter# 273 LastInputSerial# 6875
2025-12-24T02:12:19.643768Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:19.643830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241575930062660:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:19.643859Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241575930062660:2048] VirtualId# [1:7587241575930062654:0]
2025-12-24T02:12:19.643877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:19.645529Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241575554195018:2227] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:19.645693Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:19.645710Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241575554195010:2048] [node 1] ICS09 handshake done sender: [2:7587241575554195018:2227] self: [2:7587241575554195008:0] peer: [1:7587241575930062654:0] socket: 17 qp: 38188
2025-12-24T02:12:19.646373Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241575554195010:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:19.646394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:19.646519Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:19.646530Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241575554195010:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:19.646535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241575554195010:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:19.646543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:19.646577Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:19.646634Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241575554195010:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:19.646674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241575554195010:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:19.649484Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241575930062673:2206]) peer tries to resume nonexistent session Self# [2:7587241575554195008:0] Peer# [1:7587241575930062654:0]
2025-12-24T02:12:19.649506Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241575930062673:2206] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:19.649576Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241575554195023:2229] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:19.650119Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:19.650153Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:19.650168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241575554195010:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:19.650196Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241575554195010:2048] VirtualId# [2:7587241575554195008:0]
2025-12-24T02:12:19.650263Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 50097
TEvUndelivered Cookie# 50098
TEvUndelivered Cookie# 50099
TEvUndelivered Cookie# 50100
TEvUndelivered Cookie# 50101
TEvUndelivered Cookie# 50102
TEvUndelivered Cookie# 50103
TEvUndelivered Cookie# 50104
TEvUndelivered Cookie# 50105
2025-12-24T02:12:19.652600Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241575930062674:2207] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:19.652635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:19.653173Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241575554195026:2230] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:19.656832Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241575930062674:2207] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:19.656907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241575554195026:2230] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:19.656931Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:19.656955Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:19.656966Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241575930062677:2048]
2025-12-24T02:12:19.656979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241575930062677:2048] [node 2] ICS09 handshake done sender: [1:7587241575930062674:2207] self: [1:7587241575930062675:0] peer: [2:7587241575554195025:0] socket: 17 qp: 38202
2025-12-24T02:12:19.656989Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241575930062677:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:19.657008Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:19.657042Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:19.657059Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:19.657067Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241575554195029:2048]
2025-12-24T02:12:19.657075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241575554195029:2048] [node 1] ICS09 handshake done sender: [2:7587241575554195026:2230] self: [2:7587241575554195025:0] peer: [1:7587241575930062675:0] socket: 16 qp: 38201
2025-12-24T02:12:19.657087Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241575554195029:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:19.657102Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:19.993892Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:19.993971Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:19.994003Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:19.994059Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241575554195029:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:19.994076Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241575930062677:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:19.994088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241575930062677:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:19.994110Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241575554195029:2048] VirtualId# [2:7587241575554195025:0]
2025-12-24T02:12:19.994123Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241575554195029:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:19.994136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:19.994136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:19.994153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241575930062677:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:19.994181Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:19.997611Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241575554195031:2231]) peer tries to resume nonexistent session Self# [1:7587241575930062675:0] Peer# [2:7587241575554195025:0]
2025-12-24T02:12:19.997717Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241575554195031:2231] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:19.998201Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241575930062679:2209] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:19.998750Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:19.998780Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:19.998785Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241575930062677:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:19.998814Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241575930062677:2048] VirtualId# [1:7587241575930062675:0]
2025-12-24T02:12:19.998836Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:20.002014Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241575930062681:2210] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.003548Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241575554195035:2233] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.003594Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:20.003789Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241575930062683:2211] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.005849Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241575554195035:2233] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:20.005928Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:20.005920Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241575930062683:2211] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:20.005949Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:20.005962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241579849162334:2048]
2025-12-24T02:12:20.005970Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241579849162334:2048] [node 1] ICS09 handshake done sender: [2:7587241575554195035:2233] self: [2:7587241579849162332:0] peer: [1:7587241575930062682:0] socket: 17 qp: 38543
2025-12-24T02:12:20.005978Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241579849162334:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:20.005989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:20.006057Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:20.006079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:20.006087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241580225029983:2048]
2025-12-24T02:12:20.006098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241580225029983:2048] [node 2] ICS09 handshake done sender: [1:7587241575930062683:2211] self: [1:7587241575930062682:0] peer: [2:7587241579849162332:0] socket: 16 qp: 38544
2025-12-24T02:12:20.006103Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241580225029983:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:20.006112Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:20.464893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241579849162334:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:20.464950Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241579849162334:2048] VirtualId# [2:7587241579849162332:0]
2025-12-24T02:12:20.464959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241579849162334:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:20.464997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:20.465034Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:20.465051Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:20.465156Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241580225029983:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:20.465169Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241580225029983:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:20.465220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:20.465248Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241580225029983:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 52649
TEvUndelivered Cookie# 52650
TEvUndelivered Cookie# 52651
2025-12-24T02:12:20.467230Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241579849162336:2235]) peer tries to resume nonexistent session Self# [1:7587241575930062682:0] Peer# [2:7587241579849162332:0]
2025-12-24T02:12:20.467259Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241579849162336:2235] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:20.467343Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241580225029985:2212] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:20.468045Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:20.468083Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:20.468094Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241580225029983:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:20.468129Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241580225029983:2048] VirtualId# [1:7587241575930062682:0]
2025-12-24T02:12:20.468153Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:20.472573Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241580225029989:2214] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.472711Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241579849162340:2237] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.472757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:20.472904Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241580225029988:2213] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:20.474974Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241579849162340:2237] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:20.475029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241580225029988:2213] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:20.475062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:20.475085Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:20.475094Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241579849162343:2048]
2025-12-24T02:12:20.475100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241579849162343:2048] [node 1] ICS09 handshake done sender: [2:7587241579849162340:2237] self: [2:7587241579849162341:0] peer: [1:7587241580225029987:0] socket: 21 qp: 39151
2025-12-24T02:12:20.475108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241579849162343:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:20.475117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:20.475159Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:20.475182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:20.475191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241580225029993:2048]
2025-12-24T02:12:20.475204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241580225029993:2048] [node 2] ICS09 handshake done sender: [1:7587241580225029988:2213] self: [1:7587241580225029987:0] peer: [2:7587241579849162341:0] socket: 20 qp: 39153
2025-12-24T02:12:20.475210Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241580225029993:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:20.475221Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:21.030126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241580225029993:2048] [node 2] ICS01 socket: 20 reason# Debug
2025-12-24T02:12:21.030191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241580225029993:2048] VirtualId# [1:7587241580225029987:0]
2025-12-24T02:12:21.030202Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241580225029993:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:21.030254Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:21.030302Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:21.031035Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:21.031266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241579849162343:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:21.031282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241579849162343:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:21.031295Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:21.031324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241579849162343:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:21.033254Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241584519997291:2215]) peer tries to resume nonexistent session Self# [2:7587241579849162341:0] Peer# [1:7587241580225029987:0]
2025-12-24T02:12:21.033285Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241584519997291:2215] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:21.033543Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241584144129641:2239] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:21.034103Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:21.034145Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:21.034155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241579849162343:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:21.034191Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241579849162343:2048] VirtualId# [2:7587241579849162341:0]
2025-12-24T02:12:21.034236Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 54240
TEvUndelivered Cookie# 54241
TEvUndelivered Cookie# 54242
TEvUndelivered Cookie# 54243
TEvUndelivered Cookie# 54244
TEvUndelivered Cookie# 54245
TEvUndelivered Cookie# 54246
TEvUndelivered Cookie# 54247
TEvUndelivered Cookie# 54248
2025-12-24T02:12:21.036490Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241584519997292:2216] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:21.036552Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:21.036766Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241584144129644:2240] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:21.038711Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241584519997292:2216] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:21.038835Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241584144129644:2240] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:21.038810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:21.038833Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:21.038840Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241584519997295:2048]
2025-12-24T02:12:21.038847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241584519997295:2048] [node 2] ICS09 handshake done sender: [1:7587241584519997292:2216] self: [1:7587241584519997293:0] peer: [2:7587241584144129643:0] socket: 17 qp: 39714
2025-12-24T02:12:21.038856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241584519997295:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:21.038865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:21.038965Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:21.038985Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:21.038996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241584144129647:2048]
2025-12-24T02:12:21.039008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241584144129647:2048] [node 1] ICS09 handshake done sender: [2:7587241584144129644:2240] self: [2:7587241584144129643:0] peer: [1:7587241584519997293:0] socket: 16 qp: 39713
2025-12-24T02:12:21.039016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241584144129647:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:21.039031Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:21.555381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241584519997295:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:21.555499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241584519997295:2048] VirtualId# [1:7587241584519997293:0]
2025-12-24T02:12:21.555512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241584519997295:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:21.555543Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:21.555564Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:21.557301Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:21.557430Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241584144129647:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:21.557499Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241584144129647:2048] VirtualId# [2:7587241584144129643:0]
2025-12-24T02:12:21.557516Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241584144129647:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:21.557578Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:21.557615Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:21.562829Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241584144129649:2241] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:21.562906Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:21.563157Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241584519997298:2218] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:21.565393Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241584519997298:2218] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:21.565600Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:21.565619Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:21.565627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241584519997301:2048]
2025-12-24T02:12:21.565637Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241584519997301:2048] [node 2] ICS09 handshake done sender: [1:7587241584519997298:2218] self: [1:7587241584519997297:0] peer: [2:7587241584144129650:0] socket: 17 qp: 40211
2025-12-24T02:12:21.565644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241584519997301:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:21.565658Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:21.565324Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241584144129649:2241] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:21.565448Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:21.565464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:21.565472Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241584144129652:2048]
2025-12-24T02:12:21.565479Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241584144129652:2048] [node 1] ICS09 handshake done sender: [2:7587241584144129649:2241] self: [2:7587241584144129650:0] peer: [1:7587241584519997297:0] socket: 16 qp: 40213
2025-12-24T02:12:21.565488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241584144129652:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:21.565500Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:22.018356Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:22.018458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241584144129652:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:22.018470Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241584144129652:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:22.018501Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.018525Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241584144129652:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:22.018559Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.018660Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241584519997301:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:22.018673Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241584519997301:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.018742Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.018771Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241584519997301:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:22.020118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241584519997301:2048] [node 2] ICS08 incoming handshake Self# [2:7587241584144129650:0] Peer# [1:7587241584519997297:0] Counter# 123 LastInputSerial# 4804
2025-12-24T02:12:22.021612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241584144129652:2048] [node 1] ICS08 incoming handshake Self# [1:7587241584519997297:0] Peer# [2:7587241584144129650:0] Counter# 4804 LastInputSerial# 123
2025-12-24T02:12:22.021872Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964599:2219] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.022059Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.022083Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241584519997301:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964599:2219] self: [1:7587241584519997297:0] peer: [2:7587241584144129650:0] socket: 23 qp: 40557
2025-12-24T02:12:22.022438Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241584519997301:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.022462Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.024673Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241588439096952:2244] held: no
2025-12-24T02:12:22.024705Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096952:2244] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.024777Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.024793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241584144129652:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096952:2244] self: [2:7587241584144129650:0] peer: [1:7587241584519997297:0] socket: 25 qp: 40556
2025-12-24T02:12:22.025264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241584144129652:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.025287Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:22.362794Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:22.362888Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-12-24T02:12:22.362895Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241584519997301:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:22.362901Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241584519997301:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:22.362912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.363759Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.363887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241584519997301:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:22.363905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241584519997301:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:22.368100Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:22.368226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241584144129652:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:22.368234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241584144129652:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.368285Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.368305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241584144129652:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:22.370089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241584144129652:2048] [node 1] ICS08 incoming handshake Self# [1:7587241584519997297:0] Peer# [2:7587241584144129650:0] Counter# 9150 LastInputSerial# 232
2025-12-24T02:12:22.374842Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964604:2221] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.374992Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.375014Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241584519997301:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964604:2221] self: [1:7587241584519997297:0] peer: [2:7587241584144129650:0] socket: 16 qp: 40815
2025-12-24T02:12:22.375876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241584519997301:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.375905Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.375984Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241584519997301:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:22.375996Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241584519997301:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:22.376012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.376026Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241584519997301:2048] [node 2] ICS08 incoming handshake Self# [2:7587241584144129650:0] Peer# [1:7587241584519997297:0] Counter# 232 LastInputSerial# 9150
2025-12-24T02:12:22.376060Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.376140Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241584519997301:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:22.383366Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096956:2247] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.383612Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.383642Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241584144129652:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096956:2247] self: [2:7587241584144129650:0] peer: [1:7587241584519997297:0] socket: 17 qp: 40816
2025-12-24T02:12:22.384002Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241584144129652:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.384029Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:22.384156Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:22.384228Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241584144129652:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:22.384264Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241584144129652:2048] VirtualId# [2:7587241584144129650:0]
2025-12-24T02:12:22.384273Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241584144129652:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.384299Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.384350Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:22.393144Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241588814964610:2224] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:22.393196Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:22.393203Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241584519997301:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:22.393231Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241584519997301:2048] VirtualId# [1:7587241584519997297:0]
2025-12-24T02:12:22.393264Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:22.393276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:22.394415Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241588439096962:2249] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:22.396282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964610:2224] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.396337Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096962:2249] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.396372Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.396390Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:22.396404Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241588814964613:2048]
2025-12-24T02:12:22.396411Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241588814964613:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964610:2224] self: [1:7587241588814964611:0] peer: [2:7587241588439096961:0] socket: 17 qp: 40831
2025-12-24T02:12:22.396420Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241588814964613:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.396443Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.396449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.396470Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:22.396476Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241588439096965:2048]
2025-12-24T02:12:22.396482Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241588439096965:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096962:2249] self: [2:7587241588439096961:0] peer: [1:7587241588814964611:0] socket: 16 qp: 40830
2025-12-24T02:12:22.396487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241588439096965:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.396496Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:22.495385Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.495494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241588814964613:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:22.495510Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241588814964613:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.494880Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:22.495003Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241588439096965:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:22.495011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241588439096965:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:22.495048Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.495069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241588439096965:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:22.495822Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.495860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241588814964613:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:22.507204Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241588814964613:2048] [node 2] ICS08 incoming handshake Self# [2:7587241588439096961:0] Peer# [1:7587241588814964611:0] Counter# 3 LastInputSerial# 131
2025-12-24T02:12:22.509148Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241588439096965:2048] [node 1] ICS08 incoming handshake Self# [1:7587241588814964611:0] Peer# [2:7587241588439096961:0] Counter# 131 LastInputSerial# 3
2025-12-24T02:12:22.510785Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964615:2226] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.510919Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.510939Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241588814964613:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964615:2226] self: [1:7587241588814964611:0] peer: [2:7587241588439096961:0] socket: 24 qp: 40936
2025-12-24T02:12:22.511840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241588814964613:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.511867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.511887Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096969:2251] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.511983Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.512007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241588439096965:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096969:2251] self: [2:7587241588439096961:0] peer: [1:7587241588814964611:0] socket: 25 qp: 40921
2025-12-24T02:12:22.512460Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241588439096965:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.512480Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:22.734962Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:22.735021Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.735098Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241588814964613:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-12-24T02:12:22.735110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241588814964613:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.735119Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.735137Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241588814964613:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:22.735120Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:22.735328Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241588439096965:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:22.735340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241588439096965:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:22.735351Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.735384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241588439096965:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:22.736685Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241588439096965:2048] [node 1] ICS08 incoming handshake Self# [1:7587241588814964611:0] Peer# [2:7587241588439096961:0] Counter# 3145 LastInputSerial# 75
2025-12-24T02:12:22.736952Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964620:2228] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.737076Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.737093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241588814964613:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964620:2228] self: [1:7587241588814964611:0] peer: [2:7587241588439096961:0] socket: 16 qp: 41113
2025-12-24T02:12:22.738263Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241588814964613:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.738288Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.739639Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241588814964613:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:22.739658Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241588814964613:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:22.739697Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:22.739718Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241588814964613:2048] [node 2] ICS08 incoming handshake Self# [2:7587241588439096961:0] Peer# [1:7587241588814964611:0] Counter# 75 LastInputSerial# 3145
2025-12-24T02:12:22.739735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:22.739821Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241588814964613:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:22.739850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241588814964613:2048] VirtualId# [1:7587241588814964611:0]
2025-12-24T02:12:22.739867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:22.741177Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096973:2254] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.741267Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.741286Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241588439096965:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096973:2254] self: [2:7587241588439096961:0] peer: [1:7587241588814964611:0] socket: 17 qp: 41114
2025-12-24T02:12:22.741744Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241588439096965:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.741764Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:22.741926Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:22.741940Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241588439096965:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:22.741945Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241588439096965:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:22.741953Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:22.741991Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:22.742067Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241588439096965:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:22.742075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241588439096965:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:22.744197Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241588814964625:2230]) peer tries to resume nonexistent session Self# [2:7587241588439096961:0] Peer# [1:7587241588814964611:0]
2025-12-24T02:12:22.744222Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241588814964625:2230] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:22.744304Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241588439096977:2256] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:22.744927Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:22.744959Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:22.744969Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241588439096965:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:22.744993Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241588439096965:2048] VirtualId# [2:7587241588439096961:0]
2025-12-24T02:12:22.745033Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 58595
TEvUndelivered Cookie# 58596
TEvUndelivered Cookie# 58597
TEvUndelivered Cookie# 58598
TEvUndelivered Cookie# 58599
TEvUndelivered Cookie# 58600
2025-12-24T02:12:22.747161Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241588814964626:2231] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:22.747205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:22.747858Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241588439096981:2257] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:22.751653Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241588814964626:2231] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:22.751744Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:22.751766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:22.751737Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241588439096981:2257] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:22.751775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241588814964629:2048]
2025-12-24T02:12:22.751783Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241588814964629:2048] [node 2] ICS09 handshake done sender: [1:7587241588814964626:2231] self: [1:7587241588814964627:0] peer: [2:7587241588439096980:0] socket: 17 qp: 41128
2025-12-24T02:12:22.751793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241588814964629:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:22.751805Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:22.751857Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:22.751874Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:22.751886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241588439096984:2048]
2025-12-24T02:12:22.751893Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241588439096984:2048] [node 1] ICS09 handshake done sender: [2:7587241588439096981:2257] self: [2:7587241588439096980:0] peer: [1:7587241588814964627:0] socket: 16 qp: 41127
2025-12-24T02:12:22.751900Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241588439096984:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:22.751917Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:23.272205Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:23.272298Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:23.272440Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241588814964629:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:23.272454Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241588814964629:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:23.272469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:23.272506Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241588814964629:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:23.272526Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:23.272749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241588439096984:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:23.272760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241588439096984:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:23.272773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:23.272806Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241588439096984:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:23.274170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241588439096984:2048] [node 1] ICS08 incoming handshake Self# [1:7587241588814964627:0] Peer# [2:7587241588439096980:0] Counter# 5409 LastInputSerial# 137
2025-12-24T02:12:23.275563Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241593109931927:2233] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:23.275730Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:23.275749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241588814964629:2048] [node 2] ICS09 handshake done sender: [1:7587241593109931927:2233] self: [1:7587241588814964627:0] peer: [2:7587241588439096980:0] socket: 22 qp: 41532
2025-12-24T02:12:23.276136Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241588814964629:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:23.276159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:23.276200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241588814964629:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:23.276212Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241588814964629:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:23.276239Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:23.276253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241588814964629:2048] [node 2] ICS08 incoming handshake Self# [2:7587241588439096980:0] Peer# [1:7587241588814964627:0] Counter# 137 LastInputSerial# 5409
2025-12-24T02:12:23.276275Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:23.276371Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241588814964629:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:23.276396Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241588814964629:2048] VirtualId# [1:7587241588814964627:0]
2025-12-24T02:12:23.276405Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:23.278067Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241592734064283:2259] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:23.278152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:23.278168Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241588439096984:2048] [node 1] ICS09 handshake done sender: [2:7587241592734064283:2259] self: [2:7587241588439096980:0] peer: [1:7587241588814964627:0] socket: 23 qp: 41533
2025-12-24T02:12:23.278624Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241588439096984:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:23.278645Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:23.279056Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:23.279073Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241588439096984:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:23.279078Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241588439096984:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:23.279086Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:23.279128Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:23.279191Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241588439096984:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:23.279198Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241588439096984:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:23.281894Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241593109931933:2236]) peer tries to resume nonexistent session Self# [2:7587241588439096980:0] Peer# [1:7587241588814964627:0]
2025-12-24T02:12:23.281919Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241593109931933:2236] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:23.282028Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241592734064288:2261] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:23.282571Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:23.282604Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:23.282609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241588439096984:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:23.282637Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241588439096984:2048] VirtualId# [2:7587241588439096980:0]
2025-12-24T02:12:23.282683Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 59934
TEvUndelivered Cookie# 59935
TEvUndelivered Cookie# 59936
TEvUndelivered Cookie# 59937
TEvUndelivered Cookie# 59938
TEvUndelivered Cookie# 59939
TEvUndelivered Cookie# 59940
TEvUndelivered Cookie# 59941
2025-12-24T02:12:23.286633Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241593109931934:2237] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.286674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:23.287277Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241592734064291:2262] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.288907Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241593109931934:2237] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:23.288967Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241592734064291:2262] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:23.289131Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:23.289146Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:23.288998Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:23.289154Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241592734064294:2048]
2025-12-24T02:12:23.289021Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:23.289029Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241593109931937:2048]
2025-12-24T02:12:23.289160Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064294:2048] [node 1] ICS09 handshake done sender: [2:7587241592734064291:2262] self: [2:7587241592734064290:0] peer: [1:7587241593109931935:0] socket: 16 qp: 41546
2025-12-24T02:12:23.289036Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931937:2048] [node 2] ICS09 handshake done sender: [1:7587241593109931934:2237] self: [1:7587241593109931935:0] peer: [2:7587241592734064290:0] socket: 17 qp: 41547
2025-12-24T02:12:23.289042Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931937:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:23.289051Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:23.289166Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064294:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:23.289174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:23.637288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241592734064294:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:23.637347Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241592734064294:2048] VirtualId# [2:7587241592734064290:0]
2025-12-24T02:12:23.637355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064294:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:23.637394Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:23.637425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:23.637742Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:23.637888Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241593109931937:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:23.637897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931937:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:23.637908Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
TEvNodeDisconnected
2025-12-24T02:12:23.637934Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241593109931937:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:23.639231Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241592734064296:2263]) peer tries to resume nonexistent session Self# [1:7587241593109931935:0] Peer# [2:7587241592734064290:0]
2025-12-24T02:12:23.639260Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241592734064296:2263] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:23.639343Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241593109931939:2239] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:23.642384Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:23.642434Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:23.642450Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241593109931937:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:23.642484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241593109931937:2048] VirtualId# [1:7587241593109931935:0]
2025-12-24T02:12:23.642510Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:23.646083Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241592734064297:2264] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.646146Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:23.646310Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241593109931942:2240] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.650433Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241592734064297:2264] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:23.650535Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:23.650560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:23.650568Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241592734064300:2048]
2025-12-24T02:12:23.650581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064300:2048] [node 1] ICS09 handshake done sender: [2:7587241592734064297:2264] self: [2:7587241592734064298:0] peer: [1:7587241593109931941:0] socket: 17 qp: 41830
2025-12-24T02:12:23.650592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064300:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:23.650608Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:23.650648Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241593109931942:2240] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:23.650785Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:23.650803Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:23.650810Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241593109931945:2048]
2025-12-24T02:12:23.650817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931945:2048] [node 2] ICS09 handshake done sender: [1:7587241593109931942:2240] self: [1:7587241593109931941:0] peer: [2:7587241592734064298:0] socket: 16 qp: 41829
2025-12-24T02:12:23.650823Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931945:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:23.650831Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:23.820974Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:23.821076Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [1 <-> 2] socket: Broken pipe
2025-12-24T02:12:23.821093Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241593109931945:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:23.821100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931945:2048] [node 2] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:23.821110Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:23.821193Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:23.821288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241593109931945:2048] [node 2] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:23.821296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241593109931945:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:23.822209Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:23.822298Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241592734064300:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:23.822339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241592734064300:2048] VirtualId# [2:7587241592734064298:0]
2025-12-24T02:12:23.822355Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064300:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:23.822404Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:23.822439Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:23.824097Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241592734064302:2266]) peer tries to resume nonexistent session Self# [1:7587241593109931941:0] Peer# [2:7587241592734064298:0]
2025-12-24T02:12:23.824123Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241592734064302:2266] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:23.824210Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241593109931947:2241] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:23.824797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:23.824826Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:23.824829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241593109931945:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:23.824855Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241593109931945:2048] VirtualId# [1:7587241593109931941:0]
2025-12-24T02:12:23.824875Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:23.827940Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241592734064303:2267] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.827991Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:23.828165Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241593109931950:2242] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.830246Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241592734064303:2267] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:23.830305Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241593109931950:2242] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:23.830411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:23.830431Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:23.830449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241593109931953:2048]
2025-12-24T02:12:23.830457Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931953:2048] [node 2] ICS09 handshake done sender: [1:7587241593109931950:2242] self: [1:7587241593109931949:0] peer: [2:7587241592734064304:0] socket: 16 qp: 41993
2025-12-24T02:12:23.830466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931953:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:23.830478Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:23.830766Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:23.830778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:23.830783Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241592734064306:2048]
2025-12-24T02:12:23.830788Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064306:2048] [node 1] ICS09 handshake done sender: [2:7587241592734064303:2267] self: [2:7587241592734064304:0] peer: [1:7587241593109931949:0] socket: 17 qp: 41994
2025-12-24T02:12:23.830792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064306:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:23.830799Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:23.966794Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241593109931953:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:23.966847Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241593109931953:2048] VirtualId# [1:7587241593109931949:0]
2025-12-24T02:12:23.966855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931953:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:23.966915Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:23.967020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241592734064306:2048] [node 1] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:23.967090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241592734064306:2048] VirtualId# [2:7587241592734064304:0]
2025-12-24T02:12:23.967104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064306:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:23.967151Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:23.967182Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:23.966886Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:23.966923Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:23.969651Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241592734064308:2269] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.969706Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:23.969868Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241593109931956:2243] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:23.973424Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241592734064308:2269] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:23.973527Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:23.973547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:23.973510Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241593109931956:2243] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:23.973558Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241592734064311:2048]
2025-12-24T02:12:23.973570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064311:2048] [node 1] ICS09 handshake done sender: [2:7587241592734064308:2269] self: [2:7587241592734064309:0] peer: [1:7587241593109931955:0] socket: 17 qp: 42142
2025-12-24T02:12:23.973580Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064311:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:23.973591Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:23.973718Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:23.973735Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:23.973742Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241593109931959:2048]
2025-12-24T02:12:23.973749Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931959:2048] [node 2] ICS09 handshake done sender: [1:7587241593109931956:2243] self: [1:7587241593109931955:0] peer: [2:7587241592734064309:0] socket: 16 qp: 42141
2025-12-24T02:12:23.973756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931959:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:23.973765Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:24.324666Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:24.324777Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241593109931959:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:24.324789Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931959:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:24.324858Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:24.324886Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241593109931959:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:24.325734Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:24.325764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241592734064311:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:24.325770Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064311:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:24.325779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:24.326272Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:24.326352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241592734064311:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:24.326361Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241592734064311:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:24.327597Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241592734064311:2048] [node 1] ICS08 incoming handshake Self# [1:7587241593109931955:0] Peer# [2:7587241592734064309:0] Counter# 3584 LastInputSerial# 92
2025-12-24T02:12:24.329108Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241597404899257:2244] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:24.329256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241593109931959:2048] [node 2] ICS08 incoming handshake Self# [2:7587241592734064309:0] Peer# [1:7587241593109931955:0] Counter# 92 LastInputSerial# 3584
2025-12-24T02:12:24.329272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:24.329283Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931959:2048] [node 2] ICS09 handshake done sender: [1:7587241597404899257:2244] self: [1:7587241593109931955:0] peer: [2:7587241592734064309:0] socket: 22 qp: 42541
2025-12-24T02:12:24.329682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931959:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:24.329703Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:24.331587Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241597029031609:2271] held: no
2025-12-24T02:12:24.331626Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241597029031609:2271] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:24.331704Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:24.331727Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064311:2048] [node 1] ICS09 handshake done sender: [2:7587241597029031609:2271] self: [2:7587241592734064309:0] peer: [1:7587241593109931955:0] socket: 23 qp: 42542
2025-12-24T02:12:24.332135Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064311:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:24.332157Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:24.847749Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:24.847847Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241593109931959:2048] [node 2] ICS07 socket disconnect 22 reason# Debug
2025-12-24T02:12:24.847860Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931959:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:24.847890Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:24.847915Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241593109931959:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:24.848227Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:24.851533Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241597404899262:2246] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:24.853390Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:24.853408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241593109931959:2048] [node 2] ICS09 handshake done sender: [1:7587241597404899262:2246] self: [1:7587241593109931955:0] peer: [2:7587241592734064309:0] socket: 16 qp: 43182
2025-12-24T02:12:24.848436Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241592734064311:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:24.848449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064311:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:24.848461Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:24.848494Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241592734064311:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:24.850029Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241592734064311:2048] [node 1] ICS08 incoming handshake Self# [1:7587241593109931955:0] Peer# [2:7587241592734064309:0] Counter# 10519 LastInputSerial# 262
2025-12-24T02:12:24.853392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241597029031615:2275] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:24.853931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241593109931959:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:24.853957Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:24.854007Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241593109931959:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:24.854013Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241593109931959:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:24.854039Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:24.854053Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241593109931959:2048] [node 2] ICS08 incoming handshake Self# [2:7587241592734064309:0] Peer# [1:7587241593109931955:0] Counter# 262 LastInputSerial# 10519
2025-12-24T02:12:24.854077Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:24.854153Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241593109931959:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:24.854181Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241593109931959:2048] VirtualId# [1:7587241593109931955:0]
2025-12-24T02:12:24.854190Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:24.855201Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:24.855224Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241592734064311:2048] [node 1] ICS09 handshake done sender: [2:7587241597029031615:2275] self: [2:7587241592734064309:0] peer: [1:7587241593109931955:0] socket: 17 qp: 43183
2025-12-24T02:12:24.855569Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241592734064311:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:24.855583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:24.855747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:24.855751Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241592734064311:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:24.855756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241592734064311:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:24.855766Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:24.855802Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:24.855856Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241592734064311:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:24.855858Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241592734064311:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:24.858823Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241597029031619:2277] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:24.859553Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:24.859600Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:24.859605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241592734064311:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:24.859631Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241592734064311:2048] VirtualId# [2:7587241592734064309:0]
2025-12-24T02:12:24.859674Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 63816
TEvUndelivered Cookie# 63817
TEvUndelivered Cookie# 63818
TEvUndelivered Cookie# 63819
TEvUndelivered Cookie# 63820
TEvUndelivered Cookie# 63821
2025-12-24T02:12:24.858730Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241597404899267:2248]) peer tries to resume nonexistent session Self# [2:7587241592734064309:0] Peer# [1:7587241593109931955:0]
2025-12-24T02:12:24.858754Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241597404899267:2248] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:24.866214Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241597404899268:2249] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:24.866277Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:24.867100Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241597029031622:2278] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:24.869636Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241597404899268:2249] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:24.869716Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:24.869740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:24.869749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241597404899271:2048]
2025-12-24T02:12:24.869756Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241597404899271:2048] [node 2] ICS09 handshake done sender: [1:7587241597404899268:2249] self: [1:7587241597404899269:0] peer: [2:7587241597029031621:0] socket: 17 qp: 43195
2025-12-24T02:12:24.869764Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241597404899271:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:24.869779Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:24.869735Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241597029031622:2278] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:24.869867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:24.869878Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:24.869886Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241597029031625:2048]
2025-12-24T02:12:24.869898Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241597029031625:2048] [node 1] ICS09 handshake done sender: [2:7587241597029031622:2278] self: [2:7587241597029031621:0] peer: [1:7587241597404899269:0] socket: 16 qp: 43197
2025-12-24T02:12:24.869905Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241597029031625:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:24.869913Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:25.251387Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:25.251540Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241597029031625:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:25.251556Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241597029031625:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:25.251576Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:25.251594Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241597029031625:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:25.251647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:25.251745Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241597404899271:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:25.251812Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241597404899271:2048] VirtualId# [1:7587241597404899269:0]
2025-12-24T02:12:25.251826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241597404899271:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:25.251865Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:25.251904Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:25.254824Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241601699866569:2251]) peer tries to resume nonexistent session Self# [2:7587241597029031621:0] Peer# [1:7587241597404899269:0]
2025-12-24T02:12:25.254866Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241601699866569:2251] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:25.254962Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241601323998923:2279] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:25.255969Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:25.256012Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:25.256019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241597029031625:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:25.256082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241597029031625:2048] VirtualId# [2:7587241597029031621:0]
2025-12-24T02:12:25.256126Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 64868
TEvUndelivered Cookie# 64869
TEvUndelivered Cookie# 64870
TEvUndelivered Cookie# 64871
TEvUndelivered Cookie# 64872
TEvUndelivered Cookie# 64873
2025-12-24T02:12:25.259731Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241601323998927:2281] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.259749Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241601699866573:2253] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.259776Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:25.259925Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241601699866571:2252] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.265164Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241601323998927:2281] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:25.265218Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241601699866571:2252] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:25.265305Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:25.265347Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:25.265373Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241601323998931:2048]
2025-12-24T02:12:25.265360Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:25.265383Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:25.265384Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241601323998931:2048] [node 1] ICS09 handshake done sender: [2:7587241601323998927:2281] self: [2:7587241601323998929:0] peer: [1:7587241601699866570:0] socket: 16 qp: 43569
2025-12-24T02:12:25.265391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241601699866576:2048]
2025-12-24T02:12:25.265401Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241601699866576:2048] [node 2] ICS09 handshake done sender: [1:7587241601699866571:2252] self: [1:7587241601699866570:0] peer: [2:7587241601323998929:0] socket: 17 qp: 43563
2025-12-24T02:12:25.265404Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241601323998931:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:25.265408Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241601699866576:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:25.265424Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:25.265426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:25.577873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:25.577956Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:25.578051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241601699866576:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:25.578066Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241601699866576:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:25.578079Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:25.578097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241601699866576:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:25.578902Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:25.579134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241601323998931:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:25.579161Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241601323998931:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:25.579179Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:25.579209Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241601323998931:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:25.581100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241601699866576:2048] [node 2] ICS08 incoming handshake Self# [2:7587241601323998929:0] Peer# [1:7587241601699866570:0] Counter# 78 LastInputSerial# 2943
2025-12-24T02:12:25.582172Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241601323998931:2048] [node 1] ICS08 incoming handshake Self# [1:7587241601699866570:0] Peer# [2:7587241601323998929:0] Counter# 2943 LastInputSerial# 78
2025-12-24T02:12:25.582442Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241601699866578:2254] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:25.582570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:25.582589Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241601699866576:2048] [node 2] ICS09 handshake done sender: [1:7587241601699866578:2254] self: [1:7587241601699866570:0] peer: [2:7587241601323998929:0] socket: 20 qp: 43958
2025-12-24T02:12:25.582979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241601699866576:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:25.582997Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:25.585244Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241601323998934:2284] held: no
2025-12-24T02:12:25.585280Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241601323998934:2284] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:25.585438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:25.585458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241601323998931:2048] [node 1] ICS09 handshake done sender: [2:7587241601323998934:2284] self: [2:7587241601323998929:0] peer: [1:7587241601699866570:0] socket: 21 qp: 43970
2025-12-24T02:12:25.585853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241601323998931:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:25.585876Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:25.782364Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241601323998931:2048] [node 1] ICS01 socket: 21 reason# Debug
2025-12-24T02:12:25.782432Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241601323998931:2048] VirtualId# [2:7587241601323998929:0]
2025-12-24T02:12:25.782446Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241601323998931:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:25.782505Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:25.782544Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:25.782575Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:25.782715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241601699866576:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:25.782723Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241601699866576:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:25.782773Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:25.782797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241601699866576:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
2025-12-24T02:12:25.785666Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241601323998941:2287]) peer tries to resume nonexistent session Self# [1:7587241601699866570:0] Peer# [2:7587241601323998929:0]
2025-12-24T02:12:25.785692Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241601323998941:2287] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:25.786291Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241601699866583:2256] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:25.788307Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:25.788353Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:25.788358Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241601699866576:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:25.788395Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241601699866576:2048] VirtualId# [1:7587241601699866570:0]
2025-12-24T02:12:25.788426Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:25.789175Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241601699866585:2257] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.790892Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241601323998942:2288] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.790937Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:25.791098Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241601699866587:2258] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:25.794984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241601323998942:2288] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:25.795038Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241601699866587:2258] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:25.795087Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:25.795108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:25.795126Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241601323998945:2048]
2025-12-24T02:12:25.795134Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241601323998945:2048] [node 1] ICS09 handshake done sender: [2:7587241601323998942:2288] self: [2:7587241601323998943:0] peer: [1:7587241601699866586:0] socket: 20 qp: 44128
2025-12-24T02:12:25.795147Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241601323998945:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:25.795159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:25.795163Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:25.795184Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:25.795190Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241601699866591:2048]
2025-12-24T02:12:25.795200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241601699866591:2048] [node 2] ICS09 handshake done sender: [1:7587241601699866587:2258] self: [1:7587241601699866586:0] peer: [2:7587241601323998943:0] socket: 16 qp: 44127
2025-12-24T02:12:25.795207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241601699866591:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:25.795229Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:26.097120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241601699866591:2048] [node 2] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:26.097182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241601699866591:2048] VirtualId# [1:7587241601699866586:0]
2025-12-24T02:12:26.097199Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241601699866591:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:26.097244Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:26.097274Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:26.097858Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:26.098092Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241601323998945:2048] [node 1] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:26.098105Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241601323998945:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:26.098117Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:26.098141Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241601323998945:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:26.099797Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241605994833889:2259]) peer tries to resume nonexistent session Self# [2:7587241601323998943:0] Peer# [1:7587241601699866586:0]
2025-12-24T02:12:26.099824Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241605994833889:2259] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:26.099886Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241605618966243:2290] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:26.102130Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:26.102171Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:26.102176Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241601323998945:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:26.102212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241601323998945:2048] VirtualId# [2:7587241601323998943:0]
2025-12-24T02:12:26.102260Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 66962
TEvUndelivered Cookie# 66963
TEvUndelivered Cookie# 66964
TEvUndelivered Cookie# 66965
TEvUndelivered Cookie# 66966
TEvUndelivered Cookie# 66967
TEvUndelivered Cookie# 66968
TEvUndelivered Cookie# 66969
TEvUndelivered Cookie# 66970
2025-12-24T02:12:26.105998Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241605994833890:2260] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:26.106063Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:26.106233Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241605618966246:2291] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:26.109831Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241605994833890:2260] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:26.109900Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241605618966246:2291] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:26.109922Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:26.109942Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:26.109954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241605994833893:2048]
2025-12-24T02:12:26.109963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241605994833893:2048] [node 2] ICS09 handshake done sender: [1:7587241605994833890:2260] self: [1:7587241605994833891:0] peer: [2:7587241605618966245:0] socket: 17 qp: 44491
2025-12-24T02:12:26.109973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241605994833893:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:26.109984Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:26.110038Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:26.110053Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:26.110060Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241605618966249:2048]
2025-12-24T02:12:26.110079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241605618966249:2048] [node 1] ICS09 handshake done sender: [2:7587241605618966246:2291] self: [2:7587241605618966245:0] peer: [1:7587241605994833891:0] socket: 16 qp: 44490
2025-12-24T02:12:26.110089Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241605618966249:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:26.110098Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:26.363219Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:26.363311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241605618966249:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:26.363322Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241605618966249:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:26.363362Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:26.363387Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241605618966249:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:26.363596Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:26.363762Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241605994833893:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:26.363767Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241605994833893:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:26.363777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:26.363813Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241605994833893:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:26.365192Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241605994833893:2048] [node 2] ICS08 incoming handshake Self# [2:7587241605618966245:0] Peer# [1:7587241605994833891:0] Counter# 53 LastInputSerial# 2270
2025-12-24T02:12:26.366747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241605618966249:2048] [node 1] ICS08 incoming handshake Self# [1:7587241605994833891:0] Peer# [2:7587241605618966245:0] Counter# 2270 LastInputSerial# 53
2025-12-24T02:12:26.366963Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241605994833895:2262] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:26.367070Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:26.367090Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241605994833893:2048] [node 2] ICS09 handshake done sender: [1:7587241605994833895:2262] self: [1:7587241605994833891:0] peer: [2:7587241605618966245:0] socket: 24 qp: 44794
2025-12-24T02:12:26.367565Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241605994833893:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:26.367589Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:26.369558Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241605618966253:2293] held: no
2025-12-24T02:12:26.369586Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241605618966253:2293] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:26.369664Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:26.369672Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241605618966249:2048] [node 1] ICS09 handshake done sender: [2:7587241605618966253:2293] self: [2:7587241605618966245:0] peer: [1:7587241605994833891:0] socket: 25 qp: 44793
2025-12-24T02:12:26.370142Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241605618966249:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:26.370162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:26.556339Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:26.557442Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:26.557524Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241605618966249:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:12:26.557535Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241605618966249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:26.557546Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:26.557576Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241605618966249:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:26.556847Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] socket: Connection reset by peer
2025-12-24T02:12:26.556966Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241605994833893:2048] [node 2] ICS07 socket disconnect 24 reason# ECONNRESET
2025-12-24T02:12:26.556986Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241605994833893:2048] [node 2] ICS25 shutdown socket, reason# ECONNRESET
2025-12-24T02:12:26.557004Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:26.557033Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241605994833893:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:26.559549Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241605994833893:2048] [node 2] ICS08 incoming handshake Self# [2:7587241605618966245:0] Peer# [1:7587241605994833891:0] Counter# 105 LastInputSerial# 4385
2025-12-24T02:12:26.561165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241605618966249:2048] [node 1] ICS08 incoming handshake Self# [1:7587241605994833891:0] Peer# [2:7587241605618966245:0] Counter# 4385 LastInputSerial# 105
2025-12-24T02:12:26.561413Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241605994833900:2264] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:26.561526Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:26.561543Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241605994833893:2048] [node 2] ICS09 handshake done sender: [1:7587241605994833900:2264] self: [1:7587241605994833891:0] peer: [2:7587241605618966245:0] socket: 20 qp: 45043
2025-12-24T02:12:26.561871Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241605994833893:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:26.561891Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:26.564090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241605618966258:2296] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:26.564163Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241605618966258:2296] held: no
2025-12-24T02:12:26.564176Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:26.564185Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241605618966249:2048] [node 1] ICS09 handshake done sender: [2:7587241605618966258:2296] self: [2:7587241605618966245:0] peer: [1:7587241605994833891:0] socket: 21 qp: 45044
2025-12-24T02:12:26.564704Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241605618966249:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:26.564722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:26.991547Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:26.991643Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:26.991656Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241605618966249:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:26.991647Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:26.991674Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241605618966249:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:26.991694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:26.991726Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:26.991752Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241605994833893:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:26.991763Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241605994833893:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:26.991798Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241605618966249:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:26.991805Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241605618966249:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:26.991818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:26.991851Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241605994833893:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:26.994588Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241605994833893:2048] [node 2] ICS08 incoming handshake Self# [2:7587241605618966245:0] Peer# [1:7587241605994833891:0] Counter# 253 LastInputSerial# 10042
2025-12-24T02:12:26.994682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241605618966249:2048] [node 1] ICS08 incoming handshake Self# [1:7587241605994833891:0] Peer# [2:7587241605618966245:0] Counter# 10042 LastInputSerial# 253
2025-12-24T02:12:26.995439Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241605994833905:2266] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:26.995582Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:26.995605Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241605994833893:2048] [node 2] ICS09 handshake done sender: [1:7587241605994833905:2266] self: [1:7587241605994833891:0] peer: [2:7587241605618966245:0] socket: 17 qp: 45390
2025-12-24T02:12:26.996144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241605994833893:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:26.996173Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:26.997787Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241605618966263:2299] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:26.997885Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241605618966263:2299] held: no
2025-12-24T02:12:26.997900Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:26.997925Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241605618966249:2048] [node 1] ICS09 handshake done sender: [2:7587241605618966263:2299] self: [2:7587241605618966245:0] peer: [1:7587241605994833891:0] socket: 24 qp: 45395
2025-12-24T02:12:26.998557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241605618966249:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:26.998583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:27.130513Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:27.130572Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:27.130661Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241605994833893:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:27.130671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241605994833893:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:27.130680Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:27.130793Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241605994833893:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:27.131580Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:27.131748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241605618966249:2048] [node 1] ICS07 socket disconnect 24 reason# EndOfStream
2025-12-24T02:12:27.131760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241605618966249:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:27.131777Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:27.131807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241605618966249:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:27.133057Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241605618966249:2048] [node 1] ICS08 incoming handshake Self# [1:7587241605994833891:0] Peer# [2:7587241605618966245:0] Counter# 11863 LastInputSerial# 299
2025-12-24T02:12:27.133398Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241610289801206:2268] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:27.133493Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:27.133507Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241605994833893:2048] [node 2] ICS09 handshake done sender: [1:7587241610289801206:2268] self: [1:7587241605994833891:0] peer: [2:7587241605618966245:0] socket: 16 qp: 45499
2025-12-24T02:12:27.133961Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241605994833893:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:27.133977Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:27.134271Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241605994833893:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:27.134279Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241605994833893:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:27.134307Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:27.134325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241605994833893:2048] [node 2] ICS08 incoming handshake Self# [2:7587241605618966245:0] Peer# [1:7587241605994833891:0] Counter# 299 LastInputSerial# 11863
2025-12-24T02:12:27.134337Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:27.134394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241605994833893:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:27.134421Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241605994833893:2048] VirtualId# [1:7587241605994833891:0]
2025-12-24T02:12:27.134435Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:27.135853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241609913933563:2302] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:27.135979Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:27.135994Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241605618966249:2048] [node 1] ICS09 handshake done sender: [2:7587241609913933563:2302] self: [2:7587241605618966245:0] peer: [1:7587241605994833891:0] socket: 20 qp: 45500
2025-12-24T02:12:27.136378Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241605618966249:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:27.136406Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:27.136557Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:27.136570Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241605618966249:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:27.136574Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241605618966249:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:27.136583Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:27.136611Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:27.136644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241605618966249:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:27.136651Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241605618966249:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:27.139151Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241610289801212:2271]) peer tries to resume nonexistent session Self# [2:7587241605618966245:0] Peer# [1:7587241605994833891:0]
2025-12-24T02:12:27.139174Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241610289801212:2271] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:27.139334Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241609913933568:2304] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:27.140031Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:27.140069Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:27.140075Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241605618966249:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:27.140103Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241605618966249:2048] VirtualId# [2:7587241605618966245:0]
2025-12-24T02:12:27.140144Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 69895
TEvUndelivered Cookie# 69896
TEvUndelivered Cookie# 69897
TEvUndelivered Cookie# 69898
TEvUndelivered Cookie# 69899
TEvUndelivered Cookie# 69900
2025-12-24T02:12:27.143442Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241610289801213:2272] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:27.143494Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:27.144377Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241609913933571:2305] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:27.148091Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241610289801213:2272] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:27.148149Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241609913933571:2305] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:27.148178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:27.148195Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:27.148202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241610289801216:2048]
2025-12-24T02:12:27.148207Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241610289801216:2048] [node 2] ICS09 handshake done sender: [1:7587241610289801213:2272] self: [1:7587241610289801214:0] peer: [2:7587241609913933570:0] socket: 17 qp: 45512
2025-12-24T02:12:27.148214Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241610289801216:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:27.148223Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:27.148301Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:27.148315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:27.148323Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241609913933574:2048]
2025-12-24T02:12:27.148329Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241609913933574:2048] [node 1] ICS09 handshake done sender: [2:7587241609913933571:2305] self: [2:7587241609913933570:0] peer: [1:7587241610289801214:0] socket: 16 qp: 45511
2025-12-24T02:12:27.148336Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241609913933574:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:27.148344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:27.675560Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:27.675701Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241610289801216:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:27.675715Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241610289801216:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:27.675722Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:27.675742Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241610289801216:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:27.678288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241610289801216:2048] [node 2] ICS08 incoming handshake Self# [2:7587241609913933570:0] Peer# [1:7587241610289801214:0] Counter# 187 LastInputSerial# 6293
2025-12-24T02:12:27.674836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:27.674946Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241609913933574:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:27.674952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241609913933574:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:27.675019Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:27.675044Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241609913933574:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:27.679054Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241609913933574:2048] [node 1] ICS08 incoming handshake Self# [1:7587241610289801214:0] Peer# [2:7587241609913933570:0] Counter# 6293 LastInputSerial# 187
2025-12-24T02:12:27.679347Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241610289801218:2274] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:27.679705Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:27.679724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241610289801216:2048] [node 2] ICS09 handshake done sender: [1:7587241610289801218:2274] self: [1:7587241610289801214:0] peer: [2:7587241609913933570:0] socket: 24 qp: 45844
2025-12-24T02:12:27.680292Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241610289801216:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:27.680316Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:27.683288Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241609913933578:2307] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:27.683395Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241609913933578:2307] held: no
2025-12-24T02:12:27.683411Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:27.683424Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241609913933574:2048] [node 1] ICS09 handshake done sender: [2:7587241609913933578:2307] self: [2:7587241609913933570:0] peer: [1:7587241610289801214:0] socket: 25 qp: 45847
2025-12-24T02:12:27.684019Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241609913933574:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:27.684040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:27.942918Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:27.943020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241609913933574:2048] [node 1] ICS07 socket disconnect 25 reason# Debug
2025-12-24T02:12:27.943027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241609913933574:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:27.943063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:27.943084Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241609913933574:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:27.945176Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:27.945291Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241610289801216:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-12-24T02:12:27.945305Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241610289801216:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:27.945356Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:27.945377Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241610289801216:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:27.951342Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241610289801216:2048] [node 2] ICS08 incoming handshake Self# [2:7587241609913933570:0] Peer# [1:7587241610289801214:0] Counter# 282 LastInputSerial# 9684
2025-12-24T02:12:27.955170Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241610289801224:2277] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:27.956152Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:27.956180Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241610289801216:2048] [node 2] ICS09 handshake done sender: [1:7587241610289801224:2277] self: [1:7587241610289801214:0] peer: [2:7587241609913933570:0] socket: 20 qp: 46016
2025-12-24T02:12:27.954838Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241609913933574:2048] [node 1] ICS08 incoming handshake Self# [1:7587241610289801214:0] Peer# [2:7587241609913933570:0] Counter# 9684 LastInputSerial# 282
2025-12-24T02:12:27.957538Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241609913933583:2310] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:27.958252Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:27.958265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241609913933574:2048] [node 1] ICS09 handshake done sender: [2:7587241609913933583:2310] self: [2:7587241609913933570:0] peer: [1:7587241610289801214:0] socket: 21 qp: 46013
2025-12-24T02:12:27.956864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241610289801216:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:27.956892Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:27.959296Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241609913933574:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:27.959325Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:28.355960Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:28.356033Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:28.356118Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241610289801216:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:28.356128Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241610289801216:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:28.356137Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:28.356154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241610289801216:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:28.356765Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:28.356967Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241609913933574:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:28.356979Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241609913933574:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:28.356990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:28.357020Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241609913933574:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:28.359802Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241609913933574:2048] [node 1] ICS08 incoming handshake Self# [1:7587241610289801214:0] Peer# [2:7587241609913933570:0] Counter# 15667 LastInputSerial# 444
2025-12-24T02:12:28.359827Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241610289801216:2048] [node 2] ICS08 incoming handshake Self# [2:7587241609913933570:0] Peer# [1:7587241610289801214:0] Counter# 444 LastInputSerial# 15667
2025-12-24T02:12:28.360000Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241614584768524:2278] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:28.360107Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:28.360126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241610289801216:2048] [node 2] ICS09 handshake done sender: [1:7587241614584768524:2278] self: [1:7587241610289801214:0] peer: [2:7587241609913933570:0] socket: 16 qp: 46353
2025-12-24T02:12:28.360514Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241610289801216:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:28.360535Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:28.362292Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241614208900883:2313] held: no
2025-12-24T02:12:28.362331Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241614208900883:2313] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:28.362383Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:28.362398Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241609913933574:2048] [node 1] ICS09 handshake done sender: [2:7587241614208900883:2313] self: [2:7587241609913933570:0] peer: [1:7587241610289801214:0] socket: 17 qp: 46355
2025-12-24T02:12:28.362909Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241609913933574:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:28.362929Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:28.533046Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241609913933574:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:28.533082Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241609913933574:2048] VirtualId# [2:7587241609913933570:0]
2025-12-24T02:12:28.533091Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241609913933574:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:28.533116Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:28.533135Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:28.533774Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:28.533919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241610289801216:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:28.533931Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241610289801216:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:28.533940Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:28.533965Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241610289801216:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:28.535513Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241614208900887:2315]) peer tries to resume nonexistent session Self# [1:7587241610289801214:0] Peer# [2:7587241609913933570:0]
2025-12-24T02:12:28.535544Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241614208900887:2315] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:28.535659Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241614584768529:2280] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:28.536187Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:28.536221Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:28.536226Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241610289801216:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:28.536244Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241610289801216:2048] VirtualId# [1:7587241610289801214:0]
2025-12-24T02:12:28.536261Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:28.539635Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241614208900888:2316] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:28.539695Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:28.539873Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241614584768532:2281] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:28.541782Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241614208900888:2316] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:28.541848Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:28.541867Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:28.541876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241614208900891:2048]
2025-12-24T02:12:28.541819Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241614584768532:2281] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:28.541883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241614208900891:2048] [node 1] ICS09 handshake done sender: [2:7587241614208900888:2316] self: [2:7587241614208900889:0] peer: [1:7587241614584768531:0] socket: 17 qp: 46505
2025-12-24T02:12:28.541897Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241614208900891:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:28.541912Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:28.541975Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:28.541989Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:28.541996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241614584768535:2048]
2025-12-24T02:12:28.542006Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241614584768535:2048] [node 2] ICS09 handshake done sender: [1:7587241614584768532:2281] self: [1:7587241614584768531:0] peer: [2:7587241614208900889:0] socket: 16 qp: 46504
2025-12-24T02:12:28.542017Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241614584768535:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:28.542028Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:29.045212Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:29.045303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:29.045417Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241614584768535:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:29.045429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241614584768535:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:29.045465Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:29.045486Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241614584768535:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:29.046469Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:29.046567Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241614208900891:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:29.046573Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241614208900891:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:29.046621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:29.046643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241614208900891:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:29.048431Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241614208900891:2048] [node 1] ICS08 incoming handshake Self# [1:7587241614584768531:0] Peer# [2:7587241614208900889:0] Counter# 6594 LastInputSerial# 227
2025-12-24T02:12:29.048681Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241618879735833:2282] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:29.048800Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:29.048817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241614584768535:2048] [node 2] ICS09 handshake done sender: [1:7587241618879735833:2282] self: [1:7587241614584768531:0] peer: [2:7587241614208900889:0] socket: 22 qp: 46892
2025-12-24T02:12:29.049170Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241614584768535:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:29.049187Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:29.049864Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241614584768535:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:29.049874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241614584768535:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:29.049901Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:29.049914Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241614584768535:2048] [node 2] ICS08 incoming handshake Self# [2:7587241614208900889:0] Peer# [1:7587241614584768531:0] Counter# 227 LastInputSerial# 6594
2025-12-24T02:12:29.049927Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:29.050004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241614584768535:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:29.050033Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241614584768535:2048] VirtualId# [1:7587241614584768531:0]
2025-12-24T02:12:29.050084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:29.051670Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241618503868190:2319] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:29.051819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:29.051835Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241614208900891:2048] [node 1] ICS09 handshake done sender: [2:7587241618503868190:2319] self: [2:7587241614208900889:0] peer: [1:7587241614584768531:0] socket: 23 qp: 46896
2025-12-24T02:12:29.052218Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241614208900891:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:29.052237Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:29.052359Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:29.052370Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241614208900891:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:29.052376Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241614208900891:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:29.052387Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:29.052432Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:29.052485Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241614208900891:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:29.052492Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241614208900891:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:29.055126Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241618879735839:2285]) peer tries to resume nonexistent session Self# [2:7587241614208900889:0] Peer# [1:7587241614584768531:0]
2025-12-24T02:12:29.055149Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241618879735839:2285] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:29.055221Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241618503868195:2321] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:29.056462Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:29.056502Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:29.056512Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241614208900891:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:29.056538Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241614208900891:2048] VirtualId# [2:7587241614208900889:0]
2025-12-24T02:12:29.056588Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 75958
TEvUndelivered Cookie# 75959
TEvUndelivered Cookie# 75960
TEvUndelivered Cookie# 75961
TEvUndelivered Cookie# 75962
TEvUndelivered Cookie# 75963
TEvUndelivered Cookie# 75964
TEvUndelivered Cookie# 75965
2025-12-24T02:12:29.060429Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241618879735840:2286] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:29.060484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:29.061265Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241618503868198:2322] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:29.063235Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241618879735840:2286] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:29.063294Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241618503868198:2322] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:29.063337Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:29.063357Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:29.063370Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241618879735843:2048]
2025-12-24T02:12:29.063381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241618879735843:2048] [node 2] ICS09 handshake done sender: [1:7587241618879735840:2286] self: [1:7587241618879735841:0] peer: [2:7587241618503868197:0] socket: 17 qp: 46904
2025-12-24T02:12:29.063394Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241618879735843:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:29.063417Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:29.063417Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:29.063434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:29.063440Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241618503868201:2048]
2025-12-24T02:12:29.063452Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241618503868201:2048] [node 1] ICS09 handshake done sender: [2:7587241618503868198:2322] self: [2:7587241618503868197:0] peer: [1:7587241618879735841:0] socket: 16 qp: 46903
2025-12-24T02:12:29.063458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241618503868201:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:29.063467Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:29.605657Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241618879735843:2048] [node 2] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:29.605722Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241618879735843:2048] VirtualId# [1:7587241618879735841:0]
2025-12-24T02:12:29.605733Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241618879735843:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:29.605778Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:29.605806Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:29.605784Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:29.606034Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241618503868201:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:29.606049Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241618503868201:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:29.606065Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:29.606088Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241618503868201:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:29.608948Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241618879735847:2289]) peer tries to resume nonexistent session Self# [2:7587241618503868197:0] Peer# [1:7587241618879735841:0]
2025-12-24T02:12:29.608977Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241618879735847:2289] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:29.609020Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241618503868203:2323] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:29.609508Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:29.609539Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:29.609547Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241618503868201:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:29.609583Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241618503868201:2048] VirtualId# [2:7587241618503868197:0]
2025-12-24T02:12:29.609626Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 77988
TEvUndelivered Cookie# 77989
TEvUndelivered Cookie# 77990
2025-12-24T02:12:29.612381Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241618503868205:2324] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:29.612433Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:29.612608Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241618879735846:2288] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:29.615778Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241618879735850:2290] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:29.617446Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241618503868205:2324] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:29.617525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:29.617504Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241618879735846:2288] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:29.617547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:29.617562Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241618503868211:2048]
2025-12-24T02:12:29.617572Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241618503868211:2048] [node 1] ICS09 handshake done sender: [2:7587241618503868205:2324] self: [2:7587241618503868208:0] peer: [1:7587241618879735845:0] socket: 23 qp: 47400
2025-12-24T02:12:29.617581Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241618503868211:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:29.617594Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:29.617633Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:29.617659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:29.617666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241618879735852:2048]
2025-12-24T02:12:29.617676Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241618879735852:2048] [node 2] ICS09 handshake done sender: [1:7587241618879735846:2288] self: [1:7587241618879735845:0] peer: [2:7587241618503868208:0] socket: 21 qp: 47399
2025-12-24T02:12:29.617682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241618879735852:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:29.617690Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:30.065418Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241618503868211:2048] [node 1] ICS01 socket: 23 reason# Debug
2025-12-24T02:12:30.065505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241618503868211:2048] VirtualId# [2:7587241618503868208:0]
2025-12-24T02:12:30.065521Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241618503868211:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:30.065569Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:30.065603Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:30.065923Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
TEvNodeDisconnected
2025-12-24T02:12:30.066107Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241618879735852:2048] [node 2] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:30.066120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241618879735852:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:30.066131Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:30.066155Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241618879735852:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:30.067624Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241622798835509:2327]) peer tries to resume nonexistent session Self# [1:7587241618879735845:0] Peer# [2:7587241618503868208:0]
2025-12-24T02:12:30.067648Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241622798835509:2327] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:30.067770Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241623174703150:2291] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:30.068873Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:30.068919Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:30.068927Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241618879735852:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:30.068972Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241618879735852:2048] VirtualId# [1:7587241618879735845:0]
2025-12-24T02:12:30.069001Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:30.073137Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241622798835510:2328] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:30.073207Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:30.073418Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241623174703153:2292] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:30.081278Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241622798835510:2328] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:30.081403Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:30.081428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:30.081442Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241622798835513:2048]
2025-12-24T02:12:30.081391Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241623174703153:2292] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:30.081459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241622798835513:2048] [node 1] ICS09 handshake done sender: [2:7587241622798835510:2328] self: [2:7587241622798835511:0] peer: [1:7587241623174703152:0] socket: 17 qp: 47880
2025-12-24T02:12:30.081472Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241622798835513:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:30.081492Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:30.081574Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:30.081593Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:30.081603Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241623174703156:2048]
2025-12-24T02:12:30.081615Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241623174703156:2048] [node 2] ICS09 handshake done sender: [1:7587241623174703153:2292] self: [1:7587241623174703152:0] peer: [2:7587241622798835511:0] socket: 16 qp: 47879
2025-12-24T02:12:30.081628Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241623174703156:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:30.081639Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:30.524096Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:30.524197Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241623174703156:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:30.524216Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241623174703156:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:30.524240Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:30.524264Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241623174703156:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:30.525648Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:30.525765Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241622798835513:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:30.525782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241622798835513:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:30.525841Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:30.525866Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241622798835513:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:30.527302Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241622798835513:2048] [node 1] ICS08 incoming handshake Self# [1:7587241623174703152:0] Peer# [2:7587241622798835511:0] Counter# 5215 LastInputSerial# 144
2025-12-24T02:12:30.529029Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241623174703158:2293] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:30.529229Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241623174703156:2048] [node 2] ICS08 incoming handshake Self# [2:7587241622798835511:0] Peer# [1:7587241623174703152:0] Counter# 144 LastInputSerial# 5215
2025-12-24T02:12:30.529245Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:30.529340Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241623174703156:2048] [node 2] ICS09 handshake done sender: [1:7587241623174703158:2293] self: [1:7587241623174703152:0] peer: [2:7587241622798835511:0] socket: 22 qp: 48400
2025-12-24T02:12:30.529705Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241623174703156:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:30.529727Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:30.530400Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241622798835516:2331] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:30.530995Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:30.531011Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241622798835513:2048] [node 1] ICS09 handshake done sender: [2:7587241622798835516:2331] self: [2:7587241622798835511:0] peer: [1:7587241623174703152:0] socket: 23 qp: 48402
2025-12-24T02:12:30.531363Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241622798835513:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:30.531380Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:30.531436Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:422: Proxy [2:7587241498244783284:2048] [node 1] ICP27 obsolete handshake fail ignored
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:31.121635Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:31.121730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:31.121832Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241623174703156:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-12-24T02:12:31.121845Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241623174703156:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:31.121857Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:31.121887Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241623174703156:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:31.122495Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:31.122753Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241622798835513:2048] [node 1] ICS07 socket disconnect 23 reason# EndOfStream
2025-12-24T02:12:31.122773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241622798835513:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:31.122796Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:31.122841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241622798835513:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:31.124188Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241622798835513:2048] [node 1] ICS08 incoming handshake Self# [1:7587241623174703152:0] Peer# [2:7587241622798835511:0] Counter# 13808 LastInputSerial# 360
2025-12-24T02:12:31.124807Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241627469670459:2295] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:31.124955Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:31.124976Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241623174703156:2048] [node 2] ICS09 handshake done sender: [1:7587241627469670459:2295] self: [1:7587241623174703152:0] peer: [2:7587241622798835511:0] socket: 16 qp: 48994
2025-12-24T02:12:31.125413Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241623174703156:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:31.125434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:31.125677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241623174703156:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:31.125688Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241623174703156:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:31.125719Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:31.125748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241623174703156:2048] [node 2] ICS08 incoming handshake Self# [2:7587241622798835511:0] Peer# [1:7587241623174703152:0] Counter# 360 LastInputSerial# 13808
2025-12-24T02:12:31.125762Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:31.125843Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241623174703156:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:31.125878Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241623174703156:2048] VirtualId# [1:7587241623174703152:0]
2025-12-24T02:12:31.125896Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:31.127462Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241627093802817:2334] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:31.127625Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:31.127644Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241622798835513:2048] [node 1] ICS09 handshake done sender: [2:7587241627093802817:2334] self: [2:7587241622798835511:0] peer: [1:7587241623174703152:0] socket: 17 qp: 48996
2025-12-24T02:12:31.128110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241622798835513:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:31.128129Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:31.128303Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:31.128317Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241622798835513:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:31.128331Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241622798835513:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:31.128344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:31.128389Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:31.128466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241622798835513:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:31.128474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241622798835513:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:31.130985Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241627469670465:2298]) peer tries to resume nonexistent session Self# [2:7587241622798835511:0] Peer# [1:7587241623174703152:0]
2025-12-24T02:12:31.131014Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241627469670465:2298] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:31.131107Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241627093802822:2336] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:31.131712Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:31.131744Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:31.131747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241622798835513:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:31.131767Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241622798835513:2048] VirtualId# [2:7587241622798835511:0]
2025-12-24T02:12:31.131818Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 82468
TEvUndelivered Cookie# 82469
TEvUndelivered Cookie# 82470
TEvUndelivered Cookie# 82471
TEvUndelivered Cookie# 82472
TEvUndelivered Cookie# 82473
TEvUndelivered Cookie# 82474
2025-12-24T02:12:31.134129Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241627469670466:2299] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.134178Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:31.134964Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241627093802825:2337] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.136941Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241627469670466:2299] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:31.137011Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241627093802825:2337] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:31.137041Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:31.137063Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:31.137075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241627469670469:2048]
2025-12-24T02:12:31.137101Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241627469670469:2048] [node 2] ICS09 handshake done sender: [1:7587241627469670466:2299] self: [1:7587241627469670467:0] peer: [2:7587241627093802824:0] socket: 17 qp: 49015
2025-12-24T02:12:31.137110Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241627469670469:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:31.137127Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:31.137156Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:31.137174Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:31.137182Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241627093802828:2048]
2025-12-24T02:12:31.137193Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241627093802828:2048] [node 1] ICS09 handshake done sender: [2:7587241627093802825:2337] self: [2:7587241627093802824:0] peer: [1:7587241627469670467:0] socket: 16 qp: 49013
2025-12-24T02:12:31.137200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241627093802828:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:31.137212Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:31.429730Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:31.429834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241627469670469:2048] [node 2] ICS07 socket disconnect 17 reason# Debug
2025-12-24T02:12:31.429846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241627469670469:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:31.429884Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:31.429910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241627469670469:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:31.430837Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:31.431146Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241627093802828:2048] [node 1] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:31.431165Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241627093802828:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:31.431183Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:31.431242Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241627093802828:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:31.433077Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241627093802828:2048] [node 1] ICS08 incoming handshake Self# [1:7587241627469670467:0] Peer# [2:7587241627093802824:0] Counter# 2838 LastInputSerial# 67
2025-12-24T02:12:31.433406Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241627469670471:2301] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:31.433602Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:31.433619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241627469670469:2048] [node 2] ICS09 handshake done sender: [1:7587241627469670471:2301] self: [1:7587241627469670467:0] peer: [2:7587241627093802824:0] socket: 22 qp: 49300
2025-12-24T02:12:31.433973Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241627469670469:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:31.433992Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:31.434325Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241627469670469:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:31.434335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241627469670469:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:31.434364Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:31.434383Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241627469670469:2048] [node 2] ICS08 incoming handshake Self# [2:7587241627093802824:0] Peer# [1:7587241627469670467:0] Counter# 67 LastInputSerial# 2838
2025-12-24T02:12:31.434396Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:31.434495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241627469670469:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:31.434524Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241627469670469:2048] VirtualId# [1:7587241627469670467:0]
2025-12-24T02:12:31.434540Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:31.435984Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241627093802831:2339] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:31.436128Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:31.436144Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241627093802828:2048] [node 1] ICS09 handshake done sender: [2:7587241627093802831:2339] self: [2:7587241627093802824:0] peer: [1:7587241627469670467:0] socket: 23 qp: 49306
2025-12-24T02:12:31.436640Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241627093802828:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:31.436657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:31.436836Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:31.436848Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241627093802828:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:31.436853Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241627093802828:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:31.436860Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:31.436895Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:31.436952Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241627093802828:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:31.436959Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241627093802828:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:31.439496Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241627469670477:2304]) peer tries to resume nonexistent session Self# [2:7587241627093802824:0] Peer# [1:7587241627469670467:0]
2025-12-24T02:12:31.439522Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241627469670477:2304] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:31.439624Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241627093802836:2341] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:31.440324Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:31.440361Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:31.440369Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241627093802828:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:31.440391Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241627093802828:2048] VirtualId# [2:7587241627093802824:0]
2025-12-24T02:12:31.440423Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 83149
TEvUndelivered Cookie# 83150
TEvUndelivered Cookie# 83151
TEvUndelivered Cookie# 83152
TEvUndelivered Cookie# 83153
TEvUndelivered Cookie# 83154
2025-12-24T02:12:31.442335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241627469670478:2305] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.442379Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:31.443253Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241627093802839:2342] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.450609Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241627469670478:2305] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:31.450684Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241627093802839:2342] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:31.450740Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:31.450762Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:31.450775Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241627469670481:2048]
2025-12-24T02:12:31.450786Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241627469670481:2048] [node 2] ICS09 handshake done sender: [1:7587241627469670478:2305] self: [1:7587241627469670479:0] peer: [2:7587241627093802838:0] socket: 17 qp: 49323
2025-12-24T02:12:31.450797Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241627469670481:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:31.450820Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:31.450850Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:31.450869Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:31.450876Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241627093802842:2048]
2025-12-24T02:12:31.450883Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241627093802842:2048] [node 1] ICS09 handshake done sender: [2:7587241627093802839:2342] self: [2:7587241627093802838:0] peer: [1:7587241627469670479:0] socket: 16 qp: 49322
2025-12-24T02:12:31.450890Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241627093802842:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:31.450899Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:31.696862Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:31.696963Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241627093802842:2048] [node 1] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:31.696975Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241627093802842:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:31.697012Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:31.697038Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241627093802842:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:31.697049Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:31.697129Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241627469670481:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:31.697177Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241627469670481:2048] VirtualId# [1:7587241627469670479:0]
2025-12-24T02:12:31.697189Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241627469670481:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:31.697204Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:31.697241Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:31.700773Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241627469670483:2307]) peer tries to resume nonexistent session Self# [2:7587241627093802838:0] Peer# [1:7587241627469670479:0]
2025-12-24T02:12:31.700811Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241627469670483:2307] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:31.700898Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241627093802844:2343] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:31.701626Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:31.701666Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:31.701679Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241627093802842:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:31.701757Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241627093802842:2048] VirtualId# [2:7587241627093802838:0]
2025-12-24T02:12:31.701802Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:31.705847Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241627093802846:2344] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.705905Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:31.707021Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241627469670485:2308] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.708824Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241627469670487:2309] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:31.712090Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241627093802846:2344] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:31.712148Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241627469670485:2308] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:31.712194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:31.712220Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:31.712234Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241627093802852:2048]
2025-12-24T02:12:31.712250Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241627093802852:2048] [node 1] ICS09 handshake done sender: [2:7587241627093802846:2344] self: [2:7587241627093802850:0] peer: [1:7587241627469670484:0] socket: 23 qp: 49640
2025-12-24T02:12:31.712267Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241627093802852:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:31.712269Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:31.712286Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:31.712292Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:31.712299Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241627469670490:2048]
2025-12-24T02:12:31.712306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241627469670490:2048] [node 2] ICS09 handshake done sender: [1:7587241627469670485:2308] self: [1:7587241627469670484:0] peer: [2:7587241627093802850:0] socket: 21 qp: 49639
2025-12-24T02:12:31.712313Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241627469670490:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:31.712320Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:32.190910Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241627469670490:2048] [node 2] ICS01 socket: 21 reason# Debug
2025-12-24T02:12:32.190968Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241627469670490:2048] VirtualId# [1:7587241627469670484:0]
2025-12-24T02:12:32.190977Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241627469670490:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:32.191005Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:32.191040Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:32.191111Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:32.191133Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241627093802852:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:32.191139Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241627093802852:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:32.191162Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:32.191205Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:32.191297Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241627093802852:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:32.191310Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241627093802852:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:32.193044Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241631764637788:2310]) peer tries to resume nonexistent session Self# [2:7587241627093802850:0] Peer# [1:7587241627469670484:0]
2025-12-24T02:12:32.193071Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241631764637788:2310] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:32.193209Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241631388770150:2347] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:32.193686Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:32.193717Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:32.193724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241627093802852:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:32.193749Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241627093802852:2048] VirtualId# [2:7587241627093802850:0]
2025-12-24T02:12:32.193781Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 85089
TEvUndelivered Cookie# 85090
TEvUndelivered Cookie# 85091
TEvUndelivered Cookie# 85092
TEvUndelivered Cookie# 85093
TEvUndelivered Cookie# 85094
2025-12-24T02:12:32.195820Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241631764637789:2311] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:32.195873Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:32.196003Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241631388770153:2348] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:32.199006Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241631764637789:2311] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:32.199095Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:32.199114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:32.199120Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241631764637792:2048]
2025-12-24T02:12:32.199125Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241631764637792:2048] [node 2] ICS09 handshake done sender: [1:7587241631764637789:2311] self: [1:7587241631764637790:0] peer: [2:7587241631388770152:0] socket: 17 qp: 50210
2025-12-24T02:12:32.199131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241631764637792:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:32.199141Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:32.199282Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241631388770153:2348] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:32.199413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:32.199428Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:32.199438Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241631388770156:2048]
2025-12-24T02:12:32.199449Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241631388770156:2048] [node 1] ICS09 handshake done sender: [2:7587241631388770153:2348] self: [2:7587241631388770152:0] peer: [1:7587241631764637790:0] socket: 16 qp: 50209
2025-12-24T02:12:32.199456Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241631388770156:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:32.199469Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:32.427555Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:32.427657Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:32.427671Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241631388770156:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:32.427678Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241631388770156:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:32.427663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:32.427694Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:32.427745Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:32.427760Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241631764637792:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:32.427773Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241631764637792:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:32.427807Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:32.427807Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241631388770156:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:32.427815Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241631388770156:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:32.427829Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241631764637792:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:32.430173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241631764637792:2048] [node 2] ICS08 incoming handshake Self# [2:7587241631388770152:0] Peer# [1:7587241631764637790:0] Counter# 89 LastInputSerial# 2300
2025-12-24T02:12:32.430326Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241631388770156:2048] [node 1] ICS08 incoming handshake Self# [1:7587241631764637790:0] Peer# [2:7587241631388770152:0] Counter# 2300 LastInputSerial# 89
2025-12-24T02:12:32.430554Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241631764637794:2313] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:32.430640Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:32.430654Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241631764637792:2048] [node 2] ICS09 handshake done sender: [1:7587241631764637794:2313] self: [1:7587241631764637790:0] peer: [2:7587241631388770152:0] socket: 22 qp: 50557
2025-12-24T02:12:32.430944Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241631764637792:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:32.430960Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:32.432612Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241631388770160:2350] held: no
2025-12-24T02:12:32.432647Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241631388770160:2350] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:32.432712Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:32.432724Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241631388770156:2048] [node 1] ICS09 handshake done sender: [2:7587241631388770160:2350] self: [2:7587241631388770152:0] peer: [1:7587241631764637790:0] socket: 24 qp: 50560
2025-12-24T02:12:32.433074Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241631388770156:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:32.433089Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:32.848507Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:32.848599Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:32.848612Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241631388770156:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:32.848619Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241631388770156:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:32.848640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:32.848663Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:32.848678Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:32.848830Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241631388770156:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:32.848840Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241631388770156:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:32.848857Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241631764637792:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-12-24T02:12:32.848868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241631764637792:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:32.848927Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:32.848954Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241631764637792:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:32.851100Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241631388770156:2048] [node 1] ICS08 incoming handshake Self# [1:7587241631764637790:0] Peer# [2:7587241631388770152:0] Counter# 8483 LastInputSerial# 240
2025-12-24T02:12:32.851747Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241631764637792:2048] [node 2] ICS08 incoming handshake Self# [2:7587241631388770152:0] Peer# [1:7587241631764637790:0] Counter# 240 LastInputSerial# 8483
2025-12-24T02:12:32.851942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241631764637799:2315] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:32.852090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:32.852108Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241631764637792:2048] [node 2] ICS09 handshake done sender: [1:7587241631764637799:2315] self: [1:7587241631764637790:0] peer: [2:7587241631388770152:0] socket: 20 qp: 51109
2025-12-24T02:12:32.852488Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241631764637792:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:32.852502Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:32.855145Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241631388770165:2353] held: no
2025-12-24T02:12:32.855189Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241631388770165:2353] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:32.855263Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:32.855281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241631388770156:2048] [node 1] ICS09 handshake done sender: [2:7587241631388770165:2353] self: [2:7587241631388770152:0] peer: [1:7587241631764637790:0] socket: 21 qp: 51111
2025-12-24T02:12:32.855792Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241631388770156:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:32.855811Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvClosePeerSocket
2025-12-24T02:12:33.165157Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [2:7587241498244783284:2048] [node 1] ICP34 closed connection by debug command
2025-12-24T02:12:33.165212Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:33.165311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241631388770156:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:33.165324Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241631388770156:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:33.165297Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:33.165346Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:33.165372Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241631388770156:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:33.165381Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241631764637792:2048] [node 2] ICS07 socket disconnect 20 reason# EndOfStream
2025-12-24T02:12:33.165392Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241631764637792:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:33.165434Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:33.165461Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241631764637792:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:33.169491Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241631764637792:2048] [node 2] ICS08 incoming handshake Self# [2:7587241631388770152:0] Peer# [1:7587241631764637790:0] Counter# 349 LastInputSerial# 12984
2025-12-24T02:12:33.169489Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241631388770156:2048] [node 1] ICS08 incoming handshake Self# [1:7587241631764637790:0] Peer# [2:7587241631388770152:0] Counter# 12984 LastInputSerial# 349
2025-12-24T02:12:33.169728Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241636059605100:2317] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:33.169844Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:33.169865Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241631764637792:2048] [node 2] ICS09 handshake done sender: [1:7587241636059605100:2317] self: [1:7587241631764637790:0] peer: [2:7587241631388770152:0] socket: 22 qp: 51446
2025-12-24T02:12:33.170256Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241631764637792:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:33.170281Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:33.172573Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241635683737466:2356] held: no
2025-12-24T02:12:33.172605Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241635683737466:2356] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:33.172666Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:33.172684Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241631388770156:2048] [node 1] ICS09 handshake done sender: [2:7587241635683737466:2356] self: [2:7587241631388770152:0] peer: [1:7587241631764637790:0] socket: 24 qp: 51445
2025-12-24T02:12:33.173247Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241631388770156:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:33.173269Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 1 peerNodeId# 2 TEvClosePeerSocket
2025-12-24T02:12:33.682279Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:861: Proxy [1:7587241498620650951:2048] [node 2] ICP34 closed connection by debug command
2025-12-24T02:12:33.682333Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:33.682346Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:33.682432Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241631764637792:2048] [node 2] ICS07 socket disconnect 22 reason# EndOfStream
2025-12-24T02:12:33.682434Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241631388770156:2048] [node 1] ICS01 socket: 24 reason# EndOfStream
2025-12-24T02:12:33.682443Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241631764637792:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:33.682454Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:33.682473Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241631388770156:2048] VirtualId# [2:7587241631388770152:0]
2025-12-24T02:12:33.682476Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241631764637792:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:33.682487Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241631388770156:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:33.682526Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:33.682557Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:33.686343Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241635683737469:2358]) peer tries to resume nonexistent session Self# [1:7587241631764637790:0] Peer# [2:7587241631388770152:0]
2025-12-24T02:12:33.686374Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241635683737469:2358] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:33.686461Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241636059605105:2319] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:33.687182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:33.687215Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:33.687219Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241631764637792:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:33.687259Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241631764637792:2048] VirtualId# [1:7587241631764637790:0]
2025-12-24T02:12:33.687290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:33.691685Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241636059605109:2321] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.693438Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241635683737473:2360] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.693484Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:33.693673Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241636059605108:2320] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.696176Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241635683737473:2360] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:33.696272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:33.696298Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:33.696312Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241635683737476:2048]
2025-12-24T02:12:33.696319Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241635683737476:2048] [node 1] ICS09 handshake done sender: [2:7587241635683737473:2360] self: [2:7587241635683737474:0] peer: [1:7587241636059605107:0] socket: 21 qp: 52132
2025-12-24T02:12:33.696300Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241636059605108:2320] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:33.696332Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241635683737476:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:33.696345Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:33.696495Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:33.696528Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:33.696540Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241636059605113:2048]
2025-12-24T02:12:33.696546Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241636059605113:2048] [node 2] ICS09 handshake done sender: [1:7587241636059605108:2320] self: [1:7587241636059605107:0] peer: [2:7587241635683737474:0] socket: 20 qp: 52134
2025-12-24T02:12:33.696557Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241636059605113:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:33.696568Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvPoisonSession
2025-12-24T02:12:33.835390Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241636059605113:2048] [node 2] ICS01 socket: 20 reason# Debug
2025-12-24T02:12:33.835449Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241636059605113:2048] VirtualId# [1:7587241636059605107:0]
2025-12-24T02:12:33.835458Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241636059605113:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:33.835486Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:33.835517Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:33.835516Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:33.835631Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241635683737476:2048] [node 1] ICS07 socket disconnect 21 reason# EndOfStream
2025-12-24T02:12:33.835643Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241635683737476:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:33.835687Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:33.835714Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241635683737476:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:33.838121Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241636059605115:2322]) peer tries to resume nonexistent session Self# [2:7587241635683737474:0] Peer# [1:7587241636059605107:0]
2025-12-24T02:12:33.838147Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241636059605115:2322] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:33.838231Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241635683737478:2362] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:33.838965Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:33.838999Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:33.839004Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241635683737476:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:33.839037Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241635683737476:2048] VirtualId# [2:7587241635683737474:0]
2025-12-24T02:12:33.839061Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:33.842294Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241635683737482:2364] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.843801Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:33.843859Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241636059605119:2324] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.843977Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241636059605117:2323] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:33.845994Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241635683737482:2364] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:33.846050Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241636059605117:2323] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:33.846075Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:33.846096Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:33.846109Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241635683737486:2048]
2025-12-24T02:12:33.846115Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241635683737486:2048] [node 1] ICS09 handshake done sender: [2:7587241635683737482:2364] self: [2:7587241635683737484:0] peer: [1:7587241636059605116:0] socket: 17 qp: 52320
2025-12-24T02:12:33.846127Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241635683737486:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:33.846139Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:33.846167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:33.846186Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:33.846194Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241636059605122:2048]
2025-12-24T02:12:33.846200Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241636059605122:2048] [node 2] ICS09 handshake done sender: [1:7587241636059605117:2323] self: [1:7587241636059605116:0] peer: [2:7587241635683737484:0] socket: 16 qp: 52317
2025-12-24T02:12:33.846206Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241636059605122:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:33.846227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 1 peerNodeId# 2 TEvCloseInputSession
2025-12-24T02:12:34.191505Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by debug command
2025-12-24T02:12:34.191592Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241636059605122:2048] [node 2] ICS07 socket disconnect 16 reason# Debug
2025-12-24T02:12:34.191601Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241636059605122:2048] [node 2] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:34.191632Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:34.191650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241636059605122:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:34.191689Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:34.191874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241635683737486:2048] [node 1] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:34.191884Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241635683737486:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:34.191893Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:34.191919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241635683737486:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:34.193260Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241635683737486:2048] [node 1] ICS08 incoming handshake Self# [1:7587241636059605116:0] Peer# [2:7587241635683737484:0] Counter# 3976 LastInputSerial# 114
2025-12-24T02:12:34.194683Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241640354572420:2325] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:34.194828Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:34.194841Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241636059605122:2048] [node 2] ICS09 handshake done sender: [1:7587241640354572420:2325] self: [1:7587241636059605116:0] peer: [2:7587241635683737484:0] socket: 20 qp: 52854
2025-12-24T02:12:34.195234Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241636059605122:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:34.195249Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:34.195275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [1:7587241636059605122:2048] [node 2] ICS13 reestablish connection
2025-12-24T02:12:34.195288Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241636059605122:2048] [node 2] ICS25 shutdown socket, reason# NewSession
2025-12-24T02:12:34.195315Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:34.195338Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241636059605122:2048] [node 2] ICS08 incoming handshake Self# [2:7587241635683737484:0] Peer# [1:7587241636059605116:0] Counter# 114 LastInputSerial# 3976
2025-12-24T02:12:34.195360Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:34.195429Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241636059605122:2048] [node 2] ICS01 socket: -1 reason# EndOfStream
2025-12-24T02:12:34.195451Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241636059605122:2048] VirtualId# [1:7587241636059605116:0]
2025-12-24T02:12:34.195464Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:34.196467Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241639978704785:2367] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:34.196601Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:34.196621Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241635683737486:2048] [node 1] ICS09 handshake done sender: [2:7587241639978704785:2367] self: [2:7587241635683737484:0] peer: [1:7587241636059605116:0] socket: 21 qp: 52856
2025-12-24T02:12:34.197069Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241635683737486:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:34.197087Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:34.197243Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_session.cpp:768: [2 <-> 1] socket: Broken pipe
2025-12-24T02:12:34.197258Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:535: Session [2:7587241635683737486:2048] [node 1] ICS13 reestablish connection
2025-12-24T02:12:34.197265Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241635683737486:2048] [node 1] ICS25 shutdown socket, reason# EPIPE
2025-12-24T02:12:34.197273Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:34.197313Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:34.197352Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241635683737486:2048] [node 1] ICS07 socket disconnect -1 reason# EndOfStream
2025-12-24T02:12:34.197359Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241635683737486:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:34.200475Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [1:7587241498620650951:2048] [node 2] ICP12 (actor [1:7587241640354572426:2328]) peer tries to resume nonexistent session Self# [2:7587241635683737484:0] Peer# [1:7587241636059605116:0]
2025-12-24T02:12:34.200509Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [1:7587241640354572426:2328] [node 2] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:34.200576Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [2:7587241639978704790:2369] [node 1] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake
2025-12-24T02:12:34.201222Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:20066) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:34.201248Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [2:7587241498244783284:2048] [node 1] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:34.201253Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241635683737486:2048] [node 1] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:34.201276Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241635683737486:2048] VirtualId# [2:7587241635683737484:0]
2025-12-24T02:12:34.201344Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
TEvUndelivered Cookie# 91475
TEvUndelivered Cookie# 91476
TEvUndelivered Cookie# 91477
TEvUndelivered Cookie# 91478
TEvUndelivered Cookie# 91479
TEvUndelivered Cookie# 91480
2025-12-24T02:12:34.203290Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241640354572427:2329] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.203334Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241498620650951:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:34.203997Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241639978704793:2370] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.207480Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241640354572427:2329] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:34.207525Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241639978704793:2370] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:34.207564Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241498620650951:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:34.207585Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:34.207599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241640354572430:2048]
2025-12-24T02:12:34.207607Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241640354572430:2048] [node 2] ICS09 handshake done sender: [1:7587241640354572427:2329] self: [1:7587241640354572428:0] peer: [2:7587241639978704792:0] socket: 17 qp: 52877
2025-12-24T02:12:34.207625Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241640354572430:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:34.207637Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:34.207627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241498244783284:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:34.207640Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:34.207645Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241639978704796:2048]
2025-12-24T02:12:34.207650Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241639978704796:2048] [node 1] ICS09 handshake done sender: [2:7587241639978704793:2370] self: [2:7587241639978704792:0] peer: [1:7587241640354572428:0] socket: 16 qp: 52876
2025-12-24T02:12:34.207655Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241639978704796:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:34.207662Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:34.365855Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241639978704796:2048] [node 1] ICS01 socket: 16 reason# Debug
2025-12-24T02:12:34.365915Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241639978704796:2048] VirtualId# [2:7587241639978704792:0]
2025-12-24T02:12:34.365924Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241639978704796:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:34.365965Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:34.366002Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:34.366014Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:34.366126Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241640354572430:2048] [node 2] ICS07 socket disconnect 17 reason# EndOfStream
2025-12-24T02:12:34.366152Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241640354572430:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:34.366214Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:34.366246Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241640354572430:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 91691
TEvUndelivered Cookie# 91692
TEvUndelivered Cookie# 91693
TEvUndelivered Cookie# 91694
TEvUndelivered Cookie# 91695
2025-12-24T02:12:34.368409Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241639978704798:2371]) peer tries to resume nonexistent session Self# [1:7587241640354572428:0] Peer# [2:7587241639978704792:0]
2025-12-24T02:12:34.368445Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241639978704798:2371] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:34.368868Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241640354572432:2331] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:34.369422Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:34.369459Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:34.369466Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241640354572430:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:34.369496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241640354572430:2048] VirtualId# [1:7587241640354572428:0]
2025-12-24T02:12:34.369525Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:34.373615Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241640354572434:2332] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.376780Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241639978704802:2373] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.376831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:34.377017Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241640354572436:2333] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.379590Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241639978704802:2373] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:34.379659Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241640354572436:2333] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:34.379686Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:34.379717Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:34.379727Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241639978704805:2048]
2025-12-24T02:12:34.379736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241639978704805:2048] [node 1] ICS09 handshake done sender: [2:7587241639978704802:2373] self: [2:7587241639978704803:0] peer: [1:7587241640354572435:0] socket: 17 qp: 53093
2025-12-24T02:12:34.379748Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241639978704805:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:34.379761Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:34.379819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:34.379847Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:34.379861Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241640354572440:2048]
2025-12-24T02:12:34.379868Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241640354572440:2048] [node 2] ICS09 handshake done sender: [1:7587241640354572436:2333] self: [1:7587241640354572435:0] peer: [2:7587241639978704803:0] socket: 16 qp: 53090
2025-12-24T02:12:34.379876Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241640354572440:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:34.379885Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:34.804735Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:34.804834Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241639978704805:2048] [node 1] ICS07 socket disconnect 17 reason# Debug
2025-12-24T02:12:34.804846Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241639978704805:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:34.804877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:34.804903Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241639978704805:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:34.805646Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:34.805736Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241640354572440:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:34.805746Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241640354572440:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:34.805789Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:34.805817Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241640354572440:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:34.808131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241640354572440:2048] [node 2] ICS08 incoming handshake Self# [2:7587241639978704803:0] Peer# [1:7587241640354572435:0] Counter# 177 LastInputSerial# 5500
2025-12-24T02:12:34.808154Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241639978704805:2048] [node 1] ICS08 incoming handshake Self# [1:7587241640354572435:0] Peer# [2:7587241639978704803:0] Counter# 5500 LastInputSerial# 177
2025-12-24T02:12:34.808349Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241640354572442:2334] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:34.808581Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:34.808602Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241640354572440:2048] [node 2] ICS09 handshake done sender: [1:7587241640354572442:2334] self: [1:7587241640354572435:0] peer: [2:7587241639978704803:0] socket: 24 qp: 53526
2025-12-24T02:12:34.809079Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241640354572440:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:34.809093Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:34.810815Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241639978704809:2376] held: no
2025-12-24T02:12:34.810853Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241639978704809:2376] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:34.810908Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:34.810919Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241639978704805:2048] [node 1] ICS09 handshake done sender: [2:7587241639978704809:2376] self: [2:7587241639978704803:0] peer: [1:7587241640354572435:0] socket: 25 qp: 53525
2025-12-24T02:12:34.811389Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241639978704805:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:34.811408Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:34.946772Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241639978704805:2048] [node 1] ICS01 socket: 25 reason# Debug
2025-12-24T02:12:34.946819Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241639978704805:2048] VirtualId# [2:7587241639978704803:0]
2025-12-24T02:12:34.946826Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241639978704805:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:34.946853Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:34.946877Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:34.946909Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:34.947016Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241640354572440:2048] [node 2] ICS07 socket disconnect 24 reason# EndOfStream
2025-12-24T02:12:34.947027Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241640354572440:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:34.947077Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:34.947106Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241640354572440:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 93491
TEvUndelivered Cookie# 93492
TEvUndelivered Cookie# 93493
TEvUndelivered Cookie# 93494
TEvUndelivered Cookie# 93495
TEvUndelivered Cookie# 93496
TEvUndelivered Cookie# 93497
TEvUndelivered Cookie# 93498
TEvUndelivered Cookie# 93499
2025-12-24T02:12:34.949976Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241639978704814:2379]) peer tries to resume nonexistent session Self# [1:7587241640354572435:0] Peer# [2:7587241639978704803:0]
2025-12-24T02:12:34.949996Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241639978704814:2379] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:34.950592Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241640354572447:2336] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:34.951182Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:34.951221Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:34.951225Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241640354572440:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:34.951248Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241640354572440:2048] VirtualId# [1:7587241640354572435:0]
2025-12-24T02:12:34.951270Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:34.951919Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241640354572449:2337] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.953464Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241639978704816:2380] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.953496Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:34.953643Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241640354572451:2338] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:34.955429Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241639978704816:2380] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:34.955497Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:34.955518Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:34.955490Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241640354572451:2338] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:34.955525Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241639978704819:2048]
2025-12-24T02:12:34.955533Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241639978704819:2048] [node 1] ICS09 handshake done sender: [2:7587241639978704816:2380] self: [2:7587241639978704817:0] peer: [1:7587241640354572450:0] socket: 17 qp: 53660
2025-12-24T02:12:34.955539Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241639978704819:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:34.955547Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:34.955604Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:34.955621Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:34.955627Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241640354572455:2048]
2025-12-24T02:12:34.955632Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241640354572455:2048] [node 2] ICS09 handshake done sender: [1:7587241640354572451:2338] self: [1:7587241640354572450:0] peer: [2:7587241639978704817:0] socket: 16 qp: 53659
2025-12-24T02:12:34.955638Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241640354572455:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:34.955649Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvPoisonSession
2025-12-24T02:12:35.079051Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241639978704819:2048] [node 1] ICS01 socket: 17 reason# Debug
2025-12-24T02:12:35.079106Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241639978704819:2048] VirtualId# [2:7587241639978704817:0]
2025-12-24T02:12:35.079114Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241639978704819:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:35.079142Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:35.079165Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:12:35.079223Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:35.079335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241640354572455:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:35.079349Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241640354572455:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:35.079425Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:35.079459Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241640354572455:2048] [node 2] ICS15 start handshake
TEvNodeDisconnected
TEvUndelivered Cookie# 93654
TEvUndelivered Cookie# 93655
TEvUndelivered Cookie# 93656
2025-12-24T02:12:35.109428Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:240: Proxy [2:7587241498244783284:2048] [node 1] ICP12 (actor [2:7587241644273672117:2382]) peer tries to resume nonexistent session Self# [1:7587241640354572450:0] Peer# [2:7587241639978704817:0]
2025-12-24T02:12:35.109456Z :INTERCONNECT NOTICE: interconnect_handshake.cpp:1112: Handshake [2:7587241644273672117:2382] [node 1] ICH08 Continuation request rejected by proxy
2025-12-24T02:12:35.109615Z :INTERCONNECT INFO: interconnect_handshake.cpp:1583: Handshake [1:7587241644649539753:2339] [node 2] ICH03 handshake failed, explanation# outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake
2025-12-24T02:12:35.110489Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [1:7587241498620650951:2048] [node 2] ICP25 outgoing handshake failed, temporary: 2 explanation: outgoing handshake Peer# ::1(::1:8121) Peer rejected session continuation handshake incoming: [0:0:0] held: no
2025-12-24T02:12:35.110526Z :INTERCONNECT INFO: interconnect_tcp_proxy.h:481: Proxy [1:7587241498620650951:2048] [node 2] ICP04 terminating current session as we are negotiating a new one
2025-12-24T02:12:35.110531Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241640354572455:2048] [node 2] ICS01 socket: -1 reason# NewSession
2025-12-24T02:12:35.110570Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241498620650951:2048] [node 2] ICP30 unregister session Session# [1:7587241640354572455:2048] VirtualId# [1:7587241640354572450:0]
2025-12-24T02:12:35.110605Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:35.114595Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241644649539755:2340] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:35.115957Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241644273672121:2384] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:35.115996Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241498244783284:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:35.116124Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241644649539757:2341] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:35.119956Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241644273672121:2384] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:35.120014Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241644649539757:2341] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:35.120062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:35.120084Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:35.120093Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241498244783284:2048] [node 1] ICP22 created new session: [2:7587241644273672124:2048]
2025-12-24T02:12:35.120104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241644273672124:2048] [node 1] ICS09 handshake done sender: [2:7587241644273672121:2384] self: [2:7587241644273672122:0] peer: [1:7587241644649539756:0] socket: 21 qp: 53801
2025-12-24T02:12:35.120113Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241644273672124:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:35.120136Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:35.120135Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:35.120159Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:35.120167Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241498620650951:2048] [node 2] ICP22 created new session: [1:7587241644649539761:2048]
2025-12-24T02:12:35.120173Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241644649539761:2048] [node 2] ICS09 handshake done sender: [1:7587241644649539757:2341] self: [1:7587241644649539756:0] peer: [2:7587241644273672122:0] socket: 16 qp: 53798
2025-12-24T02:12:35.120184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241644649539761:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:35.120192Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
Subscribe
TEvNodeConnected
nodeId# 2 peerNodeId# 1 TEvCloseInputSession
2025-12-24T02:12:35.322964Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by debug command
2025-12-24T02:12:35.323120Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241644273672124:2048] [node 1] ICS07 socket disconnect 21 reason# Debug
2025-12-24T02:12:35.323131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241644273672124:2048] [node 1] ICS25 shutdown socket, reason# Debug
2025-12-24T02:12:35.323160Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:35.323184Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241644273672124:2048] [node 1] ICS15 start handshake
2025-12-24T02:12:35.323574Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:35.323682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [1:7587241644649539761:2048] [node 2] ICS07 socket disconnect 16 reason# EndOfStream
2025-12-24T02:12:35.323687Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241644649539761:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:35.323740Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:35.323782Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [1:7587241644649539761:2048] [node 2] ICS15 start handshake
2025-12-24T02:12:35.325677Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [1:7587241644649539761:2048] [node 2] ICS08 incoming handshake Self# [2:7587241644273672122:0] Peer# [1:7587241644649539756:0] Counter# 47 LastInputSerial# 1943
2025-12-24T02:12:35.327031Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:264: Session [2:7587241644273672124:2048] [node 1] ICS08 incoming handshake Self# [1:7587241644649539756:0] Peer# [2:7587241644273672122:0] Counter# 1943 LastInputSerial# 47
2025-12-24T02:12:35.327268Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241644649539763:2342] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:35.327413Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241498620650951:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:35.327437Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241644649539761:2048] [node 2] ICS09 handshake done sender: [1:7587241644649539763:2342] self: [1:7587241644649539756:0] peer: [2:7587241644273672122:0] socket: 24 qp: 53997
2025-12-24T02:12:35.328171Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241644649539761:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:35.328194Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:35.330137Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241644273672128:2387] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:35.330214Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241498244783284:2048] [node 1] ICP25 outgoing handshake failed, temporary: 0 explanation: outgoing handshake Peer# ::1(::1:20066) Socket error# connection unexpectedly closed state# ReceiveResponse processed# 0 remain# 52 incoming: [2:7587241644273672128:2387] held: no
2025-12-24T02:12:35.330225Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241498244783284:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:35.330232Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241644273672124:2048] [node 1] ICS09 handshake done sender: [2:7587241644273672128:2387] self: [2:7587241644273672122:0] peer: [1:7587241644649539756:0] socket: 25 qp: 53996
2025-12-24T02:12:35.330725Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241644273672124:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:35.330748Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
2025-12-24T02:12:35.822151Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:35.822275Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241644273672124:2048] [node 1] ICS01 socket: 25 reason# EndOfStream
2025-12-24T02:12:35.822326Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241498244783284:2048] [node 1] ICP30 unregister session Session# [2:7587241644273672124:2048] VirtualId# [2:7587241644273672122:0]
2025-12-24T02:12:35.822335Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241644273672124:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:35.822400Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:35.822440Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:35.839768Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241644545829440:2048] [node 2] ICP01 ready to work
Subscribe
2025-12-24T02:12:35.843750Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241642765282829:2048] [node 1] ICP01 ready to work
2025-12-24T02:12:35.847878Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241644545829449:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:35.848249Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241644545829440:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:35.848391Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241642765282839:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:35.853095Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241644545829449:2056] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:35.853144Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241642765282839:2056] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:35.853202Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241644545829440:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:35.853227Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:35.853240Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241644545829440:2048] [node 2] ICP22 created new session: [1:7587241644545829452:2048]
2025-12-24T02:12:35.853266Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241644545829452:2048] [node 2] ICS09 handshake done sender: [1:7587241644545829449:2056] self: [1:7587241644545829450:0] peer: [2:7587241642765282838:0] socket: 17 qp: 54476
2025-12-24T02:12:35.853282Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241644545829452:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:35.853272Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241642765282829:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:35.853290Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:35.853297Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241642765282829:2048] [node 1] ICP22 created new session: [2:7587241642765282842:2048]
2025-12-24T02:12:35.853304Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241642765282842:2048] [node 1] ICS09 handshake done sender: [2:7587241642765282839:2056] self: [2:7587241642765282838:0] peer: [1:7587241644545829450:0] socket: 16 qp: 54475
2025-12-24T02:12:35.853311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241642765282842:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:35.853313Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:35.853324Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
TEvNodeConnected
2025-12-24T02:12:35.943909Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [1:7587241644545829440:2048] [node 2] ICP31 proxy http called
2025-12-24T02:12:35.944239Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:622: Proxy [2:7587241642765282829:2048] [node 1] ICP31 proxy http called
2025-12-24T02:12:35.948355Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:35.948474Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241642765282842:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:35.948542Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241642765282829:2048] [node 1] ICP30 unregister session Session# [2:7587241642765282842:2048] VirtualId# [2:7587241642765282838:0]
2025-12-24T02:12:35.948558Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241642765282842:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:35.948613Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:35.948657Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvNodeDisconnected
2025-12-24T02:12:35.961359Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241644074047878:2048] [node 2] ICP01 ready to work
2025-12-24T02:12:35.962831Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241644849638621:2048] [node 1] ICP01 ready to work
2025-12-24T02:12:36.065156Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241648369015183:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:36.065699Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241644074047878:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:36.065924Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241649144605927:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:36.068392Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241648369015183:2056] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:36.068469Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241649144605927:2056] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:36.068505Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241644074047878:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:36.068560Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:36.068575Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241644074047878:2048] [node 2] ICP22 created new session: [1:7587241648369015186:2048]
2025-12-24T02:12:36.068587Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241648369015186:2048] [node 2] ICS09 handshake done sender: [1:7587241648369015183:2056] self: [1:7587241648369015184:0] peer: [2:7587241649144605926:0] socket: 17 qp: 54644
2025-12-24T02:12:36.068600Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241648369015186:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:36.068634Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:36.068641Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241644849638621:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:36.068664Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:36.068674Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241644849638621:2048] [node 1] ICP22 created new session: [2:7587241649144605930:2048]
2025-12-24T02:12:36.068682Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241649144605930:2048] [node 1] ICS09 handshake done sender: [2:7587241649144605927:2056] self: [2:7587241649144605926:0] peer: [1:7587241648369015184:0] socket: 16 qp: 54643
2025-12-24T02:12:36.068690Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241649144605930:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:36.068699Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
RECEIVED TEvTest
2025-12-24T02:12:36.356261Z :INTERCONNECT CRIT: interconnect_tcp_session.cpp:517: Session [2:7587241649144605930:2048] [node 1] ICS31 serialized event Type# 0x7ff00000 is too large
2025-12-24T02:12:36.356281Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241649144605930:2048] [node 1] ICS01 socket: 16 reason# EventTooLarge
2025-12-24T02:12:36.356338Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241644849638621:2048] [node 1] ICP30 unregister session Session# [2:7587241649144605930:2048] VirtualId# [2:7587241649144605926:0]
2025-12-24T02:12:36.356347Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241649144605930:2048] [node 1] ICS25 shutdown socket, reason# EventTooLarge
2025-12-24T02:12:36.356396Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:36.356408Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [1 <-> 2] connection closed by peer
2025-12-24T02:12:36.356552Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [1:7587241648369015186:2048] [node 2] ICS01 socket: 17 reason# EndOfStream
2025-12-24T02:12:36.356599Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [1:7587241644074047878:2048] [node 2] ICP30 unregister session Session# [1:7587241648369015186:2048] VirtualId# [1:7587241648369015184:0]
2025-12-24T02:12:36.356609Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [1:7587241648369015186:2048] [node 2] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:36.356659Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [2] disconnected
2025-12-24T02:12:36.356692Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [2] session destroyed
2025-12-24T02:12:36.360541Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
TEvUndelivered
2025-12-24T02:12:36.379006Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241648369015188:2058] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:36.379089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [1:7587241644074047878:2048] [node 2] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:36.379335Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241649144605933:2057] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:36.383848Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241648369015188:2058] [node 2] ICH04 handshake succeeded
2025-12-24T02:12:36.383954Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [1:7587241644074047878:2048] [node 2] ICP19 incoming handshake succeeded
2025-12-24T02:12:36.383975Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:36.383943Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241649144605933:2057] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:36.383984Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241644074047878:2048] [node 2] ICP22 created new session: [1:7587241648369015191:2048]
2025-12-24T02:12:36.383998Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241648369015191:2048] [node 2] ICS09 handshake done sender: [1:7587241648369015188:2058] self: [1:7587241648369015189:0] peer: [2:7587241649144605932:0] socket: 17 qp: 54883
2025-12-24T02:12:36.384008Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241648369015191:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:36.384020Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:36.384089Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [2:7587241644849638621:2048] [node 1] ICP20 outgoing handshake succeeded
2025-12-24T02:12:36.384108Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:36.384118Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241644849638621:2048] [node 1] ICP22 created new session: [2:7587241649144605936:2048]
2025-12-24T02:12:36.384124Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241649144605936:2048] [node 1] ICS09 handshake done sender: [2:7587241649144605933:2057] self: [2:7587241649144605932:0] peer: [1:7587241648369015189:0] socket: 16 qp: 54871
2025-12-24T02:12:36.384131Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241649144605936:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:36.384147Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
RECEIVED TEvTest
2025-12-24T02:12:36.404747Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:12:36.404831Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241649144605936:2048] [node 1] ICS01 socket: 16 reason# EndOfStream
2025-12-24T02:12:36.404867Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241644849638621:2048] [node 1] ICP30 unregister session Session# [2:7587241649144605936:2048] VirtualId# [2:7587241649144605932:0]
2025-12-24T02:12:36.404874Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241649144605936:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:12:36.404916Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:12:36.404990Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
written 2560000 bytes
written 3437884 bytes
written 4133621 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
written 4195061 bytes
2025-12-24T02:12:50.577397Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [1:7587241710166326015:2048] [node 2] ICP01 ready to work
2025-12-24T02:12:50.580658Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:43: Proxy [2:7587241707147096628:2048] [node 1] ICP01 ready to work
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:12:50.582295Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241707147096637:2056] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18031
2025-12-24T02:12:50.584822Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [2:7587241707147096637:2056] [node 1] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:50.585309Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:220: Proxy [2:7587241707147096628:2048] [node 1] ICP08 No active sessions, becoming PendingConnection
2025-12-24T02:12:50.585433Z :INTERCONNECT WARN: interconnect_handshake.cpp:542: Handshake [1:7587241710166326025:2056] [node 2] ICH09 Neither CompatibilityInfo nor VersionTag of the peer can be validated, accepting by default
2025-12-24T02:12:50.585452Z :INTERCONNECT ERROR: interconnect_handshake.cpp:955: Handshake [1:7587241710166326025:2056] [node 2] ICRDMA Non success qp response from remote side
2025-12-24T02:12:50.585942Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [1:7587241710166326025:2056] [node 2] ICH04 handshake succeeded
No RDMA context found for GID: ::ffff:127.1.0.0
2025-12-24T02:12:50.586036Z :INTERCONNECT WARN: interconnect_handshake.cpp:1390: Handshake [2:7587241707147096639:2057] [node 0] ICRDMA Unable to find verbs context using address 127.1.0.0:18031
2025-12-24T02:12:50.586062Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:339: Proxy [1:7587241710166326015:2048] [node 2] ICP20 outgoing handshake succeeded
2025-12-24T02:12:50.586083Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [2] session created
2025-12-24T02:12:50.586090Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [1:7587241710166326015:2048] [node 2] ICP22 created new session: [1:7587241710166326028:2048]
2025-12-24T02:12:50.586097Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [1:7587241710166326028:2048] [node 2] ICS09 handshake done sender: [1:7587241710166326025:2056] self: [1:7587241710166326024:0] peer: [2:7587241707147096638:0] socket: 24 qp: -1
2025-12-24T02:12:50.586104Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [1:7587241710166326028:2048] [node 2] ICS10 traffic start
2025-12-24T02:12:50.586114Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [2] connected
2025-12-24T02:12:50.586219Z :INTERCONNECT INFO: interconnect_handshake.cpp:414: Handshake [2:7587241707147096637:2056] [node 1] ICH04 handshake succeeded
2025-12-24T02:12:50.586283Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:335: Proxy [2:7587241707147096628:2048] [node 1] ICP19 incoming handshake succeeded
2025-12-24T02:12:50.586297Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:66: [1] session created
2025-12-24T02:12:50.586302Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:378: Proxy [2:7587241707147096628:2048] [node 1] ICP22 created new session: [2:7587241707147096640:2048]
2025-12-24T02:12:50.586306Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:278: Session [2:7587241707147096640:2048] [node 1] ICS09 handshake done sender: [2:7587241707147096637:2056] self: [2:7587241707147096638:0] peer: [1:7587241710166326024:0] socket: 25 qp: -1
2025-12-24T02:12:50.586311Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:303: Session [2:7587241707147096640:2048] [node 1] ICS10 traffic start
2025-12-24T02:12:50.586343Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:329: [1] connected
Updated MaxRTT# 0.006324s
Updated MaxRTT# 0.018212s
Updated MaxRTT# 0.018239s
2025-12-24T02:13:00.590248Z :INTERCONNECT_NETWORK NOTICE: interconnect_tcp_input_session.cpp:975: [2 <-> 1] connection closed by peer
2025-12-24T02:13:00.590414Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:552: Session [2:7587241707147096640:2048] [node 1] ICS07 socket disconnect 25 reason# EndOfStream
2025-12-24T02:13:00.590428Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:571: Session [2:7587241707147096640:2048] [node 1] ICS25 shutdown socket, reason# EndOfStream
2025-12-24T02:13:00.590446Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:581: [1] disconnected
2025-12-24T02:13:00.590495Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:524: Session [2:7587241707147096640:2048] [node 1] ICS15 start handshake
2025-12-24T02:13:00.591252Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:409: Proxy [2:7587241707147096628:2048] [node 1] ICP25 outgoing handshake failed, temporary: 1 explanation: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19648) Couldn't connect to any resolved address incoming: [0:0:0] held: no
2025-12-24T02:13:00.591272Z :INTERCONNECT_SESSION INFO: interconnect_tcp_session.cpp:105: Session [2:7587241707147096640:2048] [node 1] ICS01 socket: -1 reason# HandshakeFailPermanent
2025-12-24T02:13:00.591278Z :INTERCONNECT INFO: interconnect_tcp_proxy.cpp:543: Proxy [2:7587241707147096628:2048] [node 1] ICP30 unregister session Session# [2:7587241707147096640:2048] VirtualId# [2:7587241707147096638:0]
2025-12-24T02:13:00.591314Z :INTERCONNECT_STATUS INFO: interconnect_tcp_session.cpp:144: [1] session destroyed
2025-12-24T02:13:00.591331Z :INTERCONNECT NOTICE: interconnect_tcp_proxy.cpp:754: Proxy [2:7587241707147096628:2048] [node 1] ICP32 transit to hold-by-error state Explanation# outgoing handshake Peer# 127.1.0.0(127.1.0.0:19648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
2025-12-24T02:13:00.591339Z :INTERCONNECT_STATUS INFO: interconnect_tcp_proxy.cpp:755: [1] error state: outgoing handshake Peer# 127.1.0.0(127.1.0.0:19648) Couldn't connect to any resolved address LastSessionDieTime# 1970-01-01T00:00:00.000000Z
